Skip to content

Commit

Permalink
test: run C3 e2e tests locally without having to deploy to cloudflare
Browse files Browse the repository at this point in the history 
This still tests that the frameworks are generated, compile and can be run locally;
while avoiding the time-const, flakiness, and denial of external contributors from running the tests.
  • Loading branch information
@petebacondarwin
petebacondarwin committed Sep 27, 2024
1 parent 93743af commit 4189054
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 92 deletions.
2 changes: 1 addition & 1 deletion packages/create-cloudflare/e2e-tests/fixtures/qwik/src/routes/test/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ export const onGet: RequestHandler = async ({ platform, json }) => {
return;
}

json(200, { value: platform.env["TEST"], success: true });
json(200, { value: (platform.env as any)["TEST"], success: true });
};
109 changes: 46 additions & 63 deletions packages/create-cloudflare/e2e-tests/frameworks.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import assert from "assert";
import { existsSync } from "fs";
import { cp } from "fs/promises";
import { join } from "path";
Expand All @@ -16,6 +15,7 @@ import {
getDiffsPath,
isQuarantineMode,
keys,
kill,
recreateDiffsFolder,
recreateLogFolder,
runC3,
Expand All @@ -28,13 +28,11 @@ import type { TemplateConfig } from "../src/templates";
import type { RunnerConfig } from "./helpers";
import type { JsonMap } from "@iarna/toml";
import type { Writable } from "stream";
import type { Suite } from "vitest";

const TEST_TIMEOUT = 1000 * 60 * 5;
const LONG_TIMEOUT = 1000 * 60 * 10;
const TEST_PM = process.env.TEST_PM ?? "";
const NO_DEPLOY = process.env.E2E_NO_DEPLOY ?? true;
// const NO_DEPLOY = process.env.E2E_NO_DEPLOY ?? false;
const TEST_RETRIES = process.env.E2E_RETRIES
? parseInt(process.env.E2E_RETRIES)
: 1;
Expand All @@ -43,10 +41,6 @@ type FrameworkTestConfig = RunnerConfig & {
testCommitMessage: boolean;
unsupportedPms?: string[];
unsupportedOSs?: string[];
verifyPreview: null | {
route: string;
expectedText: string;
};
verifyBuildCfTypes?: {
outputFile: string;
envInterfaceName: string;
Expand Down Expand Up @@ -231,19 +225,13 @@ function getFrameworkTests(opts: {
expectedText: "Welcome to Qwik",
},
verifyPreview: {
route: "/test",
expectedText: "C3_TEST",
route: "/",
expectedText: "Welcome to Qwik",
},
verifyBuildCfTypes: {
outputFile: "worker-configuration.d.ts",
envInterfaceName: "Env",
},
verifyBuild: {
outputDir: "./dist",
script: "build",
route: "/test",
expectedText: "C3_TEST",
},
},
remix: {
testCommitMessage: true,
Expand Down Expand Up @@ -428,31 +416,15 @@ describe.concurrent(
`E2E: Web frameworks (experimental:${experimental})`,
() => {
beforeAll(async (ctx) => {
recreateLogFolder({ experimental }, ctx as Suite);
recreateLogFolder({ experimental }, ctx);
recreateDiffsFolder({ experimental });
});

Object.keys(frameworkTests).forEach((frameworkId) => {
const frameworkConfig = frameworkMap[frameworkId];
const testConfig = frameworkTests[frameworkId];

const quarantineModeMatch =
isQuarantineMode() == (testConfig.quarantine ?? false);

// If the framework in question is being run in isolation, always run it.
// Otherwise, only run the test if it's configured `quarantine` value matches
// what is set in E2E_QUARANTINE
const frameworkToTest = getFrameworkToTest({ experimental });
let shouldRun = frameworkToTest
? frameworkToTest === frameworkId
: quarantineModeMatch;

// Skip if the package manager is unsupported
shouldRun &&= !testConfig.unsupportedPms?.includes(TEST_PM);

// Skip if the OS is unsupported
shouldRun &&= !testConfig.unsupportedOSs?.includes(process.platform);
test({ experimental }).runIf(shouldRun)(
test({ experimental }).runIf(shouldRunTest(frameworkId, testConfig))(
frameworkId,
async ({ logStream, project }) => {
if (!testConfig.verifyDeploy) {
Expand Down Expand Up @@ -506,7 +478,7 @@ describe.concurrent(
});
}

await verifyDevScript(
await verifyPreviewScript(
testConfig,
frameworkConfig,
project.path,
Expand Down Expand Up @@ -561,7 +533,10 @@ const runCli = async (
framework: string,
projectPath: string,
logStream: Writable,
{ argv = [], promptHandlers = [] }: RunnerConfig,
{
argv = [],
promptHandlers = [],
}: Pick<RunnerConfig, "argv" | "promptHandlers">,
) => {
const args = [
projectPath,
Expand Down Expand Up @@ -642,18 +617,12 @@ const verifyDeployment = async (
});
};

const verifyDevScript = async (
const verifyPreviewScript = async (
{ verifyPreview }: FrameworkTestConfig,
{ devScript, previewScript }: TemplateConfig,
{ previewScript }: TemplateConfig,
projectPath: string,
logStream: Writable,
) => {
if (!verifyPreview) {
return;
}

assert(devScript !== undefined, "Expected `devScript` to be defined");

if (!verifyPreview || !previewScript) {
return;
}
Expand All @@ -665,46 +634,39 @@ const verifyDevScript = async (
[
pm,
"run",
devScript,
previewScript,
...(pm === "npm" ? ["--"] : []),
"--port",
`${TEST_PORT}`,
],
{
cwd: projectPath,
env: {
NODE_ENV: "development",
VITEST: undefined,
},
},
logStream,
);

try {
// Retry requesting the test route from the dev-server
await retry({ times: 10 }, async () => {
await sleep(2000);
const res = await fetch(
`http://127.0.0.1:${TEST_PORT}${verifyPreview.route}`,
);
const body = await res.text();
if (!body.match(verifyPreview?.expectedText)) {
throw new Error("Expected text not found in response from dev-server.");
}
});
// Wait for the dev-server to be ready
await retry(
{ times: 20, sleepMs: 5000 },
async () =>
await fetch(`http://127.0.0.1:${TEST_PORT}${verifyPreview.route}`),
);

// Make a request to the specified test route
const res = await fetch(
`http://127.0.0.1:${TEST_PORT}${verifyPreview.route}`,
);
const body = await res.text();
expect(await res.text()).toContain(verifyPreview.expectedText);
} finally {
// Kill the process gracefully so ports can be cleaned up
await kill(proc);
// Wait for a second to allow process to exit cleanly. Otherwise, the port might
// end up camped and cause future runs to fail
await sleep(1000);
expect(body).toContain(verifyPreview.expectedText);
} finally {
// Kill the process gracefully so ports can be cleaned up
proc.kill("SIGINT");
}
};

Expand Down Expand Up @@ -789,11 +751,11 @@ const verifyBuildScript = async (
await sleep(7000);

// Make a request to the specified test route
const res = await fetch(`http://localhost:${TEST_PORT}${route}`);
const res = await fetch(`http://127.0.0.1:${TEST_PORT}${route}`);
const body = await res.text();

// Kill the process gracefully so ports can be cleaned up
devProc.kill("SIGINT");
await kill(devProc);

// Wait for a second to allow process to exit cleanly. Otherwise, the port might
// end up camped and cause future runs to fail
Expand All @@ -802,3 +764,24 @@ const verifyBuildScript = async (
// Verify expectation after killing the process so that it exits cleanly in case of failure
expect(body).toContain(expectedText);
};

function shouldRunTest(frameworkId: string, testConfig: FrameworkTestConfig) {
const quarantineModeMatch =
isQuarantineMode() == (testConfig.quarantine ?? false);

// If the framework in question is being run in isolation, always run it.
// Otherwise, only run the test if it's configured `quarantine` value matches
// what is set in E2E_QUARANTINE
const frameworkToTest = getFrameworkToTest({ experimental });
let shouldRun = frameworkToTest
? frameworkToTest === frameworkId
: quarantineModeMatch;

// Skip if the package manager is unsupported
shouldRun &&= !testConfig.unsupportedPms?.includes(TEST_PM);

// Skip if the OS is unsupported
shouldRun &&= !testConfig.unsupportedOSs?.includes(process.platform);

return shouldRun;
}
20 changes: 17 additions & 3 deletions packages/create-cloudflare/e2e-tests/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ import { setTimeout } from "timers/promises";
import { stripAnsi } from "@cloudflare/cli";
import { spawn } from "cross-spawn";
import { retry } from "helpers/retry";
import treeKill from "tree-kill";
import { fetch } from "undici";
import { expect, test as originalTest } from "vitest";
import { version } from "../package.json";
import type {
ChildProcess,
ChildProcessWithoutNullStreams,
SpawnOptionsWithoutStdio,
} from "child_process";
Expand All @@ -41,7 +43,7 @@ const testEnv = {
// do not use the same global cache and accidentally hit race conditions.
YARN_CACHE_FOLDER: "./.yarn/cache",
YARN_ENABLE_GLOBAL_CACHE: "false",
PNPM_HOME: "./.pnpm",
// PNPM_HOME: "./.pnpm",
npm_config_cache: "./.npm/cache",
// unset the VITEST env variable as this causes e2e issues with some frameworks
VITEST: undefined,
Expand All @@ -64,7 +66,11 @@ export type RunnerConfig = {
argv?: string[];
quarantine?: boolean;
timeout?: number;
verifyDeploy?: {
verifyDeploy: null | {
route: string;
expectedText: string;
};
verifyPreview: null | {
route: string;
expectedText: string;
};
Expand Down Expand Up @@ -361,7 +367,9 @@ export const testProjectDir = (suite: string, test: string) => {
const randomSuffix = crypto.randomBytes(4).toString("hex");
const baseProjectName = `${C3_E2E_PREFIX}${randomSuffix}`;

const getName = () => `${baseProjectName}-${test}`;
const getName = () =>
// Worker project names cannot be longer than 58 characters
`${baseProjectName}-${test.substring(0, 57 - baseProjectName.length)}`;
const getPath = () => path.join(tmpDirPath, getName());
const clean = () => {
try {
Expand Down Expand Up @@ -470,3 +478,9 @@ export const test = (opts: { experimental: boolean }) =>
logStream.close();
},
});

export function kill(proc: ChildProcess) {
return new Promise<void>(
(resolve) => proc.pid && treeKill(proc.pid, "SIGINT", () => resolve()),
);
}
Loading

0 comments on commit 4189054

Please sign in to comment.