test: data-driven generator test system

docs/development/generation-testing.md

@@ -0,0 +1,29 @@
+# Generation Testing
+
+Generation tests are data-driven instead of being specified explicitly. This document explains how to add a new
+generation test.
+
+## Adding a generation test
+
+1. Create a new **folder** (not just a file!) in the `tests/resources/generation` directory or any subdirectory. Give
+   the folder a descriptive name, since the folder name becomes part of the test name.
+
+    !!! tip "Skipping a test"
+
+        If you want to skip a test, add the prefix `skip-` to the folder name.
+
+2. Add files with the extension `.sdstest`, `.sdspipe`, or `.sdsstub` **directly inside the folder**. All files in a
+   folder will be loaded into the same workspace, so they can reference each other. Files in different folders are
+   loaded into different workspaces, so they cannot reference each other. Generation will be triggered for all files in
+   the folder.
+3. Add the Safe-DS code that you want to test to the files.
+4. If you want to run the program only up to a specific placeholder of a pipeline, surround **the name** of that
+   placeholder with test markers, e.g. `val »a« = 1;`. You may only mark a single placeholder this way. Add a comment in
+   the preceding line with the following format:
+    ```ts
+    // $TEST$ run_until
+    ```
+5. Add another folder called `output` inside the folder that you created in step 1. Place folders and Python files
+   inside the `output` folder to specify the expected output of the program. The relative paths to the Python files and
+   the contents of the Python files will be compared to the actual generation output.
+6. Run the tests. The test runner will automatically pick up the new test.

docs/development/partial-evaluation-testing.md

@@ -15,7 +15,7 @@ partial evaluation test.
 2. Add files with the extension `.sdstest` **directly inside the folder**. All files in a folder will be loaded into the
    same workspace, so they can reference each other. Files in different folders are loaded into different workspaces, so
    they cannot reference each other.
-3. Add the Safe-DS code that you want to test to the file.
+3. Add the Safe-DS code that you want to test to the files.
 4. Surround entire nodes whose value you want to check with test markers, e.g. `1 + 2`.
 5. For each pair of test markers, add a test comment with one of the formats listed below. Test comments and test
    markers are mapped to each other by their position in the file, i.e. the first test comment corresponds to the first

docs/development/scoping-testing.md

@@ -15,7 +15,7 @@ test.
 2. Add files with the extension `.sdstest` **directly inside
    the folder**. All files in a folder will be loaded into the same workspace, so they can
    reference each other. Files in different folders are loaded into different workspaces, so they cannot reference each other.
-3. Add the Safe-DS code that you want to test to the file.
+3. Add the Safe-DS code that you want to test to the files.
 4. Surround **the name** of any declaration that you want to reference with test markers, e.g. `class »C«`. Add a
    comment in the preceding line with the following format (replace `<id>` with some unique identifier):
     ```ts

docs/development/typing-testing.md

@@ -15,7 +15,7 @@ test.
 2. Add files with the extension `.sdstest` **directly inside the folder**. All files in a folder will be loaded into the
    same workspace, so they can reference each other. Files in different folders are loaded into different workspaces, so
    they cannot reference each other.
-3. Add the Safe-DS code that you want to test to the file.
+3. Add the Safe-DS code that you want to test to the files.
 4. Surround entire nodes whose type you want to check with test markers, e.g. `1 + 2`. For declarations, it is also
    possible to surround only their name, e.g. `class »C«`.
 5. For each pair of test markers, add a test comment with one of the formats listed below. Test comments and test

docs/development/validation-testing.md

@@ -15,7 +15,7 @@ validation test.
 2. Add files with the extension `.sdstest`, `.sdspipe`, or `.sdsstub` **directly inside the folder**. All files in a
    folder will be loaded into the same workspace, so they can reference each other. Files in different folders are
    loaded into different workspaces, so they cannot reference each other.
-3. Add the Safe-DS code that you want to test to the file.
+3. Add the Safe-DS code that you want to test to the files.
 4. Specify the expected validation results using test comments (see [below](#format-of-test-comments)) and test
    markers (e.g. `fun »F«()`). The test comments are used to specify
     * the presence or absence of an issue,

mkdocs.yml

@@ -41,6 +41,7 @@ nav:
       - Typing Testing: development/typing-testing.md
       - Partial Evaluation Testing: development/partial-evaluation-testing.md
       - Validation Testing: development/validation-testing.md
+      - Generation Testing: development/generation-testing.md
       - Formatting Testing: development/formatting-testing.md
       - Langium Quickstart: development/langium-quickstart.md
 

src/cli/cli-util.ts

@@ -3,6 +3,14 @@ import path from 'path';
 import fs from 'fs';
 import { AstNode, LangiumDocument, LangiumServices, URI } from 'langium';
 
+/* c8 ignore start */
+export const extractAstNode = async function <T extends AstNode>(
+    fileName: string,
+    services: LangiumServices,
+): Promise<T> {
+    return (await extractDocument(fileName, services)).parseResult?.value as T;
+};
+
 export const extractDocument = async function (fileName: string, services: LangiumServices): Promise<LangiumDocument> {
     const extensions = services.LanguageMetaData.fileExtensions;
     if (!extensions.includes(path.extname(fileName))) {
@@ -20,11 +28,11 @@ export const extractDocument = async function (fileName: string, services: Langi
     const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
     await services.shared.workspace.DocumentBuilder.build([document], { validation: true });
 
-    const validationErrors = (document.diagnostics ?? []).filter((e) => e.severity === 1);
-    if (validationErrors.length > 0) {
+    const errors = (document.diagnostics ?? []).filter((e) => e.severity === 1);
+    if (errors.length > 0) {
         // eslint-disable-next-line no-console
-        console.error(chalk.red('There are validation errors:'));
-        for (const validationError of validationErrors) {
+        console.error(chalk.red(`The document ${fileName} has errors:`));
+        for (const validationError of errors) {
             // eslint-disable-next-line no-console
             console.error(
                 chalk.red(
@@ -39,13 +47,7 @@ export const extractDocument = async function (fileName: string, services: Langi
 
     return document;
 };
-
-export const extractAstNode = async function <T extends AstNode>(
-    fileName: string,
-    services: LangiumServices,
-): Promise<T> {
-    return (await extractDocument(fileName, services)).parseResult?.value as T;
-};
+/* c8 ignore stop */
 
 interface FilePathData {
     destination: string;

src/cli/generator.ts

@@ -7,19 +7,25 @@ import chalk from 'chalk';
 import { createSafeDsServices } from '../language/safe-ds-module.js';
 import { NodeFileSystem } from 'langium/node';
 
+/* c8 ignore start */
 export const generateAction = async (fileName: string, opts: GenerateOptions): Promise<void> => {
     const services = createSafeDsServices(NodeFileSystem).SafeDs;
     const module = await extractAstNode<SdsModule>(fileName, services);
     const generatedFilePath = generatePython(module, fileName, opts.destination);
     // eslint-disable-next-line no-console
     console.log(chalk.green(`Python code generated successfully: ${generatedFilePath}`));
 };
+/* c8 ignore stop */
 
 export type GenerateOptions = {
     destination?: string;
 };
 
-export const generatePython = function (module: SdsModule, filePath: string, destination: string | undefined): string {
+export const generatePython = function (
+    module: SdsModule,
+    filePath: string,
+    destination: string | undefined,
+): string[] {
     const data = extractDestinationAndName(filePath, destination);
     const generatedFilePath = `${path.join(data.destination, data.name)}.py`;
 
@@ -31,5 +37,5 @@ export const generatePython = function (module: SdsModule, filePath: string, des
         fs.mkdirSync(data.destination, { recursive: true });
     }
     fs.writeFileSync(generatedFilePath, toString(fileNode));
-    return generatedFilePath;
+    return [generatedFilePath];
 };

tests/helpers/diagnostics.ts

@@ -12,12 +12,8 @@ let nextId = 0;
  * @returns The syntax errors.
  */
 export const getSyntaxErrors = async (services: LangiumServices, code: string): Promise<Diagnostic[]> => {
-    const diagnostics = await getDiagnostics(services, code);
-    return diagnostics.filter(
-        (d) =>
-            d.severity === DiagnosticSeverity.Error &&
-            (d.data?.code === 'lexing-error' || d.data?.code === 'parsing-error'),
-    );
+    const diagnostics = await getErrors(services, code);
+    return diagnostics.filter((d) => d.data?.code === 'lexing-error' || d.data?.code === 'parsing-error');
 };
 
 /**
@@ -28,8 +24,20 @@ export const getSyntaxErrors = async (services: LangiumServices, code: string):
  * @returns The errors.
  */
 export const getLinkingErrors = async (services: LangiumServices, code: string): Promise<Diagnostic[]> => {
+    const diagnostics = await getErrors(services, code);
+    return diagnostics.filter((d) => d.data?.code === 'linking-error');
+};
+
+/**
+ * Get all errors from a code snippet.
+ *
+ * @param services The language services.
+ * @param code The code snippet to check.
+ * @returns The errors.
+ */
+export const getErrors = async (services: LangiumServices, code: string): Promise<Diagnostic[]> => {
     const diagnostics = await getDiagnostics(services, code);
-    return diagnostics.filter((d) => d.severity === DiagnosticSeverity.Error && d.data?.code === 'linking-error');
+    return diagnostics.filter((d) => d.severity === DiagnosticSeverity.Error);
 };
 
 /**
@@ -53,8 +61,19 @@ const getDiagnostics = async (services: LangiumServices, code: string): Promise<
  */
 export class SyntaxErrorsInCodeError extends Error {
     constructor(readonly syntaxErrors: Diagnostic[]) {
-        const syntaxErrorsAsString = syntaxErrors.map((e) => `- ${e.message}`).join(`\n`);
+        const syntaxErrorsAsString = syntaxErrors.map((e) => `    - ${e.message}`).join(`\n`);
 
         super(`Code has syntax errors:\n${syntaxErrorsAsString}`);
     }
 }
+
+/**
+ * The code contains syntax errors.
+ */
+export class ErrorsInCodeError extends Error {
+    constructor(readonly errors: Diagnostic[]) {
+        const syntaxErrorsAsString = errors.map((e) => `    - ${e.message}`).join(`\n`);
+
+        super(`Code has errors:\n${syntaxErrorsAsString}`);
+    }
+}

tests/helpers/testDescription.ts

@@ -0,0 +1,14 @@
+/**
+ * A description of a test. This interface should be extended to describe tests of specific components.
+ */
+export interface TestDescription {
+    /**
+     * The name of the test.
+     */
+    testName: string;
+
+    /**
+     * An error that occurred while creating the test. If this is undefined, the test is valid.
+     */
+    error?: Error;
+}

tests/helpers/testResources.test.ts

@@ -1,9 +1,9 @@
 import { describe, expect, it } from 'vitest';
-import { listTestResources, listTestsResourcesGroupedByParentDirectory } from './testResources.js';
+import { listSafeDSResources, listTestsResourcesGroupedByParentDirectory } from './testResources.js';
 
 describe('listTestResources', () => {
     it('should yield all Safe-DS files in a directory that are not skipped', () => {
-        const result = listTestResources('helpers/listTestResources');
+        const result = listSafeDSResources('helpers/listTestResources');
         const expected = [
             'pipeline file.sdspipe',
             'stub file.sdsstub',

tests/helpers/testResources.ts

@@ -1,6 +1,6 @@
 import path from 'path';
 import { globSync } from 'glob';
-import { SAFE_DS_FILE_EXTENSIONS } from '../../src/language/helpers/fileExtensions';
+import { SAFE_DS_FILE_EXTENSIONS } from '../../src/language/helpers/fileExtensions.js';
 import { group } from 'radash';
 
 const resourcesPath = path.join(__dirname, '..', 'resources');
@@ -21,13 +21,26 @@ export const resolvePathRelativeToResources = (pathRelativeToResources: string)
  * @param pathRelativeToResources The root directory relative to `tests/resources/`.
  * @return Paths to the Safe-DS files relative to `pathRelativeToResources`.
  */
-export const listTestResources = (pathRelativeToResources: string): string[] => {
+export const listSafeDSResources = (pathRelativeToResources: string): string[] => {
     const pattern = `**/*.{${SAFE_DS_FILE_EXTENSIONS.join(',')}}`;
     const cwd = resolvePathRelativeToResources(pathRelativeToResources);
 
     return globSync(pattern, { cwd, nodir: true }).filter(isNotSkipped);
 };
 
+/**
+ * Lists all Python files in the given directory relative to `tests/resources/`.
+ *
+ * @param pathRelativeToResources The root directory relative to `tests/resources/`.
+ * @return Paths to the Python files relative to `pathRelativeToResources`.
+ */
+export const listPythonResources = (pathRelativeToResources: string): string[] => {
+    const pattern = `**/*.py`;
+    const cwd = resolvePathRelativeToResources(pathRelativeToResources);
+
+    return globSync(pattern, { cwd, nodir: true });
+};
+
 /**
  * Lists all Safe-DS files in the given directory relative to `tests/resources/` that are not skipped. The result is
  * grouped by the parent directory.
@@ -38,7 +51,7 @@ export const listTestResources = (pathRelativeToResources: string): string[] =>
 export const listTestsResourcesGroupedByParentDirectory = (
     pathRelativeToResources: string,
 ): Record<string, string[]> => {
-    const paths = listTestResources(pathRelativeToResources);
+    const paths = listSafeDSResources(pathRelativeToResources);
     return group(paths, (p) => path.dirname(p)) as Record<string, string[]>;
 };
 

tests/language/formatting/creator.ts

@@ -1,17 +1,18 @@
-import { listTestResources, resolvePathRelativeToResources } from '../../helpers/testResources.js';
+import { listSafeDSResources, resolvePathRelativeToResources } from '../../helpers/testResources.js';
 import path from 'path';
 import fs from 'fs';
 import { Diagnostic } from 'vscode-languageserver-types';
 import { createSafeDsServices } from '../../../src/language/safe-ds-module.js';
 import { EmptyFileSystem } from 'langium';
 import { getSyntaxErrors } from '../../helpers/diagnostics.js';
+import { TestDescription } from '../../helpers/testDescription.js';
 
 const services = createSafeDsServices(EmptyFileSystem).SafeDs;
 const root = 'formatting';
 const separator = '// -----------------------------------------------------------------------------';
 
 export const createFormattingTests = async (): Promise<FormattingTest[]> => {
-    const testCases = listTestResources(root).map(createFormattingTest);
+    const testCases = listSafeDSResources(root).map(createFormattingTest);
     return Promise.all(testCases);
 };
 
@@ -53,12 +54,12 @@ const createFormattingTest = async (relativeResourcePath: string): Promise<Forma
 /**
  * Report a test that has errors.
  *
- * @param pathRelativeToResources The path to the test file relative to the resources directory.
+ * @param relativeResourcePath The path to the test file relative to the `resources` directory.
  * @param error The error that occurred.
  */
-const invalidTest = (pathRelativeToResources: string, error: Error): FormattingTest => {
+const invalidTest = (relativeResourcePath: string, error: Error): FormattingTest => {
     return {
-        testName: `INVALID TEST FILE [${pathRelativeToResources}]`,
+        testName: `INVALID TEST FILE [${relativeResourcePath}]`,
         originalCode: '',
         expectedFormattedCode: '',
         error,
@@ -78,12 +79,7 @@ const normalizeLineBreaks = (code: string): string => {
 /**
  * A description of a formatting test.
  */
-interface FormattingTest {
-    /**
-     * The name of the test.
-     */
-    testName: string;
-
+interface FormattingTest extends TestDescription {
     /**
      * The original code before formatting.
      */
@@ -93,11 +89,6 @@ interface FormattingTest {
      * The expected formatted code.
      */
     expectedFormattedCode: string;
-
-    /**
-     * An error that occurred while creating the test. If this is undefined, the test is valid.
-     */
-    error?: Error;
 }
 
 /**