feat: support preprocess code (#103)

* feat: support preprocess code * demo: use preprocessCode to replace custom params with dt syntax * feat: use _defaults.preprocessCode
DTStack · Mar 29, 2024 · 51777c6 · 51777c6
1 parent 7a10a6a
commit 51777c6
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 19 deletions.
diff --git a/src/_.contribution.ts b/src/_.contribution.ts
@@ -178,12 +178,19 @@ export type CompletionService = (
 	entities: EntityContext[] | null
 ) => Promise<ICompletionItem[] | ICompletionList>;
 
+/**
+ * A function to preprocess code.
+ * @param code editor value
+ */
+export type PreprocessCode = (code: string) => string;
+
 export interface LanguageServiceDefaults {
 	readonly languageId: string;
 	readonly onDidChange: IEvent<LanguageServiceDefaults>;
 	readonly diagnosticsOptions: DiagnosticsOptions;
 	readonly modeConfiguration: ModeConfiguration;
 	readonly completionService?: CompletionService;
+	readonly preprocessCode?: PreprocessCode;
 	setDiagnosticsOptions(options: DiagnosticsOptions): void;
 	setModeConfiguration(modeConfiguration: ModeConfiguration): void;
 }
@@ -194,17 +201,20 @@ export class LanguageServiceDefaultsImpl implements LanguageServiceDefaults {
 	private _modeConfiguration!: ModeConfiguration;
 	private _languageId: string;
 	private _completionService?: CompletionService;
+	private _preprocessCode?: PreprocessCode;
 
 	constructor(
 		languageId: string,
 		diagnosticsOptions: DiagnosticsOptions,
 		modeConfiguration: ModeConfiguration,
-		completionService?: CompletionService
+		completionService?: CompletionService,
+		preprocessCode?: PreprocessCode
 	) {
 		this._languageId = languageId;
 		this.setDiagnosticsOptions(diagnosticsOptions);
 		this.setModeConfiguration(modeConfiguration);
 		this._completionService = completionService;
+		this._preprocessCode = preprocessCode;
 	}
 
 	get onDidChange(): IEvent<LanguageServiceDefaults> {
@@ -227,6 +237,10 @@ export class LanguageServiceDefaultsImpl implements LanguageServiceDefaults {
 		return this._completionService;
 	}
 
+	get preprocessCode(): PreprocessCode | undefined {
+		return this._preprocessCode;
+	}
+
 	setDiagnosticsOptions(options: DiagnosticsOptions): void {
 		this._diagnosticsOptions = options || Object.create(null);
 		this._onDidChange.fire(this);

diff --git a/src/languageFeatures.ts b/src/languageFeatures.ts
@@ -9,9 +9,9 @@ import {
 	CancellationToken
 } from './fillers/monaco-editor-core';
 import { debounce } from './common/utils';
+import { BaseSQLWorker } from './baseSQLWorker';
 import type { ParseError } from 'dt-sql-parser';
 import type { LanguageServiceDefaults, CompletionService, ICompletionItem } from './_.contribution';
-import { BaseSQLWorker } from './baseSQLWorker';
 
 export interface WorkerAccessor<T extends BaseSQLWorker> {
 	(first: Uri, ...more: Uri[]): Promise<T>;
@@ -24,7 +24,7 @@ export class DiagnosticsAdapter<T extends BaseSQLWorker> {
 	constructor(
 		private _languageId: string,
 		private _worker: WorkerAccessor<T>,
-		defaults: LanguageServiceDefaults
+		private _defaults: LanguageServiceDefaults
 	) {
 		const onModelAdd = (model: editor.IModel): void => {
 			let modeId = model.getLanguageId();
@@ -61,7 +61,7 @@ export class DiagnosticsAdapter<T extends BaseSQLWorker> {
 			})
 		);
 
-		defaults.onDidChange((_) => {
+		this._defaults.onDidChange((_) => {
 			editor.getModels().forEach((model) => {
 				if (model.getLanguageId() === this._languageId) {
 					onModelRemoved(model);
@@ -89,7 +89,11 @@ export class DiagnosticsAdapter<T extends BaseSQLWorker> {
 	private _doValidate(resource: Uri, languageId: string): void {
 		this._worker(resource)
 			.then((worker) => {
-				return worker.doValidation(editor.getModel(resource)?.getValue() || '');
+				let code = editor.getModel(resource)?.getValue() || '';
+				if (typeof this._defaults.preprocessCode === 'function') {
+					code = this._defaults.preprocessCode(code);
+				}
+				return worker.doValidation(code);
 			})
 			.then((diagnostics) => {
 				const markers = diagnostics.map((d) => toDiagnostics(resource, d));
@@ -146,10 +150,11 @@ export class CompletionAdapter<T extends BaseSQLWorker>
 		const resource = model.uri;
 		return this._worker(resource)
 			.then((worker) => {
-				return worker.doCompletionWithEntities(
-					editor.getModel(resource)?.getValue() || '',
-					position
-				);
+				let code = editor.getModel(resource)?.getValue() || '';
+				if (typeof this._defaults.preprocessCode === 'function') {
+					code = this._defaults.preprocessCode(code);
+				}
+				return worker.doCompletionWithEntities(code, position);
 			})
 			.then(([suggestions, allEntities]) => {
 				const completionService =

diff --git a/src/setupLanguageFeatures.ts b/src/setupLanguageFeatures.ts
@@ -1,6 +1,7 @@
 import {
 	CompletionService,
 	diagnosticDefault,
+	PreprocessCode,
 	LanguageServiceDefaults,
 	LanguageServiceDefaultsImpl,
 	modeConfigurationDefault
@@ -27,6 +28,11 @@ export interface FeatureConfiguration {
 	 * By default, only keyword autocomplete items are included.
 	 */
 	completionService?: CompletionService;
+	/**
+	 * Define a function to preprocess code.
+	 * By default, do not something.
+	 */
+	preprocessCode?: PreprocessCode;
 }
 
 const disposableMap = new Map<LanguageId, IDisposable>();
@@ -38,13 +44,15 @@ export function setupLanguageFeatures(configuration: FeatureConfiguration) {
 		return;
 	}
 
-	const { languageId, completionService, ...rest } = processConfiguration(configuration);
+	const { languageId, completionService, preprocessCode, ...rest } =
+		processConfiguration(configuration);
 
 	const defaults: LanguageServiceDefaults = new LanguageServiceDefaultsImpl(
 		languageId,
 		diagnosticDefault,
 		Object.assign({}, modeConfigurationDefault, rest),
-		completionService
+		completionService,
+		preprocessCode
 	);
 
 	function setup() {

diff --git a/website/src/languages/index.ts b/website/src/languages/index.ts
@@ -5,38 +5,100 @@ import { setupLanguageFeatures, LanguageIdEnum } from 'monaco-sql-languages/out/
 
 import { completionService } from './helpers/completionService';
 
+/**
+ * replace dtstack custom params, eg: @@{componentParams}, ${taskCustomParams}
+ * @param code editor value
+ * @returns replaced string
+ */
+const preprocessCode = (code: string): string => {
+	const regex1 = /@@{[A-Za-z0-9._-]*}/g;
+	const regex2 = /\${[A-Za-z0-9._-]*}/g;
+	let result = code;
+
+	if (regex1.test(code)) {
+		result = result.replace(regex1, (str) => {
+			return str.replace(/@|{|}|\.|-/g, '_');
+		});
+	}
+	if (regex2.test(code)) {
+		result = result.replace(regex2, (str) => {
+			return str.replace(/\$|{|}|\.|-/g, '_');
+		});
+	}
+	return result;
+};
+
+/**
+ * replace dtstack custom grammar, eg: @@{componentParams}, ${taskCustomParams}
+ * @param code editor value
+ * @param mark some sql grammar need special mark to replace the beginning and the end
+ * @returns replaced string
+ */
+const preprocessCodeHive = (code: string, mark?: string): string => {
+	const regex1 = /@@{[A-Za-z0-9._-]*}/g;
+	const regex2 = /\${[A-Za-z0-9._-]*}/g;
+	let result = code;
+
+	if (regex1.test(code)) {
+		result = result.replace(regex1, (str) => {
+			if (mark) {
+				return str
+					.replace(/@/, mark)
+					.replace(/}/, mark)
+					.replace(/@|{|\.|-/g, '_');
+			}
+			return str.replace(/@|{|}|\.|-/g, '_');
+		});
+	}
+	if (regex2.test(code)) {
+		result = result.replace(regex2, (str) => {
+			if (mark) {
+				return str.replace(/\$|}/g, mark).replace(/{|\.|-/g, '_');
+			}
+			return str.replace(/\$|{|}|\.|-/g, '_');
+		});
+	}
+	return result;
+};
+
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.FLINK,
-	completionService
+	completionService,
+	preprocessCode
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.SPARK,
-	completionService
+	completionService,
+	preprocessCode
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.HIVE,
-	completionService
+	completionService,
+	preprocessCode: (code: string) => preprocessCodeHive(code, '`')
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.MYSQL,
-	completionService
+	completionService,
+	preprocessCode
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.TRINO,
-	completionService
+	completionService,
+	preprocessCode
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.PG,
-	completionService
+	completionService,
+	preprocessCode
 });
 
 setupLanguageFeatures({
 	languageId: LanguageIdEnum.IMPALA,
-	completionService
+	completionService,
+	preprocessCode
 });
-