feat(nestjs-tools-file-storage): enable to serialize readDir output

packages/file-storage/src/lib/file-storage-fs.class.ts

@@ -1,4 +1,4 @@
-import { createReadStream, createWriteStream, ObjectEncodingOptions, stat, unlink } from 'node:fs';
+import { createReadStream, createWriteStream, Dirent, ObjectEncodingOptions, stat, unlink } from 'node:fs';
 import { access, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { normalize, resolve as resolvePath, sep } from 'node:path';
 import { finished, Readable } from 'node:stream';
@@ -8,12 +8,13 @@ import type {
   FileStorageBaseArgs,
   FileStorageConfig,
   FileStorageConfigFactory,
-  FileStorageDirBaseArgs,
 } from './file-storage.class';
 import type {
+  FileStorageLocalDeleteDir,
   FileStorageLocalDownloadFile,
   FileStorageLocalDownloadStream,
   FileStorageLocalFileExists,
+  FileStorageLocalReadDir,
   FileStorageLocalSetup,
   FileStorageLocalUploadFile,
   FileStorageLocalUploadStream,
@@ -129,18 +130,28 @@ export class FileStorageLocal implements FileStorage {
     );
   }
 
-  async deleteDir(args: FileStorageDirBaseArgs): Promise<void> {
-    const { dirPath, request } = args;
+  async deleteDir(args: FileStorageLocalDeleteDir): Promise<void> {
+    const { options = { recursive: true, force: true }, dirPath, request } = args;
     const dirName = await this.transformFilePath(dirPath, MethodTypes.DELETE, request);
-    return rm(dirName, { recursive: true, force: true });
+    return rm(dirName, options);
   }
 
-  // TODO: indicate if the item is a file or a directory
-  async readDir(args: FileStorageDirBaseArgs): Promise<string[]> {
-    const { dirPath, request } = args;
+  async readDir<R = string>(args: FileStorageLocalReadDir<R>): Promise<R[]> {
+    type RawOutput = string[] | Buffer[] | Dirent[];
+    const defaultSerializer = (v: RawOutput) =>
+      v.map((val) => {
+        if (val instanceof Buffer) {
+          return val.toString() as unknown as R;
+        } else if (val instanceof Dirent) {
+          return val.name as unknown as R;
+        }
+        return val as unknown as R;
+      });
+    const { dirPath, request, serializer = defaultSerializer, options = {} } = args;
     try {
       const transformedDirPath = await this.transformFilePath(dirPath, MethodTypes.READ, request);
-      return await readdir(transformedDirPath);
+      const result = (await readdir(transformedDirPath, options)) as string[] | Buffer[] | Dirent[];
+      return serializer(result);
     } catch (err) {
       if (err && typeof err === 'object' && 'code' in err && err['code'] === 'ENOENT') {
         return [];

packages/file-storage/src/lib/file-storage-fs.types.ts

@@ -1,6 +1,6 @@
 import type { BigIntOptions, ObjectEncodingOptions, StatOptions, WriteFileOptions } from 'node:fs';
 
-import type { FileStorageBaseArgs } from './file-storage.class';
+import type { FileStorageBaseArgs, FileStorageDirBaseArgs, FileStorageReadDirBaseArgs } from './file-storage.class';
 
 export type StreamOptions = {
   flags?: string;
@@ -51,3 +51,11 @@ export interface FileStorageLocalDownloadFile extends FileStorageBaseArgs {
 export interface FileStorageLocalDownloadStream extends FileStorageBaseArgs {
   options?: BufferEncoding | StreamOptions;
 }
+
+export interface FileStorageLocalDeleteDir extends FileStorageDirBaseArgs {
+  options?: { recursive?: boolean; force?: boolean };
+}
+
+export interface FileStorageLocalReadDir<R = string> extends FileStorageReadDirBaseArgs<R> {
+  options?: { encoding: BufferEncoding; withFileTypes?: boolean; recursive?: boolean } | BufferEncoding;
+}

packages/file-storage/src/lib/file-storage-google.class.ts

@@ -1,3 +1,4 @@
+import type { File } from '@google-cloud/storage';
 import { Injectable } from '@nestjs/common';
 import { finished, type Readable } from 'node:stream';
 
@@ -38,7 +39,6 @@ function config(setup: FileStorageGoogleSetup) {
   };
 }
 
-// TODO: import @google-cloud/storage dynamically
 @Injectable()
 export class FileStorageGoogle implements FileStorage {
   readonly config: FileStorageConfig & FileStorageGoogleConfig;
@@ -131,14 +131,17 @@ export class FileStorageGoogle implements FileStorage {
     await storage.bucket(bucket).deleteFiles({ ...options, prefix });
   }
 
-  // TODO: make filepaths compliant with the other readDir implementations
-  async readDir(args: FileStorageGoogleReadDir): Promise<string[]> {
+  // TODO: make default serializer compliant with the other readDir implementations
+  async readDir<R = string>(args: FileStorageGoogleReadDir<R>): Promise<R[]> {
+    const defaultSerializer = (files: File[]) =>
+      files.map((f) => (prefix ? (f.name.replace(prefix, '').replace('/', '') as R) : (f.name as R)));
+
     const { storage, bucket } = this.config;
-    const { options = {}, request } = args;
-    const prefix = await this.transformFilePath(args.dirPath, MethodTypes.READ, request, options);
+    const { dirPath, request, serializer = defaultSerializer, options = {} } = args;
+    const prefix = await this.transformFilePath(dirPath, MethodTypes.READ, request, options);
     const [files] = await storage
       .bucket(bucket)
       .getFiles({ includeTrailingDelimiter: false, includeFoldersAsPrefixes: false, ...options, prefix });
-    return files.map((file) => (prefix ? file.name.replace(prefix, '').replace('/', '') : file.name));
+    return serializer(files);
   }
 }

packages/file-storage/src/lib/file-storage-google.types.ts

@@ -11,7 +11,7 @@ import type {
 } from '@google-cloud/storage';
 import type { DeleteOptions, ExistsOptions } from '@google-cloud/storage/build/cjs/src/nodejs-common/service-object';
 
-import type { FileStorageBaseArgs, FileStorageDirBaseArgs } from './file-storage.class';
+import type { FileStorageBaseArgs, FileStorageDirBaseArgs, FileStorageReadDirBaseArgs } from './file-storage.class';
 
 // TODO: add authentication options
 export interface FileStorageGoogleSetup {
@@ -61,6 +61,6 @@ export interface FileStorageGoogleDeleteDir extends FileStorageDirBaseArgs {
   options?: Omit<DeleteFilesOptions, 'prefix'> & FileOptions;
 }
 
-export interface FileStorageGoogleReadDir extends FileStorageDirBaseArgs {
+export interface FileStorageGoogleReadDir<R = string> extends FileStorageReadDirBaseArgs<R> {
   options?: Omit<GetFilesOptions, 'prefix'> & FileOptions;
 }

packages/file-storage/src/lib/file-storage-s3.class.ts

@@ -1,13 +1,8 @@
-import type { DeleteObjectsCommandInput, ListObjectsCommandInput } from '@aws-sdk/client-s3';
+import type { DeleteObjectsCommandInput, ListObjectsCommandInput, ListObjectsCommandOutput } from '@aws-sdk/client-s3';
 import type { Options as UploadOptions } from '@aws-sdk/lib-storage';
 import { PassThrough, Readable } from 'node:stream';
 
-import type {
-  FileStorage,
-  FileStorageConfig,
-  FileStorageConfigFactory,
-  FileStorageDirBaseArgs,
-} from './file-storage.class';
+import type { FileStorage, FileStorageConfig, FileStorageConfigFactory } from './file-storage.class';
 import type {
   FileStorageS3Config,
   FileStorageS3DeleteDir,
@@ -16,6 +11,7 @@ import type {
   FileStorageS3DownloadStream,
   FileStorageS3FileExists,
   FileStorageS3MoveFile,
+  FileStorageS3ReadDir,
   FileStorageS3Setup,
   FileStorageS3UploadFile,
   FileStorageS3UploadStream,
@@ -63,7 +59,6 @@ function addTrailingForwardSlash(x: string) {
 }
 
 // TODO: control filesize limit
-// TODO: import @aws-sdk/client-s3 and @aws-sdk/lib-storage dynamically
 export class FileStorageS3 implements FileStorage {
   readonly config: FileStorageConfig & FileStorageS3Config;
 
@@ -201,12 +196,34 @@ export class FileStorageS3 implements FileStorage {
     }
   }
 
-  // TODO: indicate if the item is a file or a directory
-  async readDir(args: FileStorageDirBaseArgs): Promise<string[]> {
-    const { dirPath, request } = args;
+  async readDir<R = string>(args: FileStorageS3ReadDir<R>): Promise<R[]> {
+    const defaultSerializer = (list: ListObjectsCommandOutput) => {
+      const { CommonPrefixes, Contents, Prefix } = list;
+      const filesAndFilders: R[] = [];
+      //  add nested folders, CommonPrefixes contains <prefix>/<next nested dir>
+      if (CommonPrefixes?.length) {
+        const folders = CommonPrefixes.map((prefixObject) => {
+          const prefix = removeTrailingForwardSlash(prefixObject.Prefix) ?? '';
+          const key = listParams['Prefix'];
+          // If key exists, we are looking for a nested folder
+          return (key ? prefix.slice(key.length) : prefix) as R;
+        });
+        filesAndFilders.push(...folders);
+      }
+
+      // adds filenames
+      if (Contents?.length && Prefix) {
+        const files = Contents.filter((file) => !!file.Key).map((file) => file.Key?.replace(Prefix, '') as R);
+        filesAndFilders.push(...files);
+      }
+      return filesAndFilders;
+    };
+
+    const { dirPath, request, serializer = defaultSerializer, options = {} } = args;
     const { s3, bucket: Bucket } = this.config;
     const Key = await this.transformFilePath(dirPath, MethodTypes.READ, request);
     const listParams: ListObjectsCommandInput = {
+      ...options,
       Bucket,
       Delimiter: '/',
     };
@@ -215,25 +232,6 @@ export class FileStorageS3 implements FileStorage {
       listParams.Prefix = addTrailingForwardSlash(Key);
     }
     const listedObjects = await s3.listObjects(listParams);
-    const filesAndFilders: string[] = [];
-    //  add nested folders, CommonPrefixes contains <prefix>/<next nested dir>
-    if (listedObjects.CommonPrefixes?.length) {
-      const folders = listedObjects.CommonPrefixes.map((prefixObject) => {
-        const prefix = removeTrailingForwardSlash(prefixObject.Prefix) ?? '';
-        const key = listParams['Prefix'];
-        // If key exists, we are looking for a nested folder
-        return key ? prefix.slice(key.length) : prefix;
-      });
-      filesAndFilders.push(...folders);
-    }
-
-    // adds filenames
-    if (listedObjects.Contents?.length && listedObjects.Prefix) {
-      const files = listedObjects.Contents.filter((file) => !!file.Key).map((file) =>
-        file.Key?.replace(listedObjects.Prefix as string, ''),
-      ) as string[];
-      filesAndFilders.push(...files);
-    }
-    return filesAndFilders;
+    return serializer(listedObjects);
   }
 }

packages/file-storage/src/lib/file-storage-s3.types.ts

@@ -3,11 +3,12 @@ import type {
   DeleteObjectsCommandInput,
   GetObjectCommandInput,
   HeadObjectCommandInput,
+  ListObjectsCommandInput,
   PutObjectCommandInput,
   S3,
 } from '@aws-sdk/client-s3';
 
-import type { FileStorageBaseArgs, FileStorageDirBaseArgs } from './file-storage.class';
+import type { FileStorageBaseArgs, FileStorageDirBaseArgs, FileStorageReadDirBaseArgs } from './file-storage.class';
 
 /**
  * Either region or endpoint must be provided
@@ -64,3 +65,7 @@ export interface FileStorageS3DeleteFile extends FileStorageBaseArgs {
 export interface FileStorageS3DeleteDir extends FileStorageDirBaseArgs {
   options?: Omit<DeleteObjectsCommandInput, 'Bucket' | 'Delete'>;
 }
+
+export interface FileStorageS3ReadDir<R = string> extends FileStorageReadDirBaseArgs<R> {
+  options?: Omit<ListObjectsCommandInput, 'Bucket' | 'Delimiter'>;
+}

packages/file-storage/src/lib/file-storage.class.ts

@@ -30,6 +30,10 @@ export interface FileStorageDirBaseArgs {
   request?: Request;
 }
 
+export interface FileStorageReadDirBaseArgs<R = string> extends FileStorageDirBaseArgs {
+  serializer?: <T>(data: T) => R[];
+}
+
 export type FileStorageTransformPath = (
   fileName: string,
   methodType: MethodTypes,
@@ -104,7 +108,7 @@ export abstract class FileStorage {
     throw new Error(defaultErrorMessage);
   }
 
-  readDir(args: FileStorageDirBaseArgs): Promise<string[]> {
+  readDir<R = string>(args: FileStorageReadDirBaseArgs<R>): Promise<R[]> {
     throw new Error(defaultErrorMessage);
   }
 }

packages/file-storage/src/lib/file-storage.service.ts

@@ -7,6 +7,7 @@ import type {
   FileStorageLocalDownloadStream,
   FileStorageLocalFileExists,
   FileStorageLocalMoveFile,
+  FileStorageLocalReadDir,
   FileStorageLocalUploadFile,
   FileStorageLocalUploadStream,
 } from './file-storage-fs.types';
@@ -28,6 +29,7 @@ import type {
   FileStorageS3DownloadStream,
   FileStorageS3FileExists,
   FileStorageS3MoveFile,
+  FileStorageS3ReadDir,
   FileStorageS3UploadFile,
   FileStorageS3UploadStream,
 } from './file-storage-s3.types';
@@ -74,7 +76,9 @@ export class FileStorageService implements Omit<FileStorage, 'transformFilePath'
     return this.fileStorage.deleteFile(args);
   }
 
-  readDir(args: FileStorageDirBaseArgs | FileStorageGoogleReadDir): Promise<string[]> {
+  readDir<R = string>(
+    args: FileStorageLocalReadDir<R> | FileStorageS3ReadDir<R> | FileStorageGoogleReadDir<R>,
+  ): Promise<R[]> {
     return this.fileStorage.readDir(args);
   }