From e13ab9782dc969c28f64309fe9e9e006e69a548a Mon Sep 17 00:00:00 2001 From: Sebastian Rettig Date: Sat, 10 Jul 2021 17:29:32 +0200 Subject: [PATCH 1/2] fix(h5p-mongos3): library storage includes library.json in exports --- .../h5p-mongos3/src/MongoLibraryStorage.ts | 71 +++++++++++++++++- .../h5p-mongos3/src/MongoS3LibraryStorage.ts | 72 +++++++++++++++++-- scripts/mongo-s3-docker-compose.yml | 9 +-- 3 files changed, 140 insertions(+), 12 deletions(-) diff --git a/packages/h5p-mongos3/src/MongoLibraryStorage.ts b/packages/h5p-mongos3/src/MongoLibraryStorage.ts index 3213043f3..a41362e83 100644 --- a/packages/h5p-mongos3/src/MongoLibraryStorage.ts +++ b/packages/h5p-mongos3/src/MongoLibraryStorage.ts @@ -474,9 +474,23 @@ export default class MongoLibraryStorage implements ILibraryStorage { } if (!fileData) { + // The library.json file is in the MongoDB document as binary JSON, + // so we get it from there + if (file === 'library.json') { + const metadata = JSON.stringify( + await this.getMetadata(library) + ); + const readable = new ReadableStreamBuffer(); + readable.put(metadata, 'utf-8'); + readable.stop(); + return readable; + } throw new H5pError( 'library-file-missing', - { ubername: LibraryName.toUberName(library), filename: file }, + { + ubername: LibraryName.toUberName(library), + filename: file + }, 404 ); } else { @@ -613,7 +627,12 @@ export default class MongoLibraryStorage implements ILibraryStorage { try { result = await this.mongodb.findOne( { _id: LibraryName.toUberName(library) }, - { projection: { metadata: 1, additionalMetadata: 1 } } + { + projection: { + metadata: 1, + additionalMetadata: 1 + } + } ); } catch (error) { throw new H5pError( @@ -725,7 +744,7 @@ export default class MongoLibraryStorage implements ILibraryStorage { ); } - files = result.files.map((f) => f.filename); + files = result.files.map((f) => f.filename).concat(['library.json']); log.debug(`Found ${files.length} file(s) in MongoDB.`); return files; } @@ -831,6 +850,52 @@ export default class MongoLibraryStorage implements ILibraryStorage { }); } + /** + * Gets the the metadata of a library. In contrast to getLibrary this is + * only the metadata. + * @param library the library + * @returns the metadata about the locally installed library + */ + private async getMetadata( + library: ILibraryName + ): Promise { + if (!library) { + throw new Error('You must pass in a library name to getLibrary.'); + } + let result; + + try { + result = await this.mongodb.findOne( + { _id: LibraryName.toUberName(library) }, + { + projection: { + metadata: 1 + } + } + ); + } catch (error) { + console.log(error); + throw new H5pError( + 'mongo-library-storage:error-getting-library-metadata', + { ubername: LibraryName.toUberName(library) } + ); + } + if (!result) { + throw new H5pError( + 'mongo-library-storage:library-not-found', + { ubername: LibraryName.toUberName(library) }, + 404 + ); + } + if (!result.metadata) { + throw new H5pError( + 'mongo-library-storage:error-getting-library-metadata', + { ubername: LibraryName.toUberName(library) } + ); + } + return result.metadata; + } + private async readableToBuffer(readable: Readable): Promise { return new Promise((resolve) => { const chunks = []; diff --git a/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts b/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts index ca6f04206..60ae3416d 100644 --- a/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts +++ b/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts @@ -5,7 +5,7 @@ import AWS from 'aws-sdk'; import { Readable } from 'stream'; import * as path from 'path'; import { PromiseResult } from 'aws-sdk/lib/request'; - +import { ReadableStreamBuffer } from 'stream-buffers'; import { IAdditionalLibraryMetadata, IFileStats, @@ -159,7 +159,9 @@ export default class MongoS3LibraryStorage implements ILibraryStorage { } ); } - const filesToDelete = await this.listFiles(library); + const filesToDelete = await this.listFiles(library, { + withMetadata: false + }); // S3 batch deletes only work with 1000 files at a time, so we // might have to do this in several requests. try { @@ -459,6 +461,15 @@ export default class MongoS3LibraryStorage implements ILibraryStorage { ): Promise { validateFilename(file, this.options?.invalidCharactersRegexp); + // As the metadata is not S3, we need to get it from MongoDB. + if (file === 'library.json') { + const metadata = JSON.stringify(await this.getMetadata(library)); + const readable = new ReadableStreamBuffer(); + readable.put(metadata, 'utf-8'); + readable.stop(); + return readable; + } + return this.s3 .getObject({ Bucket: this.options.s3Bucket, @@ -626,10 +637,15 @@ export default class MongoS3LibraryStorage implements ILibraryStorage { /** * Gets a list of all library files that exist for this library. - * @param library + * @param library the library name + * @param withMetadata true if the 'library.json' file should be included in + * the list * @returns all files that exist for the library */ - public async listFiles(library: ILibraryName): Promise { + public async listFiles( + library: ILibraryName, + options: { withMetadata?: boolean } = { withMetadata: true } + ): Promise { const prefix = this.getS3Key(library, ''); let files: string[] = []; try { @@ -655,7 +671,7 @@ export default class MongoS3LibraryStorage implements ILibraryStorage { return []; } log.debug(`Found ${files.length} file(s) in S3.`); - return files; + return options?.withMetadata ? files.concat('library.json') : files; } /** @@ -758,6 +774,52 @@ export default class MongoS3LibraryStorage implements ILibraryStorage { }); } + /** + * Gets the the metadata of a library. In contrast to getLibrary this is + * only the metadata. + * @param library the library + * @returns the metadata about the locally installed library + */ + private async getMetadata( + library: ILibraryName + ): Promise { + if (!library) { + throw new Error('You must pass in a library name to getLibrary.'); + } + let result; + + try { + result = await this.mongodb.findOne( + { _id: LibraryName.toUberName(library) }, + { + projection: { + metadata: 1 + } + } + ); + } catch (error) { + console.log(error); + throw new H5pError( + 'mongo-library-storage:error-getting-library-metadata', + { ubername: LibraryName.toUberName(library) } + ); + } + if (!result) { + throw new H5pError( + 'mongo-library-storage:library-not-found', + { ubername: LibraryName.toUberName(library) }, + 404 + ); + } + if (!result.metadata) { + throw new H5pError( + 'mongo-library-storage:error-getting-library-metadata', + { ubername: LibraryName.toUberName(library) } + ); + } + return result.metadata; + } + private getS3Key(library: ILibraryName, filename: string): string { return `${LibraryName.toUberName(library)}/${filename}`; } diff --git a/scripts/mongo-s3-docker-compose.yml b/scripts/mongo-s3-docker-compose.yml index 982ea6b5a..0e06db9fc 100644 --- a/scripts/mongo-s3-docker-compose.yml +++ b/scripts/mongo-s3-docker-compose.yml @@ -18,11 +18,12 @@ services: volumes: - minio_data:/data ports: - - '9000:9000' - command: ['server', '/data'] + - 9000:9000 + - 9001:9001 + command: ['server', '--console-address', ':9001', '/data'] environment: - MINIO_ACCESS_KEY: minioaccesskey - MINIO_SECRET_KEY: miniosecret + MINIO_ROOT_USER: minioaccesskey + MINIO_ROOT_PASSWORD: miniosecret volumes: mongodb_data: From 0a9bc7191ec33a751d600403bca38e1b37561861 Mon Sep 17 00:00:00 2001 From: Sebastian Rettig Date: Sat, 10 Jul 2021 17:30:53 +0200 Subject: [PATCH 2/2] test(h5p-mongos3): adapted tests --- packages/h5p-mongos3/test/MongoLibraryStorage.test.ts | 3 ++- packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts b/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts index cdd986464..829260e7f 100644 --- a/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts +++ b/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts @@ -452,7 +452,8 @@ describe('MongoS3LibraryStorage', () => { ).resolves.toMatchObject({ size: realStats.size }); await expect(storage.listFiles(example1Name)).resolves.toMatchObject([ 'semantics.json', - 'language/.en.json' + 'language/.en.json', + 'library.json' ]); }); diff --git a/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts b/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts index 76f5b8569..92704159c 100644 --- a/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts +++ b/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts @@ -421,7 +421,8 @@ describe('MongoS3LibraryStorage', () => { ).resolves.toMatchObject({ size: realStats.size }); await expect(storage.listFiles(example1Name)).resolves.toMatchObject([ 'language/.en.json', - 'semantics.json' + 'semantics.json', + 'library.json' ]); });