Skip to content

Commit

Permalink
fix(file-manager): improve file manager (#609)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adrians5j
adrians5j authored Nov 8, 2019
1 parent 9ec34e5 commit cf884d2
Show file tree
Hide file tree
Showing 12 changed files with 351 additions and 70 deletions.
182 changes: 182 additions & 0 deletions components/serverless-files/__tests__/normalizeInputs.test.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
const normalizeInputs = require("./../utils/normalizeInputs");

describe("normalizeInputs function test", () => {
it("passing empty value should throw error", async () => {
try {
normalizeInputs();
} catch (e) {
return;
}

try {
normalizeInputs({});
} catch (e) {
return;
}

throw new Error(`Error should've been thrown.`);
});

it(`must throw an error if inputs is missing "region"`, async () => {
try {
normalizeInputs({});
} catch (e) {
expect(e.message).toBe('Component "region" is missing.');
return;
}
throw new Error(`Error should've been thrown.`);
});

it(`must throw an error if inputs is missing "bucket"`, async () => {
try {
normalizeInputs({ region: "test-123" });
} catch (e) {
expect(e.message).toBe('Component "bucket" is missing.');
return;
}
throw new Error(`Error should've been thrown.`);
});

it("passing old inputs structure should return new inputs structure with overrides applied correctly", async () => {
const inputs = normalizeInputs({
region: "test-region",
bucket: "test-bucket",
memory: 1,
timeout: 2,
database: {
one: 1,
two: 2,
three: 3
},
uploadMinFileSize: 3,
uploadMaxFileSize: 4,
env: {
one: 1,
two: 2,
three: 3
},
webpackConfig: {
four: 4,
five: 5,
six: 6
}
});

expect(inputs).toEqual({
region: "test-region",
bucket: "test-bucket",
functions: {
apolloService: {
memory: 1,
timeout: 2,
database: {
one: 1,
two: 2,
three: 3
},
uploadMinFileSize: 3,
uploadMaxFileSize: 4,
env: {
one: 1,
two: 2,
three: 3
},
webpackConfig: {
four: 4,
five: 5,
six: 6
}
},
downloadFile: {
memory: 512,
timeout: 10,
env: {}
},
imageTransformer: {
memory: 1024,
timeout: 10,
env: {}
}
}
});
});

it("passing new inputs structure should return overrides successfully", async () => {
const inputs = normalizeInputs({
name: "test-File",
region: "test-region",
bucket: "test-bucket",
functions: {
apolloService: {
memory: 1,
timeout: 2,
database: {
one: 1,
two: 2,
three: 3
},
uploadMinFileSize: 3,
uploadMaxFileSize: 4,
env: {
one: 1,
two: 2,
three: 3
}
},
downloadFile: {
memory: 5,
timeout: 6
},
imageTransformer: {
memory: 7,
timeout: 8
}
},
webpackConfig: {
four: 4,
five: 5,
six: 6
}
});

expect(inputs).toEqual({
region: "test-region",
bucket: "test-bucket",
functions: {
apolloService: {
memory: 1,
timeout: 2,
database: {
one: 1,
two: 2,
three: 3
},
uploadMinFileSize: 3,
uploadMaxFileSize: 4,
env: {
one: 1,
two: 2,
three: 3
},
webpackConfig: null
},
downloadFile: {
memory: 5,
timeout: 6,
env: {}
},
imageTransformer: {
memory: 7,
timeout: 8,
env: {}
}
},
name: "test-File",
webpackConfig: {
four: 4,
five: 5,
six: 6
}
});
});
});
43 changes: 34 additions & 9 deletions components/serverless-files/functions/downloadFile/handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,22 @@ const pathLib = require("path");
const { createHandler, getEnvironment, getObjectParams } = require("../utils");
const loaders = require("./../loaders");

const MAX_RETURN_CONTENT_LENGTH = 6000000; // almost 6MB

/**
* Based on given path, extracts file key and additional options sent via query params.
* @param event
*/
const extractFilenameOptions = event => {
const path = sanitizeFilename(event.pathParameters.path);
return {
filename: path,
filename: decodeURI(path),
options: event.queryStringParameters,
extension: pathLib.extname(path)
};
};

const getS3Object = async event => {
const { region } = getEnvironment();
const s3 = new S3({ region });

const getS3Object = async (event, s3) => {
const { options, filename, extension } = extractFilenameOptions(event);

for (let i = 0; i < loaders.length; i++) {
Expand Down Expand Up @@ -49,15 +48,41 @@ const getS3Object = async event => {

// If no processors handled the file request, just return the S3 object by default.
const params = getObjectParams(filename);
return s3.getObject(params).promise();
return {
object: s3.getObject(params).promise(),
params: params
};
};

module.exports.handler = createHandler(async event => {
const s3Object = await getS3Object(event);
const { region } = getEnvironment();
const s3 = new S3({ region });

const { params, object } = await getS3Object(event, s3);

if (object.ContentLength < MAX_RETURN_CONTENT_LENGTH) {
return {
data: object.Body,
headers: {
"Content-Type": object.ContentType
}
};
}

// Lambda can return max 6MB of content, so if our object's size is larger, we are sending
// a 301 Redirect, redirecting the user to the public URL of the object in S3.
await s3
.putObjectAcl({
Bucket: params.Bucket,
ACL: "public-read",
Key: params.Key
})
.promise();

return {
data: s3Object.Body,
statusCode: 301,
headers: {
"Content-Type": s3Object.ContentType
Location: `https://${params.Bucket}.s3.amazonaws.com/${params.Key}`
}
};
});
21 changes: 17 additions & 4 deletions components/serverless-files/functions/images/imageLoader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,18 @@ module.exports = {
return SUPPORTED_IMAGES.includes(file.extension);
},
async process({ s3, file, options }) {
// Loaders must return {object, params} object.
let objectParams;

const transformations = sanitizeImageTransformations(options);

if (transformations && SUPPORTED_TRANSFORMABLE_IMAGES.includes(file.extension)) {
objectParams = getObjectParams(getImageKey({ key: file.name, transformations }));
try {
return await s3.getObject(objectParams).promise();
return {
object: await s3.getObject(objectParams).promise(),
params: objectParams
};
} catch (e) {
let imageTransformerLambdaResponse = await callImageTransformerLambda({
key: file.name,
Expand All @@ -46,13 +50,19 @@ module.exports = {
throw Error(imageTransformerLambdaResponse.message);
}

return await s3.getObject(objectParams).promise();
return {
object: await s3.getObject(objectParams).promise(),
params: objectParams
};
}
}

objectParams = getObjectParams(getImageKey({ key: file.name }));
try {
return await s3.getObject(objectParams).promise();
return {
object: await s3.getObject(objectParams).promise(),
params: objectParams
};
} catch (e) {
let imageTransformerLambdaResponse = await callImageTransformerLambda({
key: file.name
Expand All @@ -62,7 +72,10 @@ module.exports = {
throw Error(imageTransformerLambdaResponse.message);
}

return await s3.getObject(objectParams).promise();
return {
object: await s3.getObject(objectParams).promise(),
params: objectParams
};
}
}
};
9 changes: 2 additions & 7 deletions components/serverless-files/functions/utils/createHandler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@ module.exports = handler => async event => {
}

try {
const { data, headers = {} } = await handler(event);
console.log("dobeo data", {
data
});
const { data, statusCode, headers = {} } = await handler(event);
const isBuffer = Buffer.isBuffer(data);
let body = isBuffer
? data.toString("base64")
Expand All @@ -32,13 +29,11 @@ module.exports = handler => async event => {

return {
isBase64Encoded: isBuffer,
statusCode: 200,
statusCode: statusCode || 200,
headers: { ...baseHeaders, ...headers },
body
};
} catch (e) {
console.log("Error thrown");
console.log(e);
return {
statusCode: 500,
headers: baseHeaders,
Expand Down
3 changes: 2 additions & 1 deletion components/serverless-files/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"@webiny/serverless-function": "^2.0.0",
"aws-sdk": "^2.539.0",
"lodash.get": "^4.4.2",
"lodash.merge": "^4.6.2",
"lodash.set": "^4.3.2",
"p-retry": "^4.1.0"
},
Expand All @@ -29,7 +30,7 @@
"build:image-transformer": "webpack --config scripts/webpack/webpack.config.imageTransformer.js",
"build:manage-files": "webpack --config scripts/webpack/webpack.config.manageFiles.js",
"build": "concurrently \"yarn build:image-transformer\" \"yarn build:download\" \"yarn build:manage-files\"",
"postbuild": "cp -r configureS3Bucket.js package.json serverless.js LICENSE README.md dist/",
"postbuild": "cp -r utils package.json serverless.js LICENSE README.md dist/",
"deploy": "cd test && sls --debug",
"prepublishOnly": "yarn build"
},
Expand Down
Loading

0 comments on commit cf884d2

Please sign in to comment.