Skip to content

Commit

Permalink
chore: prettier fixes
Browse files Browse the repository at this point in the history 
Signed-off-by: Humair Khan <[email protected]>
  • Loading branch information
@HumairAK
HumairAK committed May 9, 2024
1 parent 1084789 commit 2fb07c4
Show file tree
Hide file tree
Showing 15 changed files with 407 additions and 318 deletions.
224 changes: 120 additions & 104 deletions frontend/server/handlers/artifacts.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import fetch from 'node-fetch';
import { AWSConfigs, HttpConfigs, MinioConfigs, ProcessEnv } from '../configs';
import { Client as MinioClient } from 'minio';
import { PreviewStream, findFileOnPodVolume, parseJSONString } from '../utils';
import {createMinioClient, getObjectStream} from '../minio-helper';
import { createMinioClient, getObjectStream } from '../minio-helper';
import * as serverInfo from '../helpers/server-info';
import { Handler, Request, Response } from 'express';
import { Storage } from '@google-cloud/storage';
Expand All @@ -24,9 +24,9 @@ import { HACK_FIX_HPM_PARTIAL_RESPONSE_HEADERS } from '../consts';

import * as fs from 'fs';
import { isAllowedDomain } from './domain-checker';
import { getK8sSecret } from "../k8s-helper";
import { StorageOptions } from "@google-cloud/storage/build/src/storage";
import { CredentialBody } from "google-auth-library/build/src/auth/credentials";
import { getK8sSecret } from '../k8s-helper';
import { StorageOptions } from '@google-cloud/storage/build/src/storage';
import { CredentialBody } from 'google-auth-library/build/src/auth/credentials';

/**
* ArtifactsQueryStrings describes the expected query strings key value pairs
Expand Down Expand Up @@ -77,10 +77,10 @@ export interface GCSProviderInfo {
* @param tryExtract whether the handler try to extract content from *.tar.gz files.
*/
export function getArtifactsHandler({
artifactsConfigs,
useParameter,
tryExtract,
}: {
artifactsConfigs,
useParameter,
tryExtract,
}: {
artifactsConfigs: {
aws: AWSConfigs;
http: HttpConfigs;
Expand All @@ -95,7 +95,9 @@ export function getArtifactsHandler({
const source = useParameter ? req.params.source : req.query.source;
const bucket = useParameter ? req.params.bucket : req.query.bucket;
const key = useParameter ? req.params[0] : req.query.key;
const { peek = 0, providerInfo = "", namespace = ""} = req.query as Partial<ArtifactsQueryStrings>;
const { peek = 0, providerInfo = '', namespace = '' } = req.query as Partial<
ArtifactsQueryStrings
>;
if (!source) {
res.status(500).send('Storage source is missing from artifact request');
return;
Expand All @@ -110,10 +112,10 @@ export function getArtifactsHandler({
}
console.log(`Getting storage artifact at: ${source}: ${bucket}/${key}`);

let client : MinioClient;
let client: MinioClient;
switch (source) {
case 'gcs':
await getGCSArtifactHandler({bucket, key}, peek, providerInfo, namespace)(req, res);
await getGCSArtifactHandler({ bucket, key }, peek, providerInfo, namespace)(req, res);
break;
case 'minio':
try {
Expand All @@ -123,13 +125,13 @@ export function getArtifactsHandler({
return;
}
await getMinioArtifactHandler(
{
bucket,
client,
key,
tryExtract,
},
peek,
{
bucket,
client,
key,
tryExtract,
},
peek,
)(req, res);
break;
case 's3':
Expand All @@ -140,30 +142,30 @@ export function getArtifactsHandler({
return;
}
await getMinioArtifactHandler(
{
bucket,
client,
key,
},
peek,
{
bucket,
client,
key,
},
peek,
)(req, res);
break;
case 'http':
case 'https':
await getHttpArtifactsHandler(
allowedDomain,
getHttpUrl(source, http.baseUrl || '', bucket, key),
http.auth,
peek,
allowedDomain,
getHttpUrl(source, http.baseUrl || '', bucket, key),
http.auth,
peek,
)(req, res);
break;
case 'volume':
await getVolumeArtifactsHandler(
{
bucket,
key,
},
peek,
{
bucket,
key,
},
peek,
)(req, res);
break;
default:
Expand Down Expand Up @@ -202,17 +204,17 @@ function getHttpArtifactsHandler(
if (auth.key.length > 0) {
// inject original request's value if exists, otherwise default to provided default value
headers[auth.key] =
req.headers[auth.key] || req.headers[auth.key.toLowerCase()] || auth.defaultValue;
req.headers[auth.key] || req.headers[auth.key.toLowerCase()] || auth.defaultValue;
}
if (!isAllowedDomain(url, allowedDomain)) {
res.status(500).send(`Domain not allowed.`);
return;
}
const response = await fetch(url, { headers });
response.body
.on('error', err => res.status(500).send(`Unable to retrieve artifact: ${err}`))
.pipe(new PreviewStream({ peek }))
.pipe(res);
.on('error', err => res.status(500).send(`Unable to retrieve artifact: ${err}`))
.pipe(new PreviewStream({ peek }))
.pipe(res);
};
}

Expand All @@ -234,31 +236,45 @@ function getMinioArtifactHandler(
};
}

async function parseGCSProviderInfo(providerInfo: GCSProviderInfo, namespace: string): Promise<StorageOptions> {
async function parseGCSProviderInfo(
providerInfo: GCSProviderInfo,
namespace: string,
): Promise<StorageOptions> {
if (!providerInfo.Params.tokenKey || !providerInfo.Params.secretName) {
throw new Error('Provider info with fromEnv:false supplied with incomplete secret credential info.');
throw new Error(
'Provider info with fromEnv:false supplied with incomplete secret credential info.',
);
}
let configGCS: StorageOptions;
try {
const tokenString = await getK8sSecret(providerInfo.Params.secretName, providerInfo.Params.tokenKey, namespace);
const tokenString = await getK8sSecret(
providerInfo.Params.secretName,
providerInfo.Params.tokenKey,
namespace,
);
const credentials = parseJSONString<CredentialBody>(tokenString);
configGCS = {credentials};
configGCS.scopes = "https://www.googleapis.com/auth/devstorage.read_write";
configGCS = { credentials };
configGCS.scopes = 'https://www.googleapis.com/auth/devstorage.read_write';
return configGCS;
} catch (err) {
throw new Error('Failed to parse GCS Provider config. Error: ' + err);
}
}

function getGCSArtifactHandler(options: { key: string; bucket: string }, peek: number = 0, providerInfoString?: string, namespace?: string) {
function getGCSArtifactHandler(
options: { key: string; bucket: string },
peek: number = 0,
providerInfoString?: string,
namespace?: string,
) {
const { key, bucket } = options;
return async (_: Request, res: Response) => {
try {
let storageOptions : StorageOptions | undefined;
if(providerInfoString) {
const providerInfo = parseJSONString<GCSProviderInfo>(providerInfoString);
if (providerInfo && providerInfo.Params.fromEnv === "false") {
if (!namespace){
let storageOptions: StorageOptions | undefined;
if (providerInfoString) {
const providerInfo = parseJSONString<GCSProviderInfo>(providerInfoString);
if (providerInfo && providerInfo.Params.fromEnv === 'false') {
if (!namespace) {
res.status(500).send('Failed to parse provider info. Reason: No namespace provided');
} else {
storageOptions = await parseGCSProviderInfo(providerInfo, namespace);
Expand All @@ -279,11 +295,11 @@ function getGCSArtifactHandler(options: { key: string; bucket: string }, peek: n
// Build a RegExp object that only recognizes asterisks ('*'), and
// escapes everything else.
const regex = new RegExp(
'^' +
'^' +
key
.split(/\*+/)
.map(escapeRegexChars)
.join('.*') +
.split(/\*+/)
.map(escapeRegexChars)
.join('.*') +
'$',
);
return regex.test(f.name);
Expand All @@ -295,34 +311,34 @@ function getGCSArtifactHandler(options: { key: string; bucket: string }, peek: n
return;
}
console.log(
`Found ${matchingFiles.length} matching files: `,
matchingFiles.map(file => file.name).join(','),
`Found ${matchingFiles.length} matching files: `,
matchingFiles.map(file => file.name).join(','),
);
let contents = '';
// TODO: support peek for concatenated matching files
if (peek) {
matchingFiles[0]
.createReadStream()
.pipe(new PreviewStream({ peek }))
.pipe(res);
.createReadStream()
.pipe(new PreviewStream({ peek }))
.pipe(res);
return;
}

// if not peeking, iterate and append all the files
matchingFiles.forEach((f, i) => {
const buffer: Buffer[] = [];
f.createReadStream()
.on('data', data => buffer.push(Buffer.from(data)))
.on('end', () => {
contents +=
Buffer.concat(buffer)
.toString()
.trim() + '\n';
if (i === matchingFiles.length - 1) {
res.send(contents);
}
})
.on('error', () => res.status(500).send('Failed to read file: ' + f.name));
.on('data', data => buffer.push(Buffer.from(data)))
.on('end', () => {
contents +=
Buffer.concat(buffer)
.toString()
.trim() + '\n';
if (i === matchingFiles.length - 1) {
res.send(contents);
}
})
.on('error', () => res.status(500).send('Failed to read file: ' + f.name));
});
} catch (err) {
res.status(500).send('Failed to download GCS file(s). Error: ' + err);
Expand Down Expand Up @@ -362,14 +378,14 @@ function getVolumeArtifactsHandler(options: { bucket: string; key: string }, pee
const stat = await fs.promises.stat(filePath);
if (stat.isDirectory()) {
res
.status(400)
.send(`Failed to open volume file ${filePath} is directory, does not support now`);
.status(400)
.send(`Failed to open volume file ${filePath} is directory, does not support now`);
return;
}

fs.createReadStream(filePath)
.pipe(new PreviewStream({ peek }))
.pipe(res);
.pipe(new PreviewStream({ peek }))
.pipe(res);
} catch (err) {
console.log(`Failed to open volume: ${err}`);
res.status(500).send(`Failed to open volume.`);
Expand Down Expand Up @@ -405,10 +421,10 @@ const QUERIES = {
};

export function getArtifactsProxyHandler({
enabled,
allowedDomain,
namespacedServiceGetter,
}: {
enabled,
allowedDomain,
namespacedServiceGetter,
}: {
enabled: boolean;
allowedDomain: string;
namespacedServiceGetter: NamespacedServiceGetter;
Expand All @@ -417,36 +433,36 @@ export function getArtifactsProxyHandler({
return (req, res, next) => next();
}
return proxy(
(_pathname, req) => {
// only proxy requests with namespace query parameter
return !!getNamespaceFromUrl(req.url || '');
(_pathname, req) => {
// only proxy requests with namespace query parameter
return !!getNamespaceFromUrl(req.url || '');
},
{
changeOrigin: true,
onProxyReq: proxyReq => {
console.log('Proxied artifact request: ', proxyReq.path);
},
{
changeOrigin: true,
onProxyReq: proxyReq => {
console.log('Proxied artifact request: ', proxyReq.path);
},
pathRewrite: (pathStr, req) => {
const url = new URL(pathStr || '', DUMMY_BASE_PATH);
url.searchParams.delete(QUERIES.NAMESPACE);
return url.pathname + url.search;
},
router: req => {
const namespace = getNamespaceFromUrl(req.url || '');
if (!namespace) {
console.log(`namespace query param expected in ${req.url}.`);
throw new Error(`namespace query param expected.`);
}
const urlStr = namespacedServiceGetter(namespace!);
if (!isAllowedDomain(urlStr, allowedDomain)) {
console.log(`Domain is not allowed.`);
throw new Error(`Domain is not allowed.`);
}
return namespacedServiceGetter(namespace!);
},
target: '/artifacts',
headers: HACK_FIX_HPM_PARTIAL_RESPONSE_HEADERS,
pathRewrite: (pathStr, req) => {
const url = new URL(pathStr || '', DUMMY_BASE_PATH);
url.searchParams.delete(QUERIES.NAMESPACE);
return url.pathname + url.search;
},
router: req => {
const namespace = getNamespaceFromUrl(req.url || '');
if (!namespace) {
console.log(`namespace query param expected in ${req.url}.`);
throw new Error(`namespace query param expected.`);
}
const urlStr = namespacedServiceGetter(namespace!);
if (!isAllowedDomain(urlStr, allowedDomain)) {
console.log(`Domain is not allowed.`);
throw new Error(`Domain is not allowed.`);
}
return namespacedServiceGetter(namespace!);
},
target: '/artifacts',
headers: HACK_FIX_HPM_PARTIAL_RESPONSE_HEADERS,
},
);
}

Expand Down
Loading

0 comments on commit 2fb07c4

Please sign in to comment.