From c6efb9cfe97f9c9e6efadc15694f489245fdd9cd Mon Sep 17 00:00:00 2001 From: Shiran Pasternak Date: Thu, 21 Sep 2023 16:47:01 -0400 Subject: [PATCH] Better typing for storage-related modules --- .../file-group-picker.component.ts | 1 - desktop/src/app/models/blob-container.ts | 4 +- .../services/core/data/storage-list-getter.ts | 7 ++- .../storage/blob-storage-client-proxy.ts | 3 +- .../services/storage/models/storage-blob.ts | 56 ++++++++++--------- .../services/storage/storage-blob.service.ts | 18 +++--- .../storage/storage-client.service.ts | 10 +++- .../storage/storage-container.service.ts | 4 +- 8 files changed, 57 insertions(+), 46 deletions(-) diff --git a/desktop/src/app/components/data/shared/file-group-picker/file-group-picker.component.ts b/desktop/src/app/components/data/shared/file-group-picker/file-group-picker.component.ts index 22829fc851..18d94e0b26 100644 --- a/desktop/src/app/components/data/shared/file-group-picker/file-group-picker.component.ts +++ b/desktop/src/app/components/data/shared/file-group-picker/file-group-picker.component.ts @@ -4,7 +4,6 @@ import { import { ControlValueAccessor, FormControl, NG_VALIDATORS, NG_VALUE_ACCESSOR, } from "@angular/forms"; -import { MatOptionSelectionChange } from "@angular/material/core"; import { FilterBuilder, ListView } from "@batch-flask/core"; import { Activity, DialogService } from "@batch-flask/ui"; import { FileGroupCreateFormComponent } from "app/components/data/action"; diff --git a/desktop/src/app/models/blob-container.ts b/desktop/src/app/models/blob-container.ts index 341270c73d..f8de3b3674 100644 --- a/desktop/src/app/models/blob-container.ts +++ b/desktop/src/app/models/blob-container.ts @@ -8,7 +8,7 @@ export interface BlobContainerAttributes { name: string; publicAccessLevel: string; metadata?: any; - lastModified: Date; + lastModified?: Date; lease?: Partial; } @@ -25,7 +25,7 @@ export class BlobContainer extends Record implements Na @Prop() public publicAccessLevel: string; @Prop() public metadata: any; - @Prop() public lastModified: Date; + @Prop() public lastModified?: Date; @Prop() public lease: ContainerLease; @Prop() public storageAccountId: string; diff --git a/desktop/src/app/services/core/data/storage-list-getter.ts b/desktop/src/app/services/core/data/storage-list-getter.ts index 268797cd4a..b6c1387b88 100644 --- a/desktop/src/app/services/core/data/storage-list-getter.ts +++ b/desktop/src/app/services/core/data/storage-list-getter.ts @@ -1,5 +1,6 @@ import { Type } from "@angular/core"; import { ContinuationToken, ListGetter, ListGetterConfig, Record, ServerError } from "@batch-flask/core"; +import { BlobStorageClientProxy } from "app/services/storage"; import { StorageBlobResult } from "app/services/storage/models"; import { StorageClientService } from "app/services/storage/storage-client.service"; import { Observable, from, throwError } from "rxjs"; @@ -12,13 +13,15 @@ export interface StorageBaseParams { export interface StorageListConfig, TParams extends StorageBaseParams> extends ListGetterConfig { - getData: (client: any, params: TParams, options: any, token: any) => Promise>; + getData: (client: BlobStorageClientProxy, params: TParams, options: any, token: any) => + Promise>; } export class StorageListGetter, TParams extends StorageBaseParams> extends ListGetter { - private _getData: (client: any, params: TParams, options: any, token: any) => Promise>; + private _getData: (client: BlobStorageClientProxy, params: TParams, options: any, token: any) => + Promise>; constructor( type: Type, diff --git a/desktop/src/app/services/storage/blob-storage-client-proxy.ts b/desktop/src/app/services/storage/blob-storage-client-proxy.ts index 3ae96a6013..e3241f8dda 100644 --- a/desktop/src/app/services/storage/blob-storage-client-proxy.ts +++ b/desktop/src/app/services/storage/blob-storage-client-proxy.ts @@ -4,6 +4,7 @@ import { IpcEvent } from "common/constants"; import { SharedAccessPolicy } from "./models"; import * as blob from "./models/storage-blob"; import { BlobContentResult } from "./storage-blob.service"; +import { StorageClient } from "./storage-client.service"; const storageIpc = IpcEvent.storageBlob; @@ -18,7 +19,7 @@ export interface ListBlobResponse { }; } -export class BlobStorageClientProxy { +export class BlobStorageClientProxy implements StorageClient { private storageInfo: { url: string; account: string, key: string }; diff --git a/desktop/src/app/services/storage/models/storage-blob.ts b/desktop/src/app/services/storage/models/storage-blob.ts index 7cf2cf58f7..4042ac914e 100644 --- a/desktop/src/app/services/storage/models/storage-blob.ts +++ b/desktop/src/app/services/storage/models/storage-blob.ts @@ -1,10 +1,38 @@ -import { ContainerItem, BlobUploadCommonResponse, ContainerGetPropertiesResponse, CommonOptions } from "@azure/storage-blob"; +import { BlobUploadCommonResponse, ContainerGetPropertiesResponse, CommonOptions, ContainerItem } from "@azure/storage-blob"; import { Model, Prop, Record } from "@batch-flask/core"; import { SharedAccessPolicy } from "./shared-access-policy"; +import { BlobContainer } from "app/models"; // Placeholder; we don't use any options to storage-blob API requests export type RequestOptions = Partial; +export interface BlobProperties { + name: string; + url: string; + isDirectory: boolean; + properties: { + contentLength: number; + contentType: string; + creationTime: Date | string; + lastModified?: Date | string; + }; +} + +export type ContainerProperties = ContainerGetPropertiesResponse; + +@Model() +export class BlobItem extends Record { + @Prop() name: string; + @Prop() url: string; + @Prop() isDirectory: boolean; + @Prop() properties: { + contentLength: number; + contentType: string; + creationTime: Date | string; + lastModified?: Date | string; + } +} + export interface BaseParams { url: string; account: string; @@ -94,32 +122,6 @@ export interface GetBlobContentResult { content: string; } -export interface BlobProperties { - name: string; - url: string; - isDirectory: boolean; - properties: { - contentLength: number; - contentType: string; - creationTime: Date | string; - lastModified: Date | string; - }; -} - -@Model() -export class BlobItem extends Record { - @Prop() name: string; - @Prop() url: string; - @Prop() isDirectory: boolean; - @Prop() properties: { - contentLength: number; - contentType: string; - creationTime: Date | string; - lastModified: Date | string; - } -} - -export type ContainerProperties = ContainerGetPropertiesResponse; export interface ListBlobOptions { /** diff --git a/desktop/src/app/services/storage/storage-blob.service.ts b/desktop/src/app/services/storage/storage-blob.service.ts index 3e0a4f9b0b..7d1b91bb3a 100644 --- a/desktop/src/app/services/storage/storage-blob.service.ts +++ b/desktop/src/app/services/storage/storage-blob.service.ts @@ -19,7 +19,7 @@ import { Constants } from "common"; import { AsyncSubject, Observable, from, of, throwError } from "rxjs"; import { catchError, concat, concatMap, map, share, switchMap, take } from "rxjs/operators"; import { BlobStorageClientProxy } from "./blob-storage-client-proxy"; -import { StorageClientService } from "./storage-client.service"; +import { StorageClient, StorageClientService } from "./storage-client.service"; export interface ListBlobParams { storageAccountId: string; @@ -42,7 +42,10 @@ export interface BlobContentResult { content: string; } -export type StorageContainerProperties = ContainerProperties; +export interface StorageContainerProperties extends Omit { + lastModified?: Date; + etag?: string; +} export interface NavigateBlobsOptions { /** @@ -117,8 +120,7 @@ export class StorageBlobService { this._blobListGetter = new StorageListGetter(FileRecord, this.storageClient, { cache: (params) => this.getBlobFileCache(params), - getData: (client: BlobStorageClientProxy, - params, options, continuationToken) => { + getData: async (client: StorageClient, params, options, continuationToken) => { const blobOptions: ListBlobOptions = { folder: options.original.folder, recursive: options.original.recursive, @@ -126,12 +128,8 @@ export class StorageBlobService { maxPageSize: this.maxBlobPageSize }; - // N.B. `BlobItem` and `FileRecord` are nearly identical - return client.listBlobs( - params.container, - blobOptions, - continuationToken, - ) as Promise>; + const blobs = await client.listBlobs(params.container, blobOptions, continuationToken); + return { data: blobs.data.map(blob => new FileRecord(blob)) }; }, logIgnoreError: storageIgnoredErrors, }); diff --git a/desktop/src/app/services/storage/storage-client.service.ts b/desktop/src/app/services/storage/storage-client.service.ts index f879dd6c3c..a2ba915167 100644 --- a/desktop/src/app/services/storage/storage-client.service.ts +++ b/desktop/src/app/services/storage/storage-client.service.ts @@ -1,7 +1,7 @@ import { Injectable } from "@angular/core"; import { ServerError } from "@batch-flask/core"; import { ElectronRemote } from "@batch-flask/electron"; -import { StorageKeys } from "app/models"; +import { BlobContainer, StorageKeys } from "app/models"; import { BatchExplorerService } from "app/services/batch-explorer.service"; import { ArmResourceUtils } from "app/utils"; import { Observable, throwError } from "rxjs"; @@ -10,6 +10,7 @@ import { BatchAccountService } from "../batch-account"; import { BlobStorageClientProxy } from "./blob-storage-client-proxy"; import { StorageAccountKeysService } from "./storage-account-keys.service"; import { StorageClientProxyFactory } from "./storage-client-proxy-factory"; +import { ListBlobsResult, ListContainersResult, StorageBlobResult } from "./models"; export interface AutoStorageSettings { lastKeySync: Date; @@ -23,6 +24,13 @@ export interface StorageKeyCachedItem { keys: StorageKeys; } +export interface StorageClient { + listContainersWithPrefix(prefix: string, continuationToken?: string, options?: any): + Promise; + listBlobs(containerName: string, options: any, continuationToken?: string): + Promise; +} + @Injectable({ providedIn: "root" }) export class StorageClientService { public hasAutoStorage: Observable; diff --git a/desktop/src/app/services/storage/storage-container.service.ts b/desktop/src/app/services/storage/storage-container.service.ts index a08887577c..4323f19ab5 100644 --- a/desktop/src/app/services/storage/storage-container.service.ts +++ b/desktop/src/app/services/storage/storage-container.service.ts @@ -14,7 +14,7 @@ import { SharedAccessPolicy } from "app/services/storage/models"; import { Observable, Subject, from, throwError } from "rxjs"; import { catchError, flatMap, share } from "rxjs/operators"; import { BlobStorageClientProxy } from "./blob-storage-client-proxy"; -import { StorageClientService } from "./storage-client.service"; +import { StorageClient, StorageClientService } from "./storage-client.service"; export interface GetContainerParams { storageAccountId: string; @@ -62,7 +62,7 @@ export class StorageContainerService { }); this._containerListGetter = new StorageListGetter(BlobContainer, this.storageClient, { cache: params => this._containerCache.getCache(params), - getData: async (client, params, options, continuationToken) => { + getData: async (client: StorageClient, params, options, continuationToken) => { let prefix = null; if (options && options.filter) { prefix = options.filter.value;