From 38f4ea6c3ba4902cda70f24d6111c39b7ca6b477 Mon Sep 17 00:00:00 2001 From: Anton Kosyakov Date: Mon, 13 May 2019 09:07:37 +0000 Subject: [PATCH] fix #4732 non-blocking upload API Signed-off-by: Anton Kosyakov --- CHANGELOG.md | 2 + packages/filesystem/package.json | 2 - .../file-download-command-contribution.ts | 37 +- .../browser/download/file-download-service.ts | 105 +----- .../src/browser/file-tree/file-tree-model.ts | 77 ---- .../browser/file-tree/file-tree-widget.tsx | 39 +- .../src/browser/file-upload-service.ts | 342 ++++++++++++++++++ .../filesystem-frontend-contribution.ts | 54 ++- .../src/browser/filesystem-frontend-module.ts | 15 +- .../src/common/file-upload-server.ts | 27 ++ .../node/download/file-download-endpoint.ts | 54 +-- .../src/node/filesystem-backend-module.ts | 12 + .../src/node/node-file-upload-server.ts | 98 +++++ .../messages/src/browser/notifications.ts | 2 +- .../src/browser/style/notifications.css | 6 +- .../src/browser/navigator-contribution.ts | 3 +- .../src/browser/workspace-commands.ts | 3 +- yarn.lock | 13 - 18 files changed, 585 insertions(+), 306 deletions(-) create mode 100644 packages/filesystem/src/browser/file-upload-service.ts create mode 100644 packages/filesystem/src/common/file-upload-server.ts create mode 100644 packages/filesystem/src/node/node-file-upload-server.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a0df0d7371b67..ee19a050446b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Breaking changes: - [preferences] refactored to integrate launch configurations as preferences +- [filesystem] extracted `FileUploadService` and refactored `FileTreeWidget` to use it [#5086](https://github.com/theia-ide/theia/pull/5086) + - moved `FileDownloadCommands.UPLOAD` to `FileSystemCommands.UPLOAD` ## v0.6.0 diff --git a/packages/filesystem/package.json b/packages/filesystem/package.json index 0e96465df1d11..ff69bc191bfa9 100644 --- a/packages/filesystem/package.json +++ b/packages/filesystem/package.json @@ -6,7 +6,6 @@ "@theia/core": "^0.6.0", "@types/base64-js": "^1.2.5", "@types/body-parser": "^1.17.0", - "@types/formidable": "^1.0.31", "@types/fs-extra": "^4.0.2", "@types/mime-types": "^2.1.0", "@types/rimraf": "^2.0.2", @@ -16,7 +15,6 @@ "base64-js": "^1.2.1", "body-parser": "^1.18.3", "drivelist": "^6.4.3", - "formidable": "^1.2.1", "fs-extra": "^4.0.2", "http-status-codes": "^1.3.0", "mime-types": "^2.1.18", diff --git a/packages/filesystem/src/browser/download/file-download-command-contribution.ts b/packages/filesystem/src/browser/download/file-download-command-contribution.ts index 4e19defa86b9f..5ddaa66c34ecb 100644 --- a/packages/filesystem/src/browser/download/file-download-command-contribution.ts +++ b/packages/filesystem/src/browser/download/file-download-command-contribution.ts @@ -16,14 +16,11 @@ import { inject, injectable } from 'inversify'; import URI from '@theia/core/lib/common/uri'; +import { environment } from '@theia/application-package/lib/environment'; import { SelectionService } from '@theia/core/lib/common/selection-service'; import { Command, CommandContribution, CommandRegistry } from '@theia/core/lib/common/command'; import { UriAwareCommandHandler, UriCommandHandler } from '@theia/core/lib/common/uri-command-handler'; -import { ExpandableTreeNode } from '@theia/core/lib/browser/tree'; import { FileDownloadService } from './file-download-service'; -import { FileSelection } from '../file-selection'; -import { TreeWidgetSelection } from '@theia/core/lib/browser/tree/tree-widget-selection'; -import { isCancelled } from '@theia/core/lib/common/cancellation'; @injectable() export class FileDownloadCommandContribution implements CommandContribution { @@ -37,30 +34,6 @@ export class FileDownloadCommandContribution implements CommandContribution { registerCommands(registry: CommandRegistry): void { const handler = new UriAwareCommandHandler(this.selectionService, this.downloadHandler(), { multi: true }); registry.registerCommand(FileDownloadCommands.DOWNLOAD, handler); - registry.registerCommand(FileDownloadCommands.UPLOAD, new FileSelection.CommandHandler(this.selectionService, { - multi: false, - isEnabled: selection => this.canUpload(selection), - isVisible: selection => this.canUpload(selection), - execute: selection => this.upload(selection) - })); - } - - protected canUpload({ fileStat }: FileSelection): boolean { - return fileStat.isDirectory; - } - - protected async upload(selection: FileSelection): Promise { - try { - const source = TreeWidgetSelection.getSource(this.selectionService.selection); - await this.downloadService.upload(selection.fileStat.uri); - if (ExpandableTreeNode.is(selection) && source) { - await source.model.expandNode(selection); - } - } catch (e) { - if (!isCancelled(e)) { - console.error(e); - } - } } protected downloadHandler(): UriCommandHandler { @@ -76,7 +49,7 @@ export class FileDownloadCommandContribution implements CommandContribution { } protected isDownloadEnabled(uris: URI[]): boolean { - return uris.length > 0 && uris.every(u => u.scheme === 'file'); + return !environment.electron.is() && uris.length > 0 && uris.every(u => u.scheme === 'file'); } protected isDownloadVisible(uris: URI[]): boolean { @@ -93,10 +66,4 @@ export namespace FileDownloadCommands { label: 'Download' }; - export const UPLOAD: Command = { - id: 'file.upload', - category: 'File', - label: 'Upload Files...' - }; - } diff --git a/packages/filesystem/src/browser/download/file-download-service.ts b/packages/filesystem/src/browser/download/file-download-service.ts index 9f057c905ca38..7c522ab9716bd 100644 --- a/packages/filesystem/src/browser/download/file-download-service.ts +++ b/packages/filesystem/src/browser/download/file-download-service.ts @@ -14,15 +14,13 @@ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 ********************************************************************************/ -import { inject, injectable, postConstruct } from 'inversify'; +import { inject, injectable } from 'inversify'; import URI from '@theia/core/lib/common/uri'; -import { cancelled } from '@theia/core/lib/common/cancellation'; import { ILogger } from '@theia/core/lib/common/logger'; import { Endpoint } from '@theia/core/lib/browser/endpoint'; import { StatusBar, StatusBarAlignment } from '@theia/core/lib/browser/status-bar'; import { FileSystem } from '../../common/filesystem'; import { FileDownloadData } from '../../common/download/file-download-data'; -import { Deferred } from '@theia/core/lib/common/promise-util'; import { MessageService } from '@theia/core/lib/common/message-service'; @injectable() @@ -46,107 +44,6 @@ export class FileDownloadService { @inject(MessageService) protected readonly messageService: MessageService; - protected uploadForm: { - target: HTMLInputElement - file: HTMLInputElement - }; - - @postConstruct() - protected init(): void { - this.uploadForm = this.createUploadForm(); - } - - protected createUploadForm(): { - target: HTMLInputElement - file: HTMLInputElement - } { - const target = document.createElement('input'); - target.type = 'text'; - target.name = 'target'; - - const file = document.createElement('input'); - file.type = 'file'; - file.name = 'upload'; - file.multiple = true; - - const form = document.createElement('form'); - form.style.display = 'none'; - form.enctype = 'multipart/form-data'; - form.append(target); - form.append(file); - - document.body.appendChild(form); - - file.addEventListener('change', async () => { - if (file.value) { - const body = new FormData(form); - // clean up to allow upload to the same folder twice - file.value = ''; - const filesUrl = this.filesUrl(); - const deferredUpload = this.deferredUpload; - try { - const request = new XMLHttpRequest(); - - const cb = () => { - if (request.status === 200) { - deferredUpload.resolve(); - } else { - let statusText = request.statusText; - if (!statusText) { - if (request.status === 413) { - statusText = 'Payload Too Large'; - } else if (request.status) { - statusText = String(request.status); - } else { - statusText = 'Network Failure'; - } - } - const message = 'Upload Failed: ' + statusText; - deferredUpload.reject(new Error(message)); - this.messageService.error(message); - } - }; - request.addEventListener('load', cb); - request.addEventListener('error', cb); - request.addEventListener('abort', () => deferredUpload.reject(cancelled())); - - const progress = await this.messageService.showProgress({ - text: 'Uploading Files...', options: { cancelable: true } - }, () => { - request.upload.removeEventListener('progress', progressListener); - request.abort(); - }); - deferredUpload.promise.then(() => progress.cancel(), () => progress.cancel()); - const progressListener = (event: ProgressEvent) => { - if (event.lengthComputable) { - progress.report({ - work: { - done: event.loaded, - total: event.total - } - }); - } - }; - request.upload.addEventListener('progress', progressListener); - - request.open('POST', filesUrl); - request.send(body); - } catch (e) { - deferredUpload.reject(e); - } - } - }); - return { target, file }; - } - - protected deferredUpload = new Deferred(); - upload(targetUri: string | URI): Promise { - this.deferredUpload = new Deferred(); - this.uploadForm.target.value = String(targetUri); - this.uploadForm.file.click(); - return this.deferredUpload.promise; - } - async download(uris: URI[]): Promise { if (uris.length === 0) { return; diff --git a/packages/filesystem/src/browser/file-tree/file-tree-model.ts b/packages/filesystem/src/browser/file-tree/file-tree-model.ts index dd62b594b37bf..2b03c01a0c594 100644 --- a/packages/filesystem/src/browser/file-tree/file-tree-model.ts +++ b/packages/filesystem/src/browser/file-tree/file-tree-model.ts @@ -22,7 +22,6 @@ import { FileSystemWatcher, FileChangeType, FileChange, FileMoveEvent } from '.. import { FileStatNode, DirNode, FileNode } from './file-tree'; import { LocationService } from '../location'; import { LabelProvider } from '@theia/core/lib/browser/label-provider'; -import * as base64 from 'base64-js'; @injectable() export class FileTreeModel extends TreeModelImpl implements LocationService { @@ -188,80 +187,4 @@ export class FileTreeModel extends TreeModelImpl implements LocationService { return !!await dialog.open(); } - upload(node: DirNode, items: DataTransferItemList): void { - for (let i = 0; i < items.length; i++) { - const entry = items[i].webkitGetAsEntry() as WebKitEntry; - this.uploadEntry(node.uri, entry); - } - } - - protected uploadEntry(base: URI, entry: WebKitEntry | null): void { - if (!entry) { - return; - } - if (entry.isDirectory) { - this.uploadDirectoryEntry(base, entry as WebKitDirectoryEntry); - } else { - this.uploadFileEntry(base, entry as WebKitFileEntry); - } - } - - protected async uploadDirectoryEntry(base: URI, entry: WebKitDirectoryEntry): Promise { - const newBase = base.resolve(entry.name); - const uri = newBase.toString(); - if (!await this.fileSystem.exists(uri)) { - await this.fileSystem.createFolder(uri); - } - this.readEntries(entry, items => this.uploadEntries(newBase, items)); - } - - /** - * Read all entries within a folder by block of 100 files or folders until the - * whole folder has been read. - */ - // tslint:disable-next-line:no-any - protected readEntries(entry: WebKitDirectoryEntry, cb: (items: any) => void): void { - const reader = entry.createReader(); - const getEntries = () => { - reader.readEntries(results => { - if (results) { - cb(results); - getEntries(); // loop to read all entries - } - }); - }; - getEntries(); - } - - protected uploadEntries(base: URI, entries: WebKitEntry[]): void { - for (let i = 0; i < entries.length; i++) { - this.uploadEntry(base, entries[i]); - } - } - - protected uploadFileEntry(base: URI, entry: WebKitFileEntry): void { - // tslint:disable-next-line:no-any - entry.file(file => this.uploadFile(base, file as any)); - } - - protected uploadFile(base: URI, file: File): void { - const reader = new FileReader(); - reader.onload = () => this.uploadFileContent(base.resolve(file.name), reader.result as ArrayBuffer); - reader.readAsArrayBuffer(file); - } - - protected async uploadFileContent(base: URI, fileContent: ArrayBuffer): Promise { - const uri = base.toString(); - const encoding = 'base64'; - const content = base64.fromByteArray(new Uint8Array(fileContent)); - const stat = await this.fileSystem.getFileStat(uri); - if (stat) { - if (!stat.isDirectory) { - await this.fileSystem.setContent(stat, content, { encoding }); - } - } else { - await this.fileSystem.createFile(uri, { content, encoding }); - } - } - } diff --git a/packages/filesystem/src/browser/file-tree/file-tree-widget.tsx b/packages/filesystem/src/browser/file-tree/file-tree-widget.tsx index a979fb0c04745..0d089ed5261e7 100644 --- a/packages/filesystem/src/browser/file-tree/file-tree-widget.tsx +++ b/packages/filesystem/src/browser/file-tree/file-tree-widget.tsx @@ -14,13 +14,15 @@ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 ********************************************************************************/ +import * as React from 'react'; import { injectable, inject } from 'inversify'; +import { DisposableCollection, Disposable } from '@theia/core/lib/common'; +import { UriSelection } from '@theia/core/lib/common/selection'; +import { isCancelled } from '@theia/core/lib/common/cancellation'; import { ContextMenuRenderer, NodeProps, TreeProps, TreeNode, TreeWidget } from '@theia/core/lib/browser'; +import { FileUploadService } from '../file-upload-service'; import { DirNode, FileStatNode } from './file-tree'; import { FileTreeModel } from './file-tree-model'; -import { DisposableCollection, Disposable } from '@theia/core/lib/common'; -import { UriSelection } from '@theia/core/lib/common/selection'; -import * as React from 'react'; export const FILE_TREE_CLASS = 'theia-FileTree'; export const FILE_STAT_NODE_CLASS = 'theia-FileStatNode'; @@ -32,6 +34,9 @@ export class FileTreeWidget extends TreeWidget { protected readonly toCancelNodeExpansion = new DisposableCollection(); + @inject(FileUploadService) + protected readonly uploadService: FileUploadService; + constructor( @inject(TreeProps) readonly props: TreeProps, @inject(FileTreeModel) readonly model: FileTreeModel, @@ -127,17 +132,23 @@ export class FileTreeWidget extends TreeWidget { this.toCancelNodeExpansion.dispose(); } - protected handleDropEvent(node: TreeNode | undefined, event: React.DragEvent): void { - event.preventDefault(); - event.stopPropagation(); - event.dataTransfer.dropEffect = 'copy'; // Explicitly show this is a copy. - const containing = DirNode.getContainingDir(node); - if (containing) { - const source = this.getTreeNodeFromData(event.dataTransfer); - if (source) { - this.model.move(source, containing); - } else { - this.model.upload(containing, event.dataTransfer.items); + protected async handleDropEvent(node: TreeNode | undefined, event: React.DragEvent): Promise { + try { + event.preventDefault(); + event.stopPropagation(); + event.dataTransfer.dropEffect = 'copy'; // Explicitly show this is a copy. + const containing = DirNode.getContainingDir(node); + if (containing) { + const source = this.getTreeNodeFromData(event.dataTransfer); + if (source) { + await this.model.move(source, containing); + } else { + await this.uploadService.upload(containing.uri, { source: event.dataTransfer }); + } + } + } catch (e) { + if (!isCancelled(e)) { + console.error(e); } } } diff --git a/packages/filesystem/src/browser/file-upload-service.ts b/packages/filesystem/src/browser/file-upload-service.ts new file mode 100644 index 0000000000000..02836fd839546 --- /dev/null +++ b/packages/filesystem/src/browser/file-upload-service.ts @@ -0,0 +1,342 @@ +/******************************************************************************** + * Copyright (C) 2019 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +// tslint:disable:no-any + +import * as base64 from 'base64-js'; +import { injectable, inject, postConstruct } from 'inversify'; +import URI from '@theia/core/lib/common/uri'; +import { CancellationTokenSource, CancellationToken, checkCancelled } from '@theia/core/lib/common/cancellation'; +import { Deferred } from '@theia/core/lib/common/promise-util'; +import { MessageService } from '@theia/core/lib/common/message-service'; +import { Progress } from '@theia/core/src/common/message-service-protocol'; +import { FileUploadServer } from '../common/file-upload-server'; + +const maxChunkSize = 64 * 1024; + +export interface FileUploadParams { + source?: DataTransfer + progress?: FileUploadProgressParams +} +export interface FileUploadProgressParams { + text: string +} + +export interface FileUploadResult { + uploaded: URI[] +} + +@injectable() +export class FileUploadService { + + static TARGET = 'target'; + static UPLOAD = 'upload'; + + @inject(MessageService) + protected readonly messageService: MessageService; + + @inject(FileUploadServer) + protected readonly uploadServer: FileUploadServer; + + protected uploadForm: FileUploadService.Form; + + @postConstruct() + protected init(): void { + this.uploadForm = this.createUploadForm(); + } + + protected createUploadForm(): FileUploadService.Form { + const targetInput = document.createElement('input'); + targetInput.type = 'text'; + targetInput.name = FileUploadService.TARGET; + + const fileInput = document.createElement('input'); + fileInput.type = 'file'; + fileInput.name = FileUploadService.UPLOAD; + fileInput.multiple = true; + + const form = document.createElement('form'); + form.style.display = 'none'; + form.enctype = 'multipart/form-data'; + form.append(targetInput); + form.append(fileInput); + + document.body.appendChild(form); + + fileInput.addEventListener('change', () => { + if (this.deferredUpload && fileInput.value) { + const body = new FormData(form); + // clean up to allow upload to the same folder twice + fileInput.value = ''; + const targetUri = new URI(body.get(FileUploadService.TARGET)); + const { resolve, reject } = this.deferredUpload; + this.deferredUpload = undefined; + this.withProgress((progress, token) => { + const context: FileUploadService.Context = { totalSize: 0, entries: [], progress, token }; + body.getAll(FileUploadService.UPLOAD).forEach((file: File) => this.indexFile(targetUri, file, context)); + return this.doUpload(context); + }, this.uploadForm.progress).then(resolve, reject); + } + }); + return { targetInput, fileInput }; + } + + protected deferredUpload: Deferred | undefined; + async upload(targetUri: string | URI, params: FileUploadParams = {}): Promise { + const { source } = params; + if (source) { + return this.withProgress(async (progress, token) => { + const context: FileUploadService.Context = { totalSize: 0, entries: [], progress, token }; + await this.indexDataTransfer(new URI(String(targetUri)), source, context); + return this.doUpload(context); + }, params.progress); + } + this.deferredUpload = new Deferred(); + this.uploadForm.targetInput.value = String(targetUri); + this.uploadForm.fileInput.click(); + this.uploadForm.progress = params.progress; + return this.deferredUpload.promise; + } + + protected async doUpload({ entries, progress, token, totalSize }: FileUploadService.Context): Promise { + const result: FileUploadResult = { uploaded: [] }; + if (!entries.length) { + return result; + } + const total = totalSize; + let done = 0; + for (const entry of entries) { + progress.report({ work: { done, total } }); + const { file } = entry; + let id: string | undefined; + let readBytes = 0; + let someAppendFailed: Error | undefined; + try { + const promises: Promise[] = []; + do { + const fileSlice = await this.readFileSlice(file, readBytes); + if (someAppendFailed) { + throw someAppendFailed; + } + checkCancelled(token); + readBytes = fileSlice.read; + if (id === undefined) { + id = await this.uploadServer.open(entry.uri.toString(), fileSlice.content, readBytes >= file.size); + checkCancelled(token); + progress.report({ + work: { + done: done + fileSlice.read, + total + } + }); + } else { + promises.push(this.uploadServer.append(id, fileSlice.content, readBytes >= file.size).then(() => { + checkCancelled(token); + progress.report({ + work: { + done: done + fileSlice.read, + total + } + }); + }, appendError => { + someAppendFailed = appendError; + throw appendError; + })); + } + } while (readBytes < file.size); + await Promise.all(promises); + done += file.size; + progress.report({ work: { done, total } }); + } finally { + if (id !== undefined) { + this.uploadServer.close(id); + } + } + } + progress.report({ work: { done: total, total } }); + return result; + } + + protected readFileSlice(file: File, read: number): Promise<{ + content: string + read: number + }> { + return new Promise((resolve, reject) => { + if (file.size === 0 && read === 0) { + resolve({ content: '', read }); + return; + } + const bytesLeft = file.size - read; + if (!bytesLeft) { + reject(new Error('nothing to read')); + return; + } + const size = Math.min(maxChunkSize, bytesLeft); + const slice = file.slice(read, read + size); + const reader = new FileReader(); + reader.onload = () => { + read += size; + const buffer = reader.result as ArrayBuffer; + const content = base64.fromByteArray(new Uint8Array(buffer)); + resolve({ content, read }); + }; + reader.onerror = reject; + reader.readAsArrayBuffer(slice); + }); + } + + protected async withProgress( + cb: (progress: Progress, token: CancellationToken) => Promise, + { text }: FileUploadProgressParams = { text: 'Uploading Files...' } + ): Promise { + const cancellationSource = new CancellationTokenSource(); + const { token } = cancellationSource; + const progress = await this.messageService.showProgress({ text, options: { cancelable: true } }, () => cancellationSource.cancel()); + try { + return await cb(progress, token); + } finally { + progress.cancel(); + } + } + + protected async indexDataTransfer(targetUri: URI, dataTransfer: DataTransfer, context: FileUploadService.Context): Promise { + checkCancelled(context.token); + if (dataTransfer.items) { + await this.indexDataTransferItemList(targetUri, dataTransfer.items, context); + } else { + this.indexFileList(targetUri, dataTransfer.files, context); + } + } + + protected indexFileList(targetUri: URI, files: FileList, context: FileUploadService.Context): void { + for (let i = 0; i < files.length; i++) { + const file = files[i]; + if (file) { + this.indexFile(targetUri, file, context); + } + } + } + + protected indexFile(targetUri: URI, file: File, context: FileUploadService.Context): void { + context.entries.push({ + uri: targetUri.resolve(file.name), + file + }); + context.totalSize += file.size; + } + + protected async indexDataTransferItemList(targetUri: URI, items: DataTransferItemList, context: FileUploadService.Context): Promise { + checkCancelled(context.token); + const promises: Promise[] = []; + for (let i = 0; i < items.length; i++) { + const entry = items[i].webkitGetAsEntry() as WebKitEntry; + promises.push(this.indexEntry(targetUri, entry, context)); + } + await Promise.all(promises); + } + + protected async indexEntry(targetUri: URI, entry: WebKitEntry | null, context: FileUploadService.Context): Promise { + checkCancelled(context.token); + if (!entry) { + return; + } + if (entry.isDirectory) { + await this.indexDirectoryEntry(targetUri, entry as WebKitDirectoryEntry, context); + } else { + await this.indexFileEntry(targetUri, entry as WebKitFileEntry, context); + } + } + + protected async indexDirectoryEntry(targetUri: URI, entry: WebKitDirectoryEntry, context: FileUploadService.Context): Promise { + checkCancelled(context.token); + const newTargetUri = targetUri.resolve(entry.name); + const promises: Promise[] = []; + await this.readEntries(entry, items => promises.push(this.indexEntries(newTargetUri, items, context)), context); + await Promise.all(promises); + } + + /** + * Read all entries within a folder by block of 100 files or folders until the + * whole folder has been read. + */ + protected async readEntries(entry: WebKitDirectoryEntry, cb: (items: any) => void, context: FileUploadService.Context): Promise { + return new Promise(async (resolve, reject) => { + const reader = entry.createReader(); + const getEntries = () => reader.readEntries(results => { + if (!context.token.isCancellationRequested && results && results.length) { + cb(results); + getEntries(); // loop to read all entries + } else { + resolve(); + } + }, reject); + getEntries(); + }); + } + + protected async indexEntries(targetUri: URI, entries: WebKitEntry[], context: FileUploadService.Context): Promise { + checkCancelled(context.token); + const promises: Promise[] = []; + for (let i = 0; i < entries.length; i++) { + promises.push(this.indexEntry(targetUri, entries[i], context)); + } + await Promise.all(promises); + } + + protected async indexFileEntry(targetUri: URI, entry: WebKitFileEntry, context: FileUploadService.Context): Promise { + await new Promise((resolve, reject) => { + try { + entry.file(file => { + this.indexFile(targetUri, file, context); + resolve(); + }, reject); + } catch (e) { + reject(e); + } + }); + } + +} + +export namespace FileUploadService { + export interface UploadEntry { + file: File + uri: URI + } + export interface Context { + progress: Progress + token: CancellationToken + entries: UploadEntry[] + totalSize: number + + } + export interface Form { + targetInput: HTMLInputElement + fileInput: HTMLInputElement + progress?: FileUploadProgressParams + } + export interface SubmitOptions { + body: FormData + token: CancellationToken + onDidProgress: (event: ProgressEvent) => void + } + export interface SubmitError extends Error { + status: number; + } + export function isSubmitError(e: any): e is SubmitError { + return !!e && 'status' in e; + } +} diff --git a/packages/filesystem/src/browser/filesystem-frontend-contribution.ts b/packages/filesystem/src/browser/filesystem-frontend-contribution.ts index a4dcdf54ebb95..aa1d1c12e011e 100644 --- a/packages/filesystem/src/browser/filesystem-frontend-contribution.ts +++ b/packages/filesystem/src/browser/filesystem-frontend-contribution.ts @@ -16,18 +16,33 @@ import { injectable, inject } from 'inversify'; import URI from '@theia/core/lib/common/uri'; -import { MaybePromise } from '@theia/core/lib/common'; +import { environment } from '@theia/application-package/lib/environment'; +import { MaybePromise, SelectionService, isCancelled } from '@theia/core/lib/common'; +import { Command, CommandContribution, CommandRegistry } from '@theia/core/lib/common/command'; import { FrontendApplicationContribution, ApplicationShell, NavigatableWidget, NavigatableWidgetOptions, - Saveable, WidgetManager, StatefulWidget, FrontendApplication + Saveable, WidgetManager, StatefulWidget, FrontendApplication, ExpandableTreeNode } from '@theia/core/lib/browser'; import { FileSystemWatcher, FileChangeEvent, FileMoveEvent, FileChangeType } from './filesystem-watcher'; import { MimeService } from '@theia/core/lib/browser/mime-service'; +import { TreeWidgetSelection } from '@theia/core/lib/browser/tree/tree-widget-selection'; import { FileSystemPreferences } from './filesystem-preferences'; +import { FileSelection } from './file-selection'; +import { FileUploadService } from './file-upload-service'; + +export namespace FileSystemCommands { + + export const UPLOAD: Command = { + id: 'file.upload', + category: 'File', + label: 'Upload Files...' + }; + +} @injectable() -export class FileSystemFrontendContribution implements FrontendApplicationContribution { +export class FileSystemFrontendContribution implements FrontendApplicationContribution, CommandContribution { @inject(ApplicationShell) protected readonly shell: ApplicationShell; @@ -44,6 +59,12 @@ export class FileSystemFrontendContribution implements FrontendApplicationContri @inject(FileSystemPreferences) protected readonly preferences: FileSystemPreferences; + @inject(SelectionService) + protected readonly selectionService: SelectionService; + + @inject(FileUploadService) + protected readonly uploadService: FileUploadService; + initialize(): void { this.fileSystemWatcher.onFilesChanged(event => this.run(() => this.updateWidgets(event))); this.fileSystemWatcher.onDidMove(event => this.run(() => this.moveWidgets(event))); @@ -58,6 +79,33 @@ export class FileSystemFrontendContribution implements FrontendApplicationContri }); } + registerCommands(commands: CommandRegistry): void { + commands.registerCommand(FileSystemCommands.UPLOAD, new FileSelection.CommandHandler(this.selectionService, { + multi: false, + isEnabled: selection => this.canUpload(selection), + isVisible: selection => this.canUpload(selection), + execute: selection => this.upload(selection) + })); + } + + protected canUpload({ fileStat }: FileSelection): boolean { + return !environment.electron.is() && fileStat.isDirectory; + } + + protected async upload(selection: FileSelection): Promise { + try { + const source = TreeWidgetSelection.getSource(this.selectionService.selection); + await this.uploadService.upload(selection.fileStat.uri); + if (ExpandableTreeNode.is(selection) && source) { + await source.model.expandNode(selection); + } + } catch (e) { + if (!isCancelled(e)) { + console.error(e); + } + } + } + protected pendingOperation = Promise.resolve(); protected run(operation: () => MaybePromise): Promise { return this.pendingOperation = this.pendingOperation.then(async () => { diff --git a/packages/filesystem/src/browser/filesystem-frontend-module.ts b/packages/filesystem/src/browser/filesystem-frontend-module.ts index f1c3d1a49f7b3..46ea43b97744e 100644 --- a/packages/filesystem/src/browser/filesystem-frontend-module.ts +++ b/packages/filesystem/src/browser/filesystem-frontend-module.ts @@ -17,7 +17,7 @@ import '../../src/browser/style/index.css'; import { ContainerModule, interfaces } from 'inversify'; -import { ResourceResolver } from '@theia/core/lib/common'; +import { ResourceResolver, CommandContribution } from '@theia/core/lib/common'; import { WebSocketConnectionProvider, FrontendApplicationContribution, ConfirmDialog } from '@theia/core/lib/browser'; import { FileSystem, fileSystemPath, FileShouldOverwrite, FileStat } from '../common'; import { @@ -29,6 +29,8 @@ import { bindFileSystemPreferences } from './filesystem-preferences'; import { FileSystemWatcher } from './filesystem-watcher'; import { FileSystemFrontendContribution } from './filesystem-frontend-contribution'; import { FileSystemProxyFactory } from './filesystem-proxy-factory'; +import { FileUploadService } from './file-upload-service'; +import { fileUploadPath, FileUploadServer } from '../common/file-upload-server'; export default new ContainerModule(bind => { bindFileSystemPreferences(bind); @@ -56,7 +58,16 @@ export default new ContainerModule(bind => { bindFileResource(bind); - bind(FrontendApplicationContribution).to(FileSystemFrontendContribution).inSingletonScope(); + bind(FileUploadService).toSelf().inSingletonScope(); + + bind(FileSystemFrontendContribution).toSelf().inSingletonScope(); + bind(CommandContribution).toService(FileSystemFrontendContribution); + bind(FrontendApplicationContribution).toService(FileSystemFrontendContribution); + + bind(FileUploadServer).toDynamicValue(ctx => { + const provider = ctx.container.get(WebSocketConnectionProvider); + return provider.createProxy(fileUploadPath); + }).inSingletonScope(); }); export function bindFileResource(bind: interfaces.Bind): void { diff --git a/packages/filesystem/src/common/file-upload-server.ts b/packages/filesystem/src/common/file-upload-server.ts new file mode 100644 index 0000000000000..51c764e4f5e51 --- /dev/null +++ b/packages/filesystem/src/common/file-upload-server.ts @@ -0,0 +1,27 @@ +/******************************************************************************** + * Copyright (C) 2019 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import { Disposable } from '@theia/core/lib/common/disposable'; + +export const fileUploadPath = '/services/file-upload'; + +export const FileUploadServer = Symbol('FileUploadServer'); +/** content in base64 encoding */ +export interface FileUploadServer extends Disposable { + open(uri: string, content: string, done: boolean): Promise; + append(id: string, content: string, done: boolean): Promise; + close(id: string): Promise; +} diff --git a/packages/filesystem/src/node/download/file-download-endpoint.ts b/packages/filesystem/src/node/download/file-download-endpoint.ts index be81f6306df09..4e8f6f728e407 100644 --- a/packages/filesystem/src/node/download/file-download-endpoint.ts +++ b/packages/filesystem/src/node/download/file-download-endpoint.ts @@ -14,22 +14,15 @@ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 ********************************************************************************/ +// tslint:disable:no-any + import { injectable, inject, named } from 'inversify'; import { json } from 'body-parser'; // tslint:disable-next-line:no-implicit-dependencies -import { Application, Router, Request, Response, NextFunction } from 'express'; -import * as formidable from 'formidable'; -import URI from '@theia/core/lib/common/uri'; -import { FileUri } from '@theia/core/lib/node/file-uri'; +import { Application, Router } from 'express'; import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application'; import { FileDownloadHandler } from './file-download-handler'; -// upload max file size in MB, default 2048 -let uploadMaxFileSize = Number(process.env.THEIA_UPLOAD_MAX_FILE_SIZE); -if (typeof uploadMaxFileSize !== 'number' || Number.isNaN(uploadMaxFileSize) || !Number.isFinite(uploadMaxFileSize)) { - uploadMaxFileSize = 2048; -} - @injectable() export class FileDownloadEndpoint implements BackendApplicationContribution { @@ -44,53 +37,12 @@ export class FileDownloadEndpoint implements BackendApplicationContribution { protected readonly multiFileDownloadHandler: FileDownloadHandler; configure(app: Application): void { - const upload = this.upload.bind(this); const router = Router(); router.get('/', (request, response) => this.singleFileDownloadHandler.handle(request, response)); router.put('/', (request, response) => this.multiFileDownloadHandler.handle(request, response)); - router.post('/', upload); // Content-Type: application/json app.use(json()); app.use(FileDownloadEndpoint.PATH, router); } - protected upload(req: Request, res: Response, next: NextFunction): void { - const form = new formidable.IncomingForm(); - form.multiples = true; - form.maxFileSize = uploadMaxFileSize * 1024 * 1024; - - let targetUri: URI | undefined; - const clientErrors: string[] = []; - form.on('field', (name: string, value: string) => { - if (name === 'target') { - targetUri = new URI(value); - } - }); - form.on('fileBegin', (_: string, file: formidable.File) => { - if (targetUri) { - file.path = FileUri.fsPath(targetUri.resolve(file.name)); - } else { - clientErrors.push(`cannot upload "${file.name}", target is not provided`); - } - }); - form.on('error', (error: Error) => { - if (String(error).indexOf('maxFileSize') !== -1) { - res.writeHead(413, 'Payload Exceeded ' + uploadMaxFileSize + 'MB'); - } else { - console.error(error); - res.writeHead(500, String(error)); - } - res.end(); - }); - form.on('end', () => { - if (clientErrors.length) { - res.writeHead(400, clientErrors.join('\n')); - } else { - res.writeHead(200); - } - res.end(); - }); - form.parse(req); - } - } diff --git a/packages/filesystem/src/node/filesystem-backend-module.ts b/packages/filesystem/src/node/filesystem-backend-module.ts index 216171e9bb3e6..5309465adb2d2 100644 --- a/packages/filesystem/src/node/filesystem-backend-module.ts +++ b/packages/filesystem/src/node/filesystem-backend-module.ts @@ -21,6 +21,8 @@ import { FileSystem, FileSystemClient, fileSystemPath, DispatchingFileSystemClie import { FileSystemWatcherServer, FileSystemWatcherClient, fileSystemWatcherPath } from '../common/filesystem-watcher-protocol'; import { FileSystemWatcherServerClient } from './filesystem-watcher-client'; import { NsfwFileSystemWatcherServer } from './nsfw-watcher/nsfw-filesystem-watcher'; +import { fileUploadPath, FileUploadServer } from '../common/file-upload-server'; +import { NodeFileUploadServer } from './node-file-upload-server'; const SINGLE_THREADED = process.argv.indexOf('--no-cluster') !== -1; @@ -79,4 +81,14 @@ export default new ContainerModule(bind => { return server; }) ).inSingletonScope(); + + bind(NodeFileUploadServer).toSelf().inTransientScope(); + bind(FileUploadServer).toService(NodeFileUploadServer); + bind(ConnectionHandler).toDynamicValue(ctx => + new JsonRpcConnectionHandler(fileUploadPath, client => { + const server = ctx.container.get(FileUploadServer); + client.onDidCloseConnection(() => server.dispose()); + return server; + }) + ).inSingletonScope(); }); diff --git a/packages/filesystem/src/node/node-file-upload-server.ts b/packages/filesystem/src/node/node-file-upload-server.ts new file mode 100644 index 0000000000000..a2845d17aaab9 --- /dev/null +++ b/packages/filesystem/src/node/node-file-upload-server.ts @@ -0,0 +1,98 @@ +/******************************************************************************** + * Copyright (C) 2019 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as path from 'path'; +import * as crypto from 'crypto'; +import * as fs from 'fs-extra'; +import { injectable } from 'inversify'; +import { FileUri } from '@theia/core/lib/node/file-uri'; +import { Disposable, DisposableCollection } from '@theia/core/lib/common/disposable'; +import { FileUploadServer } from '../common/file-upload-server'; + +@injectable() +export class NodeFileUploadServer implements FileUploadServer { + + protected readonly toDispose = new DisposableCollection(); + protected readonly uploads = new Map(); + + dispose(): void { + this.toDispose.dispose(); + } + + async open(uri: string, content: string, done: boolean): Promise { + const upload = new NodeFileUpload(FileUri.fsPath(uri)); + this.toDispose.push(upload); + this.uploads.set(upload.id, upload); + this.toDispose.push(Disposable.create(() => this.uploads.delete(upload.id))); + await upload.create(content); + if (done) { + await upload.rename(); + await this.close(upload.id); + } + return upload.id; + } + + async append(id: string, content: string, done: boolean): Promise { + const upload = this.uploads.get(id); + if (!upload) { + throw new Error(`upload '${id}' does not exist`); + } + await upload.append(content); + if (done) { + await upload.rename(); + await this.close(upload.id); + } + } + + async close(id: string): Promise { + const upload = this.uploads.get(id); + if (upload) { + upload.dispose(); + } + } + +} + +export class NodeFileUpload implements Disposable { + + readonly id: string; + readonly uploadPath: string; + + constructor( + readonly fsPath: string + ) { + this.id = 'upload_' + crypto.randomBytes(16).toString('hex'); + this.uploadPath = path.join(path.dirname(fsPath), this.id); + } + + async create(content: string): Promise { + await fs.outputFile(this.uploadPath, content, 'base64'); + } + + async append(content: string): Promise { + await fs.appendFile(this.uploadPath, content, { encoding: 'base64' }); + } + + async rename(): Promise { + await fs.move(this.uploadPath, this.fsPath, { overwrite: true }); + this.dispose = () => Promise.resolve(); + } + + dispose(): void { + fs.unlink(this.uploadPath).catch(() => {/*no-op*/ }); + } + +} diff --git a/packages/messages/src/browser/notifications.ts b/packages/messages/src/browser/notifications.ts index 8eeb83771282c..53a873834ea19 100644 --- a/packages/messages/src/browser/notifications.ts +++ b/packages/messages/src/browser/notifications.ts @@ -160,7 +160,7 @@ class ProgressNotificationImpl implements ProgressNotification { container = document.getElementById('notification-container-' + this.properties.id); if (container) { const progressContainer = container.appendChild(document.createElement('div')); - progressContainer.className = 'progress'; + progressContainer.className = 'theia-notification-progress'; const progress = progressContainer.appendChild(document.createElement('p')); progress.id = 'notification-progress-' + this.properties.id; } diff --git a/packages/messages/src/browser/style/notifications.css b/packages/messages/src/browser/style/notifications.css index 5e1f4d39f05ab..b10719bb26a45 100644 --- a/packages/messages/src/browser/style/notifications.css +++ b/packages/messages/src/browser/style/notifications.css @@ -77,10 +77,12 @@ color: var(--theia-warn-color0); } -.theia-Notification .progress { +.theia-notification-progress { order: 2; width: 35px; + display: flex; align-items: center; + justify-items: left; -webkit-user-select: text; -moz-user-select: text; -ms-user-select: text; @@ -89,7 +91,7 @@ height: 100%; } -.theia-Notification .progress > p { +.theia-notification-progress > p { margin: 0px; font-size: var(--theia-ui-font-size1); vertical-align: middle; diff --git a/packages/navigator/src/browser/navigator-contribution.ts b/packages/navigator/src/browser/navigator-contribution.ts index 3390364c18dfa..89d908671b999 100644 --- a/packages/navigator/src/browser/navigator-contribution.ts +++ b/packages/navigator/src/browser/navigator-contribution.ts @@ -31,6 +31,7 @@ import { FileNavigatorFilter } from './navigator-filter'; import { WorkspaceNode } from './navigator-tree'; import { NavigatorContextKeyService } from './navigator-context-key-service'; import { TabBarToolbarContribution, TabBarToolbarRegistry } from '@theia/core/lib/browser/shell/tab-bar-toolbar'; +import { FileSystemCommands } from '@theia/filesystem/lib/browser/filesystem-frontend-contribution'; export namespace FileNavigatorCommands { export const REVEAL_IN_NAVIGATOR: Command = { @@ -195,7 +196,7 @@ export class FileNavigatorContribution extends AbstractViewContribution boolean = require('valid-filename'); @@ -144,7 +145,7 @@ export class FileMenuContribution implements MenuContribution { }); const downloadUploadMenu = [...CommonMenus.FILE, '4_downloadupload']; registry.registerMenuAction(downloadUploadMenu, { - commandId: FileDownloadCommands.UPLOAD.id, + commandId: FileSystemCommands.UPLOAD.id, order: 'a' }); registry.registerMenuAction(downloadUploadMenu, { diff --git a/yarn.lock b/yarn.lock index 09ed0997b331a..280844793069d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -213,14 +213,6 @@ dependencies: "@types/node" "*" -"@types/formidable@^1.0.31": - version "1.0.31" - resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b" - integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q== - dependencies: - "@types/events" "*" - "@types/node" "*" - "@types/fs-extra@^4.0.2": version "4.0.8" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-4.0.8.tgz#6957ddaf9173195199cb96da3db44c74700463d2" @@ -4161,11 +4153,6 @@ formatio@1.2.0: dependencies: samsam "1.x" -formidable@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.1.tgz#70fb7ca0290ee6ff961090415f4b3df3d2082659" - integrity sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg== - forwarded@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"