Patch S3 filestorage provider, avoid fragments, nextcloud#27877

We need the fragment optimisation to avoid S3 Slowdown on OTC, as described in nextcloud#27877 Signed-off-by: [email protected] <[email protected]>
nextmcloud · Oct 20, 2021 · ae79ecd · ae79ecd
1 parent 07e359b
commit ae79ecd
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 19 deletions.
diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php
@@ -74,9 +74,9 @@ protected function parseParams($params) {
 
 		$this->test = isset($params['test']);
 		$this->bucket = $params['bucket'];
-		$this->proxy = isset($params['proxy']) ? $params['proxy'] : false;
-		$this->timeout = !isset($params['timeout']) ? 15 : $params['timeout'];
-		$this->uploadPartSize = !isset($params['uploadPartSize']) ? 524288000 : $params['uploadPartSize'];
+		$this->proxy = $params['proxy'] ?? false;
+		$this->timeout = $params['timeout'] ?? 15;
+		$this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
 		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
 		$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
 		if (!isset($params['port']) || $params['port'] === '') {

diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php
@@ -28,10 +28,11 @@
 
 use Aws\S3\Exception\S3MultipartUploadException;
 use Aws\S3\MultipartUploader;
-use Aws\S3\ObjectUploader;
 use Aws\S3\S3Client;
-use Icewind\Streams\CallbackWrapper;
+use GuzzleHttp\Psr7\Utils;
 use OC\Files\Stream\SeekableHttpStream;
+use GuzzleHttp\Psr7;
+use Psr\Http\Message\StreamInterface;
 
 trait S3ObjectTrait {
 	/**
@@ -80,37 +81,81 @@ public function readObject($urn) {
 	}
 
 	/**
+	 * Single object put helper
+	 *
 	 * @param string $urn the unified resource name used to identify the object
-	 * @param resource $stream stream with the data to write
+	 * @param StreamInterface $stream stream with the data to write
 	 * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
 	 * @throws \Exception when something goes wrong, message will be logged
-	 * @since 7.0.0
 	 */
-	public function writeObject($urn, $stream, string $mimetype = null) {
-		$count = 0;
-		$countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) {
-			$count += $read;
-		});
+	protected function writeSingle(string $urn, StreamInterface $stream, string $mimetype = null): void {
+		$this->getConnection()->putObject([
+			'Bucket' => $this->bucket,
+			'Key' => $urn,
+			'Body' => $stream,
+			'ACL' => 'private',
+			'ContentType' => $mimetype,
+		]);
+	}
 
-		$uploader = new MultipartUploader($this->getConnection(), $countStream, [
+
+	/**
+	 * Multipart upload helper that tries to avoid orphaned fragments in S3
+	 *
+	 * @param string $urn the unified resource name used to identify the object
+	 * @param StreamInterface $stream stream with the data to write
+	 * @param string|null $mimetype the mimetype to set for the remove object
+	 * @throws \Exception when something goes wrong, message will be logged
+	 */
+	protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void {
+		$uploader = new MultipartUploader($this->getConnection(), $stream, [
 			'bucket' => $this->bucket,
 			'key' => $urn,
 			'part_size' => $this->uploadPartSize,
 			'params' => [
 				'ContentType' => $mimetype
+<<<<<<< HEAD
 			]
+=======
+			],
+>>>>>>> Patch S3 filestorage provider, avoid fragments, #27877
 		]);
 
 		try {
 			$uploader->upload();
 		} catch (S3MultipartUploadException $e) {
-			// This is an empty file so just touch it then
-			if ($count === 0 && feof($countStream)) {
-				$uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '');
-				$uploader->upload();
-			} else {
-				throw $e;
+			// if anything goes wrong with multipart, make sure that you don´t poison and
+			// slow down s3 bucket with orphaned fragments
+			$uploadInfo = $e->getState()->getId();
+			if ($e->getState()->isInitiated() && (array_key_exists('UploadId', $uploadInfo))) {
+				$this->getConnection()->abortMultipartUpload($uploadInfo);
 			}
+			throw $e;
+		}
+	}
+
+
+	/**
+	 * @param string $urn the unified resource name used to identify the object
+	 * @param resource $stream stream with the data to write
+	 * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0
+	 * @throws \Exception when something goes wrong, message will be logged
+	 * @since 7.0.0
+	 */
+	public function writeObject($urn, $stream, string $mimetype = null) {
+		$psrStream = Utils::streamFor($stream);
+
+		// ($psrStream->isSeekable() && $psrStream->getSize() !== null) evaluates to true for a On-Seekable stream
+		// so the optimisation does not apply
+		$buffer = new Psr7\Stream(fopen("php://memory", 'rwb+'));
+		Utils::copyToStream($psrStream, $buffer, MultipartUploader::PART_MIN_SIZE);
+		$buffer->seek(0);
+		if ($buffer->getSize() < MultipartUploader::PART_MIN_SIZE) {
+			// buffer is fully seekable, so use it directly for the small upload
+			$this->writeSingle($urn, $buffer, $mimetype);
+		} else {
+			$loadStream = new Psr7\AppendStream([$buffer, $psrStream]);
+			$this->writeMultiPart($urn, $loadStream, $mimetype);
 		}
 	}