From d8e70c1c6d7809153473bf04f0d3883d5355a8a7 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Tue, 17 Oct 2023 15:47:40 +0300 Subject: [PATCH 01/16] s3Client::uploadPartCopy method added --- manifest.json | 3 +- src/Service/S3/CHANGELOG.md | 1 + .../S3/src/Input/UploadPartCopyRequest.php | 603 ++++++++++++++++++ .../S3/src/Result/UploadPartCopyOutput.php | 155 +++++ src/Service/S3/src/S3Client.php | 132 ++++ .../S3/src/ValueObject/CopyPartResult.php | 132 ++++ .../S3/tests/Integration/S3ClientTest.php | 19 + .../Unit/Input/UploadPartCopyRequestTest.php | 30 + .../Unit/Result/UploadPartCopyOutputTest.php | 28 + src/Service/S3/tests/Unit/S3ClientTest.php | 20 + 10 files changed, 1122 insertions(+), 1 deletion(-) create mode 100644 src/Service/S3/src/Input/UploadPartCopyRequest.php create mode 100644 src/Service/S3/src/Result/UploadPartCopyOutput.php create mode 100644 src/Service/S3/src/ValueObject/CopyPartResult.php create mode 100644 src/Service/S3/tests/Unit/Input/UploadPartCopyRequestTest.php create mode 100644 src/Service/S3/tests/Unit/Result/UploadPartCopyOutputTest.php diff --git a/manifest.json b/manifest.json index cd78673d8..b51dbdb26 100644 --- a/manifest.json +++ b/manifest.json @@ -577,7 +577,8 @@ "PutObject", "PutObjectAcl", "PutObjectTagging", - "UploadPart" + "UploadPart", + "UploadPartCopy" ] }, "Scheduler": { diff --git a/src/Service/S3/CHANGELOG.md b/src/Service/S3/CHANGELOG.md index e458b89db..aed00da78 100644 --- a/src/Service/S3/CHANGELOG.md +++ b/src/Service/S3/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - AWS api-change: This release adds a new field COMPLETED to the ReplicationStatus Enum. You can now use this field to validate the replication status of S3 objects using the AWS SDK. +- Added `S3Client::uploadPartCopy()` method ### Changed diff --git a/src/Service/S3/src/Input/UploadPartCopyRequest.php b/src/Service/S3/src/Input/UploadPartCopyRequest.php new file mode 100644 index 000000000..f84b97a09 --- /dev/null +++ b/src/Service/S3/src/Input/UploadPartCopyRequest.php @@ -0,0 +1,603 @@ +::accesspoint//object/`. For + * example, to copy the object `reports/january.pdf` through access point `my-access-point` owned by account + * `123456789012` in Region `us-west-2`, use the URL encoding of + * `arn:aws:s3:us-west-2:123456789012:accesspoint/my-access-point/object/reports/january.pdf`. The value must be URL + * encoded. + * + * > Amazon S3 supports copy operations using access points only when the source and destination buckets are in the + * > same Amazon Web Services Region. + * + * Alternatively, for objects accessed through Amazon S3 on Outposts, specify the ARN of the object as accessed in the + * format `arn:aws:s3-outposts:::outpost//object/`. For + * example, to copy the object `reports/january.pdf` through outpost `my-outpost` owned by account `123456789012` in + * Region `us-west-2`, use the URL encoding of + * `arn:aws:s3-outposts:us-west-2:123456789012:outpost/my-outpost/object/reports/january.pdf`. The value must be + * URL-encoded. + * + * To copy a specific version of an object, append `?versionId=` to the value (for example, + * `awsexamplebucket/reports/january.pdf?versionId=QUpfdndhfd8438MNFDN93jdnJFkdmqnh893`). If you don't specify a version + * ID, Amazon S3 copies the latest version of the source object. + * + * [^1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-points.html + * + * @required + * + * @var string|null + */ + private $copySource; + + /** + * Copies the object if its entity tag (ETag) matches the specified tag. + * + * @var string|null + */ + private $copySourceIfMatch; + + /** + * Copies the object if it has been modified since the specified time. + * + * @var \DateTimeImmutable|null + */ + private $copySourceIfModifiedSince; + + /** + * Copies the object if its entity tag (ETag) is different than the specified ETag. + * + * @var string|null + */ + private $copySourceIfNoneMatch; + + /** + * Copies the object if it hasn't been modified since the specified time. + * + * @var \DateTimeImmutable|null + */ + private $copySourceIfUnmodifiedSince; + + /** + * The range of bytes to copy from the source object. The range value must use the form bytes=first-last, where the + * first and last are the zero-based byte offsets to copy. For example, bytes=0-9 indicates that you want to copy the + * first 10 bytes of the source. You can copy a range only if the source object is greater than 5 MB. + * + * @var string|null + */ + private $copySourceRange; + + /** + * Object key for which the multipart upload was initiated. + * + * @required + * + * @var string|null + */ + private $key; + + /** + * Part number of part being copied. This is a positive integer between 1 and 10,000. + * + * @required + * + * @var int|null + */ + private $partNumber; + + /** + * Upload ID identifying the multipart upload whose part is being copied. + * + * @required + * + * @var string|null + */ + private $uploadId; + + /** + * Specifies the algorithm to use to when encrypting the object (for example, AES256). + * + * @var string|null + */ + private $sseCustomerAlgorithm; + + /** + * Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data. This value is used to store + * the object and then it is discarded; Amazon S3 does not store the encryption key. The key must be appropriate for use + * with the algorithm specified in the `x-amz-server-side-encryption-customer-algorithm` header. This must be the same + * encryption key specified in the initiate multipart upload request. + * + * @var string|null + */ + private $sseCustomerKey; + + /** + * Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a + * message integrity check to ensure that the encryption key was transmitted without error. + * + * @var string|null + */ + private $sseCustomerKeyMd5; + + /** + * Specifies the algorithm to use when decrypting the source object (for example, AES256). + * + * @var string|null + */ + private $copySourceSseCustomerAlgorithm; + + /** + * Specifies the customer-provided encryption key for Amazon S3 to use to decrypt the source object. The encryption key + * provided in this header must be one that was used when the source object was created. + * + * @var string|null + */ + private $copySourceSseCustomerKey; + + /** + * Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a + * message integrity check to ensure that the encryption key was transmitted without error. + * + * @var string|null + */ + private $copySourceSseCustomerKeyMd5; + + /** + * @var RequestPayer::*|null + */ + private $requestPayer; + + /** + * The account ID of the expected destination bucket owner. If the destination bucket is owned by a different account, + * the request fails with the HTTP status code `403 Forbidden` (access denied). + * + * @var string|null + */ + private $expectedBucketOwner; + + /** + * The account ID of the expected source bucket owner. If the source bucket is owned by a different account, the request + * fails with the HTTP status code `403 Forbidden` (access denied). + * + * @var string|null + */ + private $expectedSourceBucketOwner; + + /** + * @param array{ + * Bucket?: string, + * CopySource?: string, + * CopySourceIfMatch?: null|string, + * CopySourceIfModifiedSince?: null|\DateTimeImmutable|string, + * CopySourceIfNoneMatch?: null|string, + * CopySourceIfUnmodifiedSince?: null|\DateTimeImmutable|string, + * CopySourceRange?: null|string, + * Key?: string, + * PartNumber?: int, + * UploadId?: string, + * SSECustomerAlgorithm?: null|string, + * SSECustomerKey?: null|string, + * SSECustomerKeyMD5?: null|string, + * CopySourceSSECustomerAlgorithm?: null|string, + * CopySourceSSECustomerKey?: null|string, + * CopySourceSSECustomerKeyMD5?: null|string, + * RequestPayer?: null|RequestPayer::*, + * ExpectedBucketOwner?: null|string, + * ExpectedSourceBucketOwner?: null|string, + * '@region'?: string|null, + * } $input + */ + public function __construct(array $input = []) + { + $this->bucket = $input['Bucket'] ?? null; + $this->copySource = $input['CopySource'] ?? null; + $this->copySourceIfMatch = $input['CopySourceIfMatch'] ?? null; + $this->copySourceIfModifiedSince = !isset($input['CopySourceIfModifiedSince']) ? null : ($input['CopySourceIfModifiedSince'] instanceof \DateTimeImmutable ? $input['CopySourceIfModifiedSince'] : new \DateTimeImmutable($input['CopySourceIfModifiedSince'])); + $this->copySourceIfNoneMatch = $input['CopySourceIfNoneMatch'] ?? null; + $this->copySourceIfUnmodifiedSince = !isset($input['CopySourceIfUnmodifiedSince']) ? null : ($input['CopySourceIfUnmodifiedSince'] instanceof \DateTimeImmutable ? $input['CopySourceIfUnmodifiedSince'] : new \DateTimeImmutable($input['CopySourceIfUnmodifiedSince'])); + $this->copySourceRange = $input['CopySourceRange'] ?? null; + $this->key = $input['Key'] ?? null; + $this->partNumber = $input['PartNumber'] ?? null; + $this->uploadId = $input['UploadId'] ?? null; + $this->sseCustomerAlgorithm = $input['SSECustomerAlgorithm'] ?? null; + $this->sseCustomerKey = $input['SSECustomerKey'] ?? null; + $this->sseCustomerKeyMd5 = $input['SSECustomerKeyMD5'] ?? null; + $this->copySourceSseCustomerAlgorithm = $input['CopySourceSSECustomerAlgorithm'] ?? null; + $this->copySourceSseCustomerKey = $input['CopySourceSSECustomerKey'] ?? null; + $this->copySourceSseCustomerKeyMd5 = $input['CopySourceSSECustomerKeyMD5'] ?? null; + $this->requestPayer = $input['RequestPayer'] ?? null; + $this->expectedBucketOwner = $input['ExpectedBucketOwner'] ?? null; + $this->expectedSourceBucketOwner = $input['ExpectedSourceBucketOwner'] ?? null; + parent::__construct($input); + } + + /** + * @param array{ + * Bucket?: string, + * CopySource?: string, + * CopySourceIfMatch?: null|string, + * CopySourceIfModifiedSince?: null|\DateTimeImmutable|string, + * CopySourceIfNoneMatch?: null|string, + * CopySourceIfUnmodifiedSince?: null|\DateTimeImmutable|string, + * CopySourceRange?: null|string, + * Key?: string, + * PartNumber?: int, + * UploadId?: string, + * SSECustomerAlgorithm?: null|string, + * SSECustomerKey?: null|string, + * SSECustomerKeyMD5?: null|string, + * CopySourceSSECustomerAlgorithm?: null|string, + * CopySourceSSECustomerKey?: null|string, + * CopySourceSSECustomerKeyMD5?: null|string, + * RequestPayer?: null|RequestPayer::*, + * ExpectedBucketOwner?: null|string, + * ExpectedSourceBucketOwner?: null|string, + * '@region'?: string|null, + * }|UploadPartCopyRequest $input + */ + public static function create($input): self + { + return $input instanceof self ? $input : new self($input); + } + + public function getBucket(): ?string + { + return $this->bucket; + } + + public function getCopySource(): ?string + { + return $this->copySource; + } + + public function getCopySourceIfMatch(): ?string + { + return $this->copySourceIfMatch; + } + + public function getCopySourceIfModifiedSince(): ?\DateTimeImmutable + { + return $this->copySourceIfModifiedSince; + } + + public function getCopySourceIfNoneMatch(): ?string + { + return $this->copySourceIfNoneMatch; + } + + public function getCopySourceIfUnmodifiedSince(): ?\DateTimeImmutable + { + return $this->copySourceIfUnmodifiedSince; + } + + public function getCopySourceRange(): ?string + { + return $this->copySourceRange; + } + + public function getCopySourceSseCustomerAlgorithm(): ?string + { + return $this->copySourceSseCustomerAlgorithm; + } + + public function getCopySourceSseCustomerKey(): ?string + { + return $this->copySourceSseCustomerKey; + } + + public function getCopySourceSseCustomerKeyMd5(): ?string + { + return $this->copySourceSseCustomerKeyMd5; + } + + public function getExpectedBucketOwner(): ?string + { + return $this->expectedBucketOwner; + } + + public function getExpectedSourceBucketOwner(): ?string + { + return $this->expectedSourceBucketOwner; + } + + public function getKey(): ?string + { + return $this->key; + } + + public function getPartNumber(): ?int + { + return $this->partNumber; + } + + /** + * @return RequestPayer::*|null + */ + public function getRequestPayer(): ?string + { + return $this->requestPayer; + } + + public function getSseCustomerAlgorithm(): ?string + { + return $this->sseCustomerAlgorithm; + } + + public function getSseCustomerKey(): ?string + { + return $this->sseCustomerKey; + } + + public function getSseCustomerKeyMd5(): ?string + { + return $this->sseCustomerKeyMd5; + } + + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @internal + */ + public function request(): Request + { + // Prepare headers + $headers = ['content-type' => 'application/xml']; + if (null === $v = $this->copySource) { + throw new InvalidArgument(sprintf('Missing parameter "CopySource" for "%s". The value cannot be null.', __CLASS__)); + } + $headers['x-amz-copy-source'] = $v; + if (null !== $this->copySourceIfMatch) { + $headers['x-amz-copy-source-if-match'] = $this->copySourceIfMatch; + } + if (null !== $this->copySourceIfModifiedSince) { + $headers['x-amz-copy-source-if-modified-since'] = $this->copySourceIfModifiedSince->setTimezone(new \DateTimeZone('GMT'))->format(\DateTimeInterface::RFC7231); + } + if (null !== $this->copySourceIfNoneMatch) { + $headers['x-amz-copy-source-if-none-match'] = $this->copySourceIfNoneMatch; + } + if (null !== $this->copySourceIfUnmodifiedSince) { + $headers['x-amz-copy-source-if-unmodified-since'] = $this->copySourceIfUnmodifiedSince->setTimezone(new \DateTimeZone('GMT'))->format(\DateTimeInterface::RFC7231); + } + if (null !== $this->copySourceRange) { + $headers['x-amz-copy-source-range'] = $this->copySourceRange; + } + if (null !== $this->sseCustomerAlgorithm) { + $headers['x-amz-server-side-encryption-customer-algorithm'] = $this->sseCustomerAlgorithm; + } + if (null !== $this->sseCustomerKey) { + $headers['x-amz-server-side-encryption-customer-key'] = $this->sseCustomerKey; + } + if (null !== $this->sseCustomerKeyMd5) { + $headers['x-amz-server-side-encryption-customer-key-MD5'] = $this->sseCustomerKeyMd5; + } + if (null !== $this->copySourceSseCustomerAlgorithm) { + $headers['x-amz-copy-source-server-side-encryption-customer-algorithm'] = $this->copySourceSseCustomerAlgorithm; + } + if (null !== $this->copySourceSseCustomerKey) { + $headers['x-amz-copy-source-server-side-encryption-customer-key'] = $this->copySourceSseCustomerKey; + } + if (null !== $this->copySourceSseCustomerKeyMd5) { + $headers['x-amz-copy-source-server-side-encryption-customer-key-MD5'] = $this->copySourceSseCustomerKeyMd5; + } + if (null !== $this->requestPayer) { + if (!RequestPayer::exists($this->requestPayer)) { + throw new InvalidArgument(sprintf('Invalid parameter "RequestPayer" for "%s". The value "%s" is not a valid "RequestPayer".', __CLASS__, $this->requestPayer)); + } + $headers['x-amz-request-payer'] = $this->requestPayer; + } + if (null !== $this->expectedBucketOwner) { + $headers['x-amz-expected-bucket-owner'] = $this->expectedBucketOwner; + } + if (null !== $this->expectedSourceBucketOwner) { + $headers['x-amz-source-expected-bucket-owner'] = $this->expectedSourceBucketOwner; + } + + // Prepare query + $query = []; + if (null === $v = $this->partNumber) { + throw new InvalidArgument(sprintf('Missing parameter "PartNumber" for "%s". The value cannot be null.', __CLASS__)); + } + $query['partNumber'] = (string) $v; + if (null === $v = $this->uploadId) { + throw new InvalidArgument(sprintf('Missing parameter "UploadId" for "%s". The value cannot be null.', __CLASS__)); + } + $query['uploadId'] = $v; + + // Prepare URI + $uri = []; + if (null === $v = $this->bucket) { + throw new InvalidArgument(sprintf('Missing parameter "Bucket" for "%s". The value cannot be null.', __CLASS__)); + } + $uri['Bucket'] = $v; + if (null === $v = $this->key) { + throw new InvalidArgument(sprintf('Missing parameter "Key" for "%s". The value cannot be null.', __CLASS__)); + } + $uri['Key'] = $v; + $uriString = '/' . rawurlencode($uri['Bucket']) . '/' . str_replace('%2F', '/', rawurlencode($uri['Key'])); + + // Prepare Body + $body = ''; + + // Return the Request + return new Request('PUT', $uriString, $query, $headers, StreamFactory::create($body)); + } + + public function setBucket(?string $value): self + { + $this->bucket = $value; + + return $this; + } + + public function setCopySource(?string $value): self + { + $this->copySource = $value; + + return $this; + } + + public function setCopySourceIfMatch(?string $value): self + { + $this->copySourceIfMatch = $value; + + return $this; + } + + public function setCopySourceIfModifiedSince(?\DateTimeImmutable $value): self + { + $this->copySourceIfModifiedSince = $value; + + return $this; + } + + public function setCopySourceIfNoneMatch(?string $value): self + { + $this->copySourceIfNoneMatch = $value; + + return $this; + } + + public function setCopySourceIfUnmodifiedSince(?\DateTimeImmutable $value): self + { + $this->copySourceIfUnmodifiedSince = $value; + + return $this; + } + + public function setCopySourceRange(?string $value): self + { + $this->copySourceRange = $value; + + return $this; + } + + public function setCopySourceSseCustomerAlgorithm(?string $value): self + { + $this->copySourceSseCustomerAlgorithm = $value; + + return $this; + } + + public function setCopySourceSseCustomerKey(?string $value): self + { + $this->copySourceSseCustomerKey = $value; + + return $this; + } + + public function setCopySourceSseCustomerKeyMd5(?string $value): self + { + $this->copySourceSseCustomerKeyMd5 = $value; + + return $this; + } + + public function setExpectedBucketOwner(?string $value): self + { + $this->expectedBucketOwner = $value; + + return $this; + } + + public function setExpectedSourceBucketOwner(?string $value): self + { + $this->expectedSourceBucketOwner = $value; + + return $this; + } + + public function setKey(?string $value): self + { + $this->key = $value; + + return $this; + } + + public function setPartNumber(?int $value): self + { + $this->partNumber = $value; + + return $this; + } + + /** + * @param RequestPayer::*|null $value + */ + public function setRequestPayer(?string $value): self + { + $this->requestPayer = $value; + + return $this; + } + + public function setSseCustomerAlgorithm(?string $value): self + { + $this->sseCustomerAlgorithm = $value; + + return $this; + } + + public function setSseCustomerKey(?string $value): self + { + $this->sseCustomerKey = $value; + + return $this; + } + + public function setSseCustomerKeyMd5(?string $value): self + { + $this->sseCustomerKeyMd5 = $value; + + return $this; + } + + public function setUploadId(?string $value): self + { + $this->uploadId = $value; + + return $this; + } +} diff --git a/src/Service/S3/src/Result/UploadPartCopyOutput.php b/src/Service/S3/src/Result/UploadPartCopyOutput.php new file mode 100644 index 000000000..667996ba8 --- /dev/null +++ b/src/Service/S3/src/Result/UploadPartCopyOutput.php @@ -0,0 +1,155 @@ +initialize(); + + return $this->bucketKeyEnabled; + } + + public function getCopyPartResult(): ?CopyPartResult + { + $this->initialize(); + + return $this->copyPartResult; + } + + public function getCopySourceVersionId(): ?string + { + $this->initialize(); + + return $this->copySourceVersionId; + } + + /** + * @return RequestCharged::*|null + */ + public function getRequestCharged(): ?string + { + $this->initialize(); + + return $this->requestCharged; + } + + /** + * @return ServerSideEncryption::*|null + */ + public function getServerSideEncryption(): ?string + { + $this->initialize(); + + return $this->serverSideEncryption; + } + + public function getSseCustomerAlgorithm(): ?string + { + $this->initialize(); + + return $this->sseCustomerAlgorithm; + } + + public function getSseCustomerKeyMd5(): ?string + { + $this->initialize(); + + return $this->sseCustomerKeyMd5; + } + + public function getSseKmsKeyId(): ?string + { + $this->initialize(); + + return $this->sseKmsKeyId; + } + + protected function populateResult(Response $response): void + { + $headers = $response->getHeaders(); + + $this->copySourceVersionId = $headers['x-amz-copy-source-version-id'][0] ?? null; + $this->serverSideEncryption = $headers['x-amz-server-side-encryption'][0] ?? null; + $this->sseCustomerAlgorithm = $headers['x-amz-server-side-encryption-customer-algorithm'][0] ?? null; + $this->sseCustomerKeyMd5 = $headers['x-amz-server-side-encryption-customer-key-md5'][0] ?? null; + $this->sseKmsKeyId = $headers['x-amz-server-side-encryption-aws-kms-key-id'][0] ?? null; + $this->bucketKeyEnabled = isset($headers['x-amz-server-side-encryption-bucket-key-enabled'][0]) ? filter_var($headers['x-amz-server-side-encryption-bucket-key-enabled'][0], \FILTER_VALIDATE_BOOLEAN) : null; + $this->requestCharged = $headers['x-amz-request-charged'][0] ?? null; + + $data = new \SimpleXMLElement($response->getContent()); + $this->copyPartResult = new CopyPartResult([ + 'ETag' => ($v = $data->ETag) ? (string) $v : null, + 'LastModified' => ($v = $data->LastModified) ? new \DateTimeImmutable((string) $v) : null, + 'ChecksumCRC32' => ($v = $data->ChecksumCRC32) ? (string) $v : null, + 'ChecksumCRC32C' => ($v = $data->ChecksumCRC32C) ? (string) $v : null, + 'ChecksumSHA1' => ($v = $data->ChecksumSHA1) ? (string) $v : null, + 'ChecksumSHA256' => ($v = $data->ChecksumSHA256) ? (string) $v : null, + ]); + } +} diff --git a/src/Service/S3/src/S3Client.php b/src/Service/S3/src/S3Client.php index 0f3465992..0fd24f69f 100644 --- a/src/Service/S3/src/S3Client.php +++ b/src/Service/S3/src/S3Client.php @@ -56,6 +56,7 @@ use AsyncAws\S3\Input\PutObjectAclRequest; use AsyncAws\S3\Input\PutObjectRequest; use AsyncAws\S3\Input\PutObjectTaggingRequest; +use AsyncAws\S3\Input\UploadPartCopyRequest; use AsyncAws\S3\Input\UploadPartRequest; use AsyncAws\S3\Result\AbortMultipartUploadOutput; use AsyncAws\S3\Result\BucketExistsWaiter; @@ -82,6 +83,7 @@ use AsyncAws\S3\Result\PutObjectAclOutput; use AsyncAws\S3\Result\PutObjectOutput; use AsyncAws\S3\Result\PutObjectTaggingOutput; +use AsyncAws\S3\Result\UploadPartCopyOutput; use AsyncAws\S3\Result\UploadPartOutput; use AsyncAws\S3\Signer\SignerV4ForS3; use AsyncAws\S3\ValueObject\AccessControlPolicy; @@ -2432,6 +2434,136 @@ public function uploadPart($input): UploadPartOutput return new UploadPartOutput($response); } + /** + * Uploads a part by copying data from an existing object as data source. You specify the data source by adding the + * request header `x-amz-copy-source` in your request and a byte range by adding the request header + * `x-amz-copy-source-range` in your request. + * + * For information about maximum and minimum part sizes and other multipart upload specifications, see Multipart upload + * limits [^1] in the *Amazon S3 User Guide*. + * + * > Instead of using an existing object as part data, you might use the UploadPart [^2] action and provide data in your + * > request. + * + * You must initiate a multipart upload before you can upload any part. In response to your initiate request. Amazon S3 + * returns a unique identifier, the upload ID, that you must include in your upload part request. + * + * For more information about using the `UploadPartCopy` operation, see the following: + * + * - For conceptual information about multipart uploads, see Uploading Objects Using Multipart Upload [^3] in the + * *Amazon S3 User Guide*. + * - For information about permissions required to use the multipart upload API, see Multipart Upload and Permissions + * [^4] in the *Amazon S3 User Guide*. + * - For information about copying objects using a single atomic action vs. a multipart upload, see Operations on + * Objects [^5] in the *Amazon S3 User Guide*. + * - For information about using server-side encryption with customer-provided encryption keys with the `UploadPartCopy` + * operation, see CopyObject [^6] and UploadPart [^7]. + * + * Note the following additional considerations about the request headers `x-amz-copy-source-if-match`, + * `x-amz-copy-source-if-none-match`, `x-amz-copy-source-if-unmodified-since`, and + * `x-amz-copy-source-if-modified-since`: + * + * - **Consideration 1** - If both of the `x-amz-copy-source-if-match` and `x-amz-copy-source-if-unmodified-since` + * headers are present in the request as follows: + * + * `x-amz-copy-source-if-match` condition evaluates to `true`, and; + * + * `x-amz-copy-source-if-unmodified-since` condition evaluates to `false`; + * + * Amazon S3 returns `200 OK` and copies the data. + * - **Consideration 2** - If both of the `x-amz-copy-source-if-none-match` and `x-amz-copy-source-if-modified-since` + * headers are present in the request as follows: + * + * `x-amz-copy-source-if-none-match` condition evaluates to `false`, and; + * + * `x-amz-copy-source-if-modified-since` condition evaluates to `true`; + * + * Amazon S3 returns `412 Precondition Failed` response code. + * + * - `Versioning`: + * + * If your bucket has versioning enabled, you could have multiple versions of the same object. By default, + * `x-amz-copy-source` identifies the current version of the object to copy. If the current version is a delete marker + * and you don't specify a versionId in the `x-amz-copy-source`, Amazon S3 returns a 404 error, because the object + * does not exist. If you specify versionId in the `x-amz-copy-source` and the versionId is a delete marker, Amazon S3 + * returns an HTTP 400 error, because you are not allowed to specify a delete marker as a version for the + * `x-amz-copy-source`. + * + * You can optionally specify a specific version of the source object to copy by adding the `versionId` subresource as + * shown in the following example: + * + * `x-amz-copy-source: /bucket/object?versionId=version id` + * - `Special errors`: + * + * - - *Code: NoSuchUpload* + * - - *Cause: The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload + * - might have been aborted or completed.* + * - - *HTTP Status Code: 404 Not Found* + * - + * - - *Code: InvalidRequest* + * - - *Cause: The specified copy source is not supported as a byte-range copy source.* + * - - *HTTP Status Code: 400 Bad Request* + * - + * + * + * The following operations are related to `UploadPartCopy`: + * + * - CreateMultipartUpload [^8] + * - UploadPart [^9] + * - CompleteMultipartUpload [^10] + * - AbortMultipartUpload [^11] + * - ListParts [^12] + * - ListMultipartUploads [^13] + * + * [^1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + * [^2]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html + * [^3]: https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + * [^4]: https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html + * [^5]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectOperations.html + * [^6]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + * [^7]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html + * [^8]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html + * [^9]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html + * [^10]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html + * [^11]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html + * [^12]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html + * [^13]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html + * + * @see http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html + * @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html + * @see https://docs.aws.amazon.com/aws-sdk-php/v3/api/api-s3-2006-03-01.html#uploadpartcopy + * + * @param array{ + * Bucket: string, + * CopySource: string, + * CopySourceIfMatch?: null|string, + * CopySourceIfModifiedSince?: null|\DateTimeImmutable|string, + * CopySourceIfNoneMatch?: null|string, + * CopySourceIfUnmodifiedSince?: null|\DateTimeImmutable|string, + * CopySourceRange?: null|string, + * Key: string, + * PartNumber: int, + * UploadId: string, + * SSECustomerAlgorithm?: null|string, + * SSECustomerKey?: null|string, + * SSECustomerKeyMD5?: null|string, + * CopySourceSSECustomerAlgorithm?: null|string, + * CopySourceSSECustomerKey?: null|string, + * CopySourceSSECustomerKeyMD5?: null|string, + * RequestPayer?: null|RequestPayer::*, + * ExpectedBucketOwner?: null|string, + * ExpectedSourceBucketOwner?: null|string, + * '@region'?: string|null, + * }|UploadPartCopyRequest $input + */ + public function uploadPartCopy($input): UploadPartCopyOutput + { + $input = UploadPartCopyRequest::create($input); + $response = $this->getResponse($input->request(), new RequestContext(['operation' => 'UploadPartCopy', 'region' => $input->getRegion()])); + + return new UploadPartCopyOutput($response); + } + protected function getAwsErrorFactory(): AwsErrorFactoryInterface { return new XmlAwsErrorFactory(); diff --git a/src/Service/S3/src/ValueObject/CopyPartResult.php b/src/Service/S3/src/ValueObject/CopyPartResult.php new file mode 100644 index 000000000..079f66af0 --- /dev/null +++ b/src/Service/S3/src/ValueObject/CopyPartResult.php @@ -0,0 +1,132 @@ +etag = $input['ETag'] ?? null; + $this->lastModified = $input['LastModified'] ?? null; + $this->checksumCrc32 = $input['ChecksumCRC32'] ?? null; + $this->checksumCrc32C = $input['ChecksumCRC32C'] ?? null; + $this->checksumSha1 = $input['ChecksumSHA1'] ?? null; + $this->checksumSha256 = $input['ChecksumSHA256'] ?? null; + } + + /** + * @param array{ + * ETag?: null|string, + * LastModified?: null|\DateTimeImmutable, + * ChecksumCRC32?: null|string, + * ChecksumCRC32C?: null|string, + * ChecksumSHA1?: null|string, + * ChecksumSHA256?: null|string, + * }|CopyPartResult $input + */ + public static function create($input): self + { + return $input instanceof self ? $input : new self($input); + } + + public function getChecksumCrc32(): ?string + { + return $this->checksumCrc32; + } + + public function getChecksumCrc32C(): ?string + { + return $this->checksumCrc32C; + } + + public function getChecksumSha1(): ?string + { + return $this->checksumSha1; + } + + public function getChecksumSha256(): ?string + { + return $this->checksumSha256; + } + + public function getEtag(): ?string + { + return $this->etag; + } + + public function getLastModified(): ?\DateTimeImmutable + { + return $this->lastModified; + } +} diff --git a/src/Service/S3/tests/Integration/S3ClientTest.php b/src/Service/S3/tests/Integration/S3ClientTest.php index a2339bb13..3f8422d68 100644 --- a/src/Service/S3/tests/Integration/S3ClientTest.php +++ b/src/Service/S3/tests/Integration/S3ClientTest.php @@ -35,6 +35,7 @@ use AsyncAws\S3\Input\PutObjectAclRequest; use AsyncAws\S3\Input\PutObjectRequest; use AsyncAws\S3\Input\PutObjectTaggingRequest; +use AsyncAws\S3\Input\UploadPartCopyRequest; use AsyncAws\S3\Input\UploadPartRequest; use AsyncAws\S3\Result\PutObjectOutput; use AsyncAws\S3\S3Client; @@ -951,6 +952,24 @@ public function testUploadPart(): void self::assertEquals(200, $result->info()['status']); } + public function testUploadPartCopy(): void + { + $client = $this->getClient(); + + $input = new UploadPartCopyRequest([ + 'Bucket' => 'foo', + 'Key' => 'destination-object.txt', + 'PartNumber' => 1, + 'UploadId' => '123', + 'CopySource' => 'foo/bar', + ]); + $result = $client->uploadPartCopy($input); + + $result->resolve(); + + self::assertEquals(200, $result->info()['status']); + } + private function getClient(): S3Client { return new S3Client([ diff --git a/src/Service/S3/tests/Unit/Input/UploadPartCopyRequestTest.php b/src/Service/S3/tests/Unit/Input/UploadPartCopyRequestTest.php new file mode 100644 index 000000000..7326ac8ca --- /dev/null +++ b/src/Service/S3/tests/Unit/Input/UploadPartCopyRequestTest.php @@ -0,0 +1,30 @@ + 'example-bucket', + 'Key' => 'copy-movie.m2ts', + 'CopySource' => 'example-bucket/my-movie.m2ts', + 'CopySourceRange' => 'bytes=0-1', + 'PartNumber' => 1, + 'UploadId' => 'VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR', + ]); + + // see example-1.json from SDK + $expected = ' + PUT /example-bucket/copy-movie.m2ts?partNumber=1&uploadId=VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR HTTP/1.1 + Content-Type: application/xml + x-amz-copy-source: example-bucket/my-movie.m2ts + x-amz-copy-source-range: bytes=0-1'; + + self::assertRequestEqualsHttpRequest($expected, $input->request()); + } +} diff --git a/src/Service/S3/tests/Unit/Result/UploadPartCopyOutputTest.php b/src/Service/S3/tests/Unit/Result/UploadPartCopyOutputTest.php new file mode 100644 index 000000000..ecacd5c13 --- /dev/null +++ b/src/Service/S3/tests/Unit/Result/UploadPartCopyOutputTest.php @@ -0,0 +1,28 @@ + + "b0c6f0e7e054ab8fa2536a2677f8734d" + 2016-12-29T21:24:43.000Z + '); + + $client = new MockHttpClient($response); + $result = new UploadPartCopyOutput(new Response($client->request('POST', 'http://localhost'), $client, new NullLogger())); + + self::assertSame('"b0c6f0e7e054ab8fa2536a2677f8734d"', $result->getCopyPartResult()->getEtag()); + self::assertEquals(new \DateTimeImmutable('2016-12-29T21:24:43.000Z'), $result->getCopyPartResult()->getLastModified()); + } +} diff --git a/src/Service/S3/tests/Unit/S3ClientTest.php b/src/Service/S3/tests/Unit/S3ClientTest.php index 955697fb5..805209041 100644 --- a/src/Service/S3/tests/Unit/S3ClientTest.php +++ b/src/Service/S3/tests/Unit/S3ClientTest.php @@ -32,6 +32,7 @@ use AsyncAws\S3\Input\PutObjectAclRequest; use AsyncAws\S3\Input\PutObjectRequest; use AsyncAws\S3\Input\PutObjectTaggingRequest; +use AsyncAws\S3\Input\UploadPartCopyRequest; use AsyncAws\S3\Input\UploadPartRequest; use AsyncAws\S3\Result\AbortMultipartUploadOutput; use AsyncAws\S3\Result\CompleteMultipartUploadOutput; @@ -54,6 +55,7 @@ use AsyncAws\S3\Result\PutObjectAclOutput; use AsyncAws\S3\Result\PutObjectOutput; use AsyncAws\S3\Result\PutObjectTaggingOutput; +use AsyncAws\S3\Result\UploadPartCopyOutput; use AsyncAws\S3\Result\UploadPartOutput; use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CORSConfiguration; @@ -540,4 +542,22 @@ public function testUploadPart(): void self::assertInstanceOf(UploadPartOutput::class, $result); self::assertFalse($result->info()['resolved']); } + + public function testUploadPartCopy(): void + { + $client = new S3Client([], new NullProvider(), new MockHttpClient()); + + $input = new UploadPartCopyRequest([ + 'Bucket' => 'destination-bucket', + 'CopySource' => 'source-bucket/image.png', + + 'Key' => 'copy-image.png', + 'PartNumber' => 1337, + 'UploadId' => '123', + ]); + $result = $client->uploadPartCopy($input); + + self::assertInstanceOf(UploadPartCopyOutput::class, $result); + self::assertFalse($result->info()['resolved']); + } } From c66b0f984f65998aeb2d54db3412695ca2d83377 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Tue, 17 Oct 2023 16:00:06 +0300 Subject: [PATCH 02/16] SimpleS3Client::copy method added --- docs/integration/simple-s3.md | 3 + src/Integration/Aws/SimpleS3/CHANGELOG.md | 4 + .../Aws/SimpleS3/src/SimpleS3Client.php | 96 +++++++++++++++++++ .../tests/Unit/SimpleS3ClientTest.php | 71 ++++++++++++++ 4 files changed, 174 insertions(+) diff --git a/docs/integration/simple-s3.md b/docs/integration/simple-s3.md index 70a68fe70..bdc8993bd 100644 --- a/docs/integration/simple-s3.md +++ b/docs/integration/simple-s3.md @@ -32,6 +32,9 @@ $resource = \fopen('/path/to/cat/image.jpg', 'r'); $s3->upload('my-image-bucket', 'photos/cat_2.jpg', $resource); $s3->upload('my-image-bucket', 'photos/cat_2.txt', 'I like this cat'); +// Copy objects between buckets +$s3->copy('source-bucket', 'source-key', 'destination-bucket', 'destination-key'); + // Check if a file exists $s3->has('my-image-bucket', 'photos/cat_2.jpg'); // true diff --git a/src/Integration/Aws/SimpleS3/CHANGELOG.md b/src/Integration/Aws/SimpleS3/CHANGELOG.md index 1f219e08e..9e2f1c79b 100644 --- a/src/Integration/Aws/SimpleS3/CHANGELOG.md +++ b/src/Integration/Aws/SimpleS3/CHANGELOG.md @@ -8,6 +8,10 @@ - Upgrade to `async-aws/s3` 2.0 +### Added + +- Added `SimpleS3Client::copy()` method + ## 1.1.1 ### Changed diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 2b62227fb..3aef20585 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -7,10 +7,15 @@ use AsyncAws\Core\Stream\FixedSizeStream; use AsyncAws\Core\Stream\ResultStream; use AsyncAws\Core\Stream\StreamFactory; +use AsyncAws\S3\Input\CompleteMultipartUploadRequest; +use AsyncAws\S3\Input\CopyObjectRequest; +use AsyncAws\S3\Input\CreateMultipartUploadRequest; use AsyncAws\S3\Input\GetObjectRequest; +use AsyncAws\S3\Input\UploadPartCopyRequest; use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CompletedMultipartUpload; use AsyncAws\S3\ValueObject\CompletedPart; +use AsyncAws\S3\ValueObject\CopyPartResult; /** * A simplified S3 client that hides some of the complexity of working with S3. @@ -47,6 +52,73 @@ public function has(string $bucket, string $key): bool return $this->objectExists(['Bucket' => $bucket, 'Key' => $key])->isSuccess(); } + /** + * @param array{ + * ACL?: \AsyncAws\S3\Enum\ObjectCannedACL::*, + * CacheControl?: string, + * ContentLength?: int, + * ContentType?: string, + * Metadata?: array, + * PartSize?: int, + * } $options + */ + public function copy(string $srcBucket, string $srcKey, string $destBucket, string $destKey, array $options = []): void + { + $megabyte = 1024 * 1024; + if (!empty($options['ContentLength'])) { + $contentLength = (int) $options['ContentLength']; + unset($options['ContentLength']); + } else { + $contentLength = (int) $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey])->getContentLength(); + } + + /* + * The maximum number of parts is 10.000. The partSize must be a power of 2. + * We default this to 64MB per part. That means that we only support to copy + * files smaller than 64 * 10 000 = 640GB. If you are coping larger files, + * please set PartSize to a higher number, like 128, 256 or 512. (Max 4096). + */ + $partSize = ($options['PartSize'] ?? 64) * $megabyte; + unset($options['PartSize']); + + // If file is less than 5GB, use normal atomic copy + if ($contentLength < 5120 * $megabyte) { + $this->copyObject( + CopyObjectRequest::create( + array_merge($options, ['Bucket' => $destBucket, 'Key' => $destKey, 'CopySource' => "{$srcBucket}/{$srcKey}"]) + ) + ); + + return; + } + + /** @var string $uploadId */ + $uploadId = $this->createMultipartUpload( + CreateMultipartUploadRequest::create( + array_merge($options, ['Bucket' => $destBucket, 'Key' => $destKey]) + ) + )->getUploadId(); + + $partIndex = 1; + $startByte = 0; + while ($startByte < $contentLength) { + $endByte = min($startByte + $partSize, $contentLength) - 1; + + $parts[] = $this->doMultipartCopy($destBucket, $destKey, $uploadId, $partIndex, "{$srcBucket}/{$srcKey}", $startByte, $endByte); + + $startByte += $partSize; + $partIndex ++; + } + $this->completeMultipartUpload( + CompleteMultipartUploadRequest::create([ + 'Bucket' => $destBucket, + 'Key' => $destKey, + 'UploadId' => $uploadId, + 'MultipartUpload' => new CompletedMultipartUpload(['Parts' => $parts]), + ]) + ); + } + /** * @param string|resource|(callable(int): string)|iterable $object * @param array{ @@ -195,4 +267,28 @@ private function doSmallFileUpload(array $options, string $bucket, string $key, 'Body' => $object, ])); } + + private function doMultipartCopy(string $bucket, string $key, string $uploadId, int $partNumber, string $copySource, int $startByte, int $endByte): CompletedPart + { + try { + $response = $this->uploadPartCopy( + UploadPartCopyRequest::create([ + 'Bucket' => $bucket, + 'Key' => $key, + 'UploadId' => $uploadId, + 'CopySource' => $copySource, + 'CopySourceRange' => sprintf('bytes=%d-%d', $startByte, $endByte), + 'PartNumber' => $partNumber, + ]) + ); + /** @var CopyPartResult $copyPartResult */ + $copyPartResult = $response->getCopyPartResult(); + + return new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $partNumber]); + } catch (\Throwable $e) { + $this->abortMultipartUpload(['Bucket' => $bucket, 'Key' => $key, 'UploadId' => $uploadId]); + + throw $e; + } + } } diff --git a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php index 32c00a92a..d5bd0401d 100644 --- a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php +++ b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php @@ -6,7 +6,11 @@ use AsyncAws\Core\Credentials\NullProvider; use AsyncAws\Core\Test\ResultMockFactory; +use AsyncAws\S3\Input\CompleteMultipartUploadRequest; use AsyncAws\S3\Result\CreateMultipartUploadOutput; +use AsyncAws\S3\Result\HeadObjectOutput; +use AsyncAws\S3\Result\UploadPartCopyOutput; +use AsyncAws\S3\ValueObject\CopyPartResult; use AsyncAws\SimpleS3\SimpleS3Client; use PHPUnit\Framework\TestCase; use Symfony\Component\HttpClient\MockHttpClient; @@ -137,6 +141,73 @@ public function testUploadSmallFileEmptyClosure() }); } + public function testCopySmallFileWithProvidedLength() + { + $megabyte = 1024 * 1024; + $s3 = $this->getMockBuilder(SimpleS3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload']) + ->getMock(); + + $s3->expects(self::never())->method('createMultipartUpload'); + $s3->expects(self::never())->method('abortMultipartUpload'); + $s3->expects(self::never())->method('completeMultipartUpload'); + $s3->expects(self::once())->method('copyObject'); + + $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt', ['ContentLength' => 5 * $megabyte]); + } + + public function testCopySmallFileWithoutProvidedLength() + { + $megabyte = 1024 * 1024; + $s3 = $this->getMockBuilder(SimpleS3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload', 'headObject']) + ->getMock(); + + $s3->expects(self::never())->method('createMultipartUpload'); + $s3->expects(self::never())->method('abortMultipartUpload'); + $s3->expects(self::never())->method('completeMultipartUpload'); + $s3->expects(self::once())->method('copyObject'); + $s3->expects(self::once())->method('headObject') + ->willReturn(ResultMockFactory::create(HeadObjectOutput::class, ['ContentLength' => 50 * $megabyte])); + + $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt'); + } + + public function testCopyLargeFile() + { + $megabyte = 1024 * 1024; + $uploadedParts = 0; + $completedParts = 0; + + $s3 = $this->getMockBuilder(SimpleS3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload', 'uploadPartCopy']) + ->getMock(); + + $s3->expects(self::once())->method('createMultipartUpload') + ->willReturn(ResultMockFactory::create(CreateMultipartUploadOutput::class, ['UploadId' => '4711'])); + $s3->expects(self::never())->method('abortMultipartUpload'); + $s3->expects(self::never())->method('copyObject'); + $s3->expects(self::any())->method('uploadPartCopy') + ->with(self::callback(function () use (&$uploadedParts) { + ++$uploadedParts; + + return true; + })) + ->willReturn(ResultMockFactory::create(UploadPartCopyOutput::class, ['copyPartResult' => new CopyPartResult(['ETag' => 'etag-4711'])])); + $s3->expects(self::once())->method('completeMultipartUpload')->with(self::callback(function (CompleteMultipartUploadRequest $request) use (&$completedParts) { + $completedParts = \count($request->getMultipartUpload()->getParts()); + + return true; + })); + + $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt', ['ContentLength' => 6144 * $megabyte]); + + self::assertEquals($completedParts, $uploadedParts); + } + private function assertSmallFileUpload(\Closure $callback, string $bucket, string $file, $object): void { $s3 = $this->getMockBuilder(SimpleS3Client::class) From 7587b8e9bd130b808e37176d53b432af54c06f2a Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Tue, 17 Oct 2023 18:08:15 +0300 Subject: [PATCH 03/16] Fix client --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 3aef20585..1866153e8 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -101,6 +101,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $partIndex = 1; $startByte = 0; + $parts = []; while ($startByte < $contentLength) { $endByte = min($startByte + $partSize, $contentLength) - 1; From 90121f12027f823f86943e070323e8e01c355d36 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 18 Oct 2023 12:31:54 +0300 Subject: [PATCH 04/16] Copy multipart concurrency added --- .../Aws/SimpleS3/src/SimpleS3Client.php | 87 ++++++++++++------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 1866153e8..6dcf1b96a 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -7,6 +7,7 @@ use AsyncAws\Core\Stream\FixedSizeStream; use AsyncAws\Core\Stream\ResultStream; use AsyncAws\Core\Stream\StreamFactory; +use AsyncAws\S3\Input\AbortMultipartUploadRequest; use AsyncAws\S3\Input\CompleteMultipartUploadRequest; use AsyncAws\S3\Input\CopyObjectRequest; use AsyncAws\S3\Input\CreateMultipartUploadRequest; @@ -72,6 +73,15 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $contentLength = (int) $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey])->getContentLength(); } + $concurrency = (int) ($options["concurrency"] ?? 25); + unset($options['concurrency']); + if ($concurrency > 500) { + $concurrency = 500; + } + if ($concurrency < 1) { + $concurrency = 25; + } + /* * The maximum number of parts is 10.000. The partSize must be a power of 2. * We default this to 64MB per part. That means that we only support to copy @@ -99,17 +109,52 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri ) )->getUploadId(); - $partIndex = 1; - $startByte = 0; - $parts = []; + $partNumber = 1; + $startByte = 0; + $parts = []; while ($startByte < $contentLength) { - $endByte = min($startByte + $partSize, $contentLength) - 1; - - $parts[] = $this->doMultipartCopy($destBucket, $destKey, $uploadId, $partIndex, "{$srcBucket}/{$srcKey}", $startByte, $endByte); - - $startByte += $partSize; - $partIndex ++; + $parallelChunks = $concurrency; + $responses = []; + while ($startByte < $contentLength && $parallelChunks > 0) { + $endByte = min($startByte + $partSize, $contentLength) - 1; + $responses[$partNumber] = $this->uploadPartCopy( + UploadPartCopyRequest::create([ + 'Bucket' => $destBucket, + 'Key' => $destKey, + 'UploadId' => $uploadId, + 'CopySource' => "{$srcBucket}/{$srcKey}", + 'CopySourceRange' => "bytes={$startByte}-{$endByte}", + 'PartNumber' => $partNumber, + ]) + ); + + $startByte += $partSize; + $partNumber ++; + $parallelChunks --; + } + $error = null; + foreach ($responses as $idx => $response) { + try { + $copyPartResult = $response->getCopyPartResult(); + $parts[] = new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $idx]); + } catch (\Throwable $e) { + $error = $e; + break; + } + } + if ($error) { + foreach ($responses as $response) { + try { + $response->cancel(); + } catch (\Throwable $e) { + continue; + } + } + $this->abortMultipartUpload(AbortMultipartUploadRequest::create(['Bucket' => $destBucket, 'Key' => $destKey, 'UploadId' => $uploadId])); + throw $error; + } } + $this->completeMultipartUpload( CompleteMultipartUploadRequest::create([ 'Bucket' => $destBucket, @@ -268,28 +313,4 @@ private function doSmallFileUpload(array $options, string $bucket, string $key, 'Body' => $object, ])); } - - private function doMultipartCopy(string $bucket, string $key, string $uploadId, int $partNumber, string $copySource, int $startByte, int $endByte): CompletedPart - { - try { - $response = $this->uploadPartCopy( - UploadPartCopyRequest::create([ - 'Bucket' => $bucket, - 'Key' => $key, - 'UploadId' => $uploadId, - 'CopySource' => $copySource, - 'CopySourceRange' => sprintf('bytes=%d-%d', $startByte, $endByte), - 'PartNumber' => $partNumber, - ]) - ); - /** @var CopyPartResult $copyPartResult */ - $copyPartResult = $response->getCopyPartResult(); - - return new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $partNumber]); - } catch (\Throwable $e) { - $this->abortMultipartUpload(['Bucket' => $bucket, 'Key' => $key, 'UploadId' => $uploadId]); - - throw $e; - } - } } From 83ac70af2199571bb69122b157b60f24c015221d Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 18 Oct 2023 12:36:47 +0300 Subject: [PATCH 05/16] phpstan/psalm fix --- .../Aws/SimpleS3/src/SimpleS3Client.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 6dcf1b96a..81b143d6d 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -16,7 +16,6 @@ use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CompletedMultipartUpload; use AsyncAws\S3\ValueObject\CompletedPart; -use AsyncAws\S3\ValueObject\CopyPartResult; /** * A simplified S3 client that hides some of the complexity of working with S3. @@ -61,6 +60,7 @@ public function has(string $bucket, string $key): bool * ContentType?: string, * Metadata?: array, * PartSize?: int, + * concurrency?: int, * } $options */ public function copy(string $srcBucket, string $srcKey, string $destBucket, string $destKey, array $options = []): void @@ -73,7 +73,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $contentLength = (int) $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey])->getContentLength(); } - $concurrency = (int) ($options["concurrency"] ?? 25); + $concurrency = (int) ($options['concurrency'] ?? 25); unset($options['concurrency']); if ($concurrency > 500) { $concurrency = 500; @@ -110,8 +110,8 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri )->getUploadId(); $partNumber = 1; - $startByte = 0; - $parts = []; + $startByte = 0; + $parts = []; while ($startByte < $contentLength) { $parallelChunks = $concurrency; $responses = []; @@ -129,8 +129,8 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri ); $startByte += $partSize; - $partNumber ++; - $parallelChunks --; + ++$partNumber; + --$parallelChunks; } $error = null; foreach ($responses as $idx => $response) { @@ -139,6 +139,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $parts[] = new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $idx]); } catch (\Throwable $e) { $error = $e; + break; } } @@ -151,6 +152,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri } } $this->abortMultipartUpload(AbortMultipartUploadRequest::create(['Bucket' => $destBucket, 'Key' => $destKey, 'UploadId' => $uploadId])); + throw $error; } } From 132c58ec555e8502cee83a25dbf025cb22740d4b Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 18 Oct 2023 12:40:48 +0300 Subject: [PATCH 06/16] phpstan/psalm fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 81b143d6d..2365be6d2 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -16,6 +16,7 @@ use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CompletedMultipartUpload; use AsyncAws\S3\ValueObject\CompletedPart; +use AsyncAws\S3\ValueObject\CopyPartResult; /** * A simplified S3 client that hides some of the complexity of working with S3. @@ -135,6 +136,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $error = null; foreach ($responses as $idx => $response) { try { + /** @var CopyPartResult $copyPartResult */ $copyPartResult = $response->getCopyPartResult(); $parts[] = new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $idx]); } catch (\Throwable $e) { From 1575060758220ef7df50c653e49f9907301bc378 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 18 Oct 2023 13:30:02 +0300 Subject: [PATCH 07/16] phpstan/psalm fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 2365be6d2..44d7b5460 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -60,8 +60,8 @@ public function has(string $bucket, string $key): bool * ContentLength?: int, * ContentType?: string, * Metadata?: array, - * PartSize?: int, - * concurrency?: int, + * PartSize?: positive-int, + * Concurrency?: positive-int, * } $options */ public function copy(string $srcBucket, string $srcKey, string $destBucket, string $destKey, array $options = []): void @@ -74,13 +74,10 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $contentLength = (int) $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey])->getContentLength(); } - $concurrency = (int) ($options['concurrency'] ?? 25); - unset($options['concurrency']); - if ($concurrency > 500) { - $concurrency = 500; - } + $concurrency = (int) ($options['Concurrency'] ?? 10); + unset($options['Concurrency']); if ($concurrency < 1) { - $concurrency = 25; + $concurrency = 10; } /* From fd8f7fb4fe6fffdf3e834b0d667515936290b639 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 25 Oct 2023 18:38:24 +0300 Subject: [PATCH 08/16] Fix doc blocks --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 44d7b5460..ac500e7c4 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -77,7 +77,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $concurrency = (int) ($options['Concurrency'] ?? 10); unset($options['Concurrency']); if ($concurrency < 1) { - $concurrency = 10; + $concurrency = 1; } /* @@ -100,7 +100,6 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri return; } - /** @var string $uploadId */ $uploadId = $this->createMultipartUpload( CreateMultipartUploadRequest::create( array_merge($options, ['Bucket' => $destBucket, 'Key' => $destKey]) @@ -133,7 +132,6 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $error = null; foreach ($responses as $idx => $response) { try { - /** @var CopyPartResult $copyPartResult */ $copyPartResult = $response->getCopyPartResult(); $parts[] = new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $idx]); } catch (\Throwable $e) { From 54091e11b7ef21f419b9ebfb042ef2ce056871b9 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Wed, 25 Oct 2023 18:57:25 +0300 Subject: [PATCH 09/16] Require new s3 version --- src/Integration/Aws/SimpleS3/composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Integration/Aws/SimpleS3/composer.json b/src/Integration/Aws/SimpleS3/composer.json index a56241c2c..aa6f063b5 100644 --- a/src/Integration/Aws/SimpleS3/composer.json +++ b/src/Integration/Aws/SimpleS3/composer.json @@ -13,7 +13,7 @@ "require": { "php": "^7.2.5 || ^8.0", "ext-json": "*", - "async-aws/s3": "^2.0" + "async-aws/s3": "^2.1" }, "autoload": { "psr-4": { From 57666cb9581601f9c8e56d0f0860271112d5160c Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Mon, 30 Oct 2023 18:34:47 +0200 Subject: [PATCH 10/16] Get Content type and Length from source object --- .../Aws/SimpleS3/src/SimpleS3Client.php | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index ac500e7c4..a3bbc1b3a 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -16,7 +16,6 @@ use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CompletedMultipartUpload; use AsyncAws\S3\ValueObject\CompletedPart; -use AsyncAws\S3\ValueObject\CopyPartResult; /** * A simplified S3 client that hides some of the complexity of working with S3. @@ -57,8 +56,6 @@ public function has(string $bucket, string $key): bool * @param array{ * ACL?: \AsyncAws\S3\Enum\ObjectCannedACL::*, * CacheControl?: string, - * ContentLength?: int, - * ContentType?: string, * Metadata?: array, * PartSize?: positive-int, * Concurrency?: positive-int, @@ -66,20 +63,13 @@ public function has(string $bucket, string $key): bool */ public function copy(string $srcBucket, string $srcKey, string $destBucket, string $destKey, array $options = []): void { - $megabyte = 1024 * 1024; - if (!empty($options['ContentLength'])) { - $contentLength = (int) $options['ContentLength']; - unset($options['ContentLength']); - } else { - $contentLength = (int) $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey])->getContentLength(); - } - + $sourceHead = $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey]); + $contentLength = (int) $sourceHead->getContentLength(); + $options['ContentType'] = $sourceHead->getContentType(); $concurrency = (int) ($options['Concurrency'] ?? 10); unset($options['Concurrency']); - if ($concurrency < 1) { - $concurrency = 1; - } + $megabyte = 1024 * 1024; /* * The maximum number of parts is 10.000. The partSize must be a power of 2. * We default this to 64MB per part. That means that we only support to copy From d67d689953efea77dd58b9174cf0bbec2165e4af Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Mon, 30 Oct 2023 18:44:21 +0200 Subject: [PATCH 11/16] Test fix --- .../Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php index d5bd0401d..8f4dba9f4 100644 --- a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php +++ b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php @@ -183,7 +183,7 @@ public function testCopyLargeFile() $s3 = $this->getMockBuilder(SimpleS3Client::class) ->disableOriginalConstructor() - ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload', 'uploadPartCopy']) + ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload', 'uploadPartCopy', 'headObject']) ->getMock(); $s3->expects(self::once())->method('createMultipartUpload') @@ -202,8 +202,11 @@ public function testCopyLargeFile() return true; })); + $s3->expects(self::once()) + ->method('headObject') + ->willReturn(ResultMockFactory::create(HeadObjectOutput::class, ['ContentLength' => 6144 * $megabyte])); - $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt', ['ContentLength' => 6144 * $megabyte]); + $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt'); self::assertEquals($completedParts, $uploadedParts); } From 91447f58d4a9ae1338883e433041b429c2ef3a88 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Mon, 30 Oct 2023 18:49:47 +0200 Subject: [PATCH 12/16] Test fix --- .../SimpleS3/tests/Unit/SimpleS3ClientTest.php | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php index 8f4dba9f4..56afe57fe 100644 --- a/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php +++ b/src/Integration/Aws/SimpleS3/tests/Unit/SimpleS3ClientTest.php @@ -141,23 +141,7 @@ public function testUploadSmallFileEmptyClosure() }); } - public function testCopySmallFileWithProvidedLength() - { - $megabyte = 1024 * 1024; - $s3 = $this->getMockBuilder(SimpleS3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['createMultipartUpload', 'abortMultipartUpload', 'copyObject', 'completeMultipartUpload']) - ->getMock(); - - $s3->expects(self::never())->method('createMultipartUpload'); - $s3->expects(self::never())->method('abortMultipartUpload'); - $s3->expects(self::never())->method('completeMultipartUpload'); - $s3->expects(self::once())->method('copyObject'); - - $s3->copy('bucket', 'robots.txt', 'bucket', 'copy-robots.txt', ['ContentLength' => 5 * $megabyte]); - } - - public function testCopySmallFileWithoutProvidedLength() + public function testCopySmallFile() { $megabyte = 1024 * 1024; $s3 = $this->getMockBuilder(SimpleS3Client::class) From 6bf1c47dd830d3c28e94c21803e66f1b1059d085 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Mon, 30 Oct 2023 19:03:37 +0200 Subject: [PATCH 13/16] Psalm fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index a3bbc1b3a..94a3a89b9 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -13,9 +13,11 @@ use AsyncAws\S3\Input\CreateMultipartUploadRequest; use AsyncAws\S3\Input\GetObjectRequest; use AsyncAws\S3\Input\UploadPartCopyRequest; +use AsyncAws\S3\Result\UploadPartCopyOutput; use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\CompletedMultipartUpload; use AsyncAws\S3\ValueObject\CompletedPart; +use AsyncAws\S3\ValueObject\CopyPartResult; /** * A simplified S3 client that hides some of the complexity of working with S3. @@ -95,12 +97,16 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri array_merge($options, ['Bucket' => $destBucket, 'Key' => $destKey]) ) )->getUploadId(); + if (!$uploadId) { + throw new \RuntimeException('UploadId can not be obtained'); + } $partNumber = 1; $startByte = 0; $parts = []; while ($startByte < $contentLength) { $parallelChunks = $concurrency; + /** @var UploadPartCopyOutput[] $responses */ $responses = []; while ($startByte < $contentLength && $parallelChunks > 0) { $endByte = min($startByte + $partSize, $contentLength) - 1; @@ -122,6 +128,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $error = null; foreach ($responses as $idx => $response) { try { + /** @var CopyPartResult $copyPartResult */ $copyPartResult = $response->getCopyPartResult(); $parts[] = new CompletedPart(['ETag' => $copyPartResult->getEtag(), 'PartNumber' => $idx]); } catch (\Throwable $e) { From 6a065bd5f1ea426dc604e7e509307b1275e1bce6 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Mon, 30 Oct 2023 19:09:25 +0200 Subject: [PATCH 14/16] Test fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 94a3a89b9..179c23e2d 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -4,6 +4,7 @@ namespace AsyncAws\SimpleS3; +use AsyncAws\Core\Exception\UnexpectedValue; use AsyncAws\Core\Stream\FixedSizeStream; use AsyncAws\Core\Stream\ResultStream; use AsyncAws\Core\Stream\StreamFactory; @@ -98,7 +99,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri ) )->getUploadId(); if (!$uploadId) { - throw new \RuntimeException('UploadId can not be obtained'); + throw new UnexpectedValue('UploadId can not be obtained'); } $partNumber = 1; From a350cb57d961c2dadfa443e6f35291a755c30211 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Thu, 9 Nov 2023 11:25:16 +0200 Subject: [PATCH 15/16] Test fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 179c23e2d..24b1f5b4a 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -62,17 +62,18 @@ public function has(string $bucket, string $key): bool * Metadata?: array, * PartSize?: positive-int, * Concurrency?: positive-int, + * mupThreshold?: positive-int, * } $options */ public function copy(string $srcBucket, string $srcKey, string $destBucket, string $destKey, array $options = []): void { + $megabyte = 1024 * 1024; $sourceHead = $this->headObject(['Bucket' => $srcBucket, 'Key' => $srcKey]); $contentLength = (int) $sourceHead->getContentLength(); $options['ContentType'] = $sourceHead->getContentType(); $concurrency = (int) ($options['Concurrency'] ?? 10); - unset($options['Concurrency']); - - $megabyte = 1024 * 1024; + $mupThreshold = ((int)($options['mupThreshold'] ?? 2 * 1024)) * $megabyte; + unset($options['Concurrency'], $options['mupThreshold']); /* * The maximum number of parts is 10.000. The partSize must be a power of 2. * We default this to 64MB per part. That means that we only support to copy @@ -82,8 +83,8 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $partSize = ($options['PartSize'] ?? 64) * $megabyte; unset($options['PartSize']); - // If file is less than 5GB, use normal atomic copy - if ($contentLength < 5120 * $megabyte) { + // If file is less than multipart upload threshold, use normal atomic copy + if ($contentLength < $mupThreshold) { $this->copyObject( CopyObjectRequest::create( array_merge($options, ['Bucket' => $destBucket, 'Key' => $destKey, 'CopySource' => "{$srcBucket}/{$srcKey}"]) From b1e91788d072afac1c0d3e1774ce55f04950a5c3 Mon Sep 17 00:00:00 2001 From: Mykola Muntianov Date: Thu, 9 Nov 2023 11:25:51 +0200 Subject: [PATCH 16/16] Code style fix --- src/Integration/Aws/SimpleS3/src/SimpleS3Client.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php index 24b1f5b4a..9f21abf0d 100644 --- a/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php +++ b/src/Integration/Aws/SimpleS3/src/SimpleS3Client.php @@ -72,7 +72,7 @@ public function copy(string $srcBucket, string $srcKey, string $destBucket, stri $contentLength = (int) $sourceHead->getContentLength(); $options['ContentType'] = $sourceHead->getContentType(); $concurrency = (int) ($options['Concurrency'] ?? 10); - $mupThreshold = ((int)($options['mupThreshold'] ?? 2 * 1024)) * $megabyte; + $mupThreshold = ((int) ($options['mupThreshold'] ?? 2 * 1024)) * $megabyte; unset($options['Concurrency'], $options['mupThreshold']); /* * The maximum number of parts is 10.000. The partSize must be a power of 2.