aws / aws/aws-sdk-java-v2

Unable to use CompleteMultipartUploadRequest with presigned URL - Same as #4196 still an Issue

Open
#5,683 0 comments 3 reactions 1 assignee Claimed by @debora-ito View on GitHub
bug investigating p2
Dominant language
Java
Stars
2.6k
Forks
1k
Avg merge
2d 9h
Merged PRs (30d)
51

Description

### Describe the bug

This issue seems to be with presigner. We want to have presigned URI that will be used by end client to upload objects and complete them the "completeMultipartUpload" request throws exception. We have also tried by changing content type from xml to octet-stream, but still the issue does not resolve. We cannot use cURL from command line with our requirements.

### Regression Issue

- [ ] Select this option if this issue appears to be a regression.

### Expected Behavior

1. CompletedMultipartUpload give http 200 not 403 with code 200.
2. The signatures used to sign match as SDK's builder adds them.

### Current Behavior

[ Response body: ] SignatureDoesNotMatch**The request signature we calculated does not match the signature you provided. Check your key and signing method.AKIA3DW4R6EQKAVGXSPXAWS4-HMAC-SHA256
20241029T101952Z**
20241029/ap-south-1/s3/aws4_request

### Reproduction Steps

We get the same error. We have refactored the code according to our requirements.

package com.cio.storage.aws.examples;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.CompleteMultipartUploadPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.CreateMultipartUploadPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedCompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedCreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
import software.amazon.awssdk.utils.IoUtils;
/**
* Actual Example taken from
* {@link : https://github.com/aws/aws-sdk-java-v2/issues/4196}
*
*/
public class TestMPUWithPresignedURLsGH
{
final static String CREATE_MPU_UR_CT = "application/octet-stream";
final static String PRESIGNED_MPU_UR_CT = "application/octet-stream";

private final XmlMapper xmlMapper = new XmlMapper();

public static void main(final String[] args)
{
try
{

new TestMPUWithPresignedURLsGH().test();

} catch (Exception ex)
{
ex.printStackTrace();

}
}

private void test() throws IOException, XMLStreamException
{
// FILL THE INFORMATION HERE
final String accessKeyId = "";
final String secretAccessKey = "";
final Region region = ;
final String bucket = "";
final String key = "";

this.putObjectMPU(region, accessKeyId, secretAccessKey, bucket, key);
}

private void putObjectMPU(final Region region, final String accessKeyId, final String secretAccessKey, final String bucket, final String key)
throws IOException, XMLStreamException
{
// Create S3 presigner
final AwsCredentialsProvider awsCredentialsProvider = () -> AwsBasicCredentials.create(accessKeyId, secretAccessKey);

final S3Configuration s3Configuration = S3Configuration.builder().pathStyleAccessEnabled(false).build();

final S3Presigner s3Presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(region)
.serviceConfiguration(s3Configuration).build();
// -->

// Create MPU

final CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucket).key(key)
.contentType(TestMPUWithPresignedURLsGH.CREATE_MPU_UR_CT).build();

final CreateMultipartUploadPresignRequest createMultipartUploadPresignRequest = CreateMultipartUploadPresignRequest.builder()
.signatureDuration(Duration.ofHours(1)).createMultipartUploadRequest(createMultipartUploadRequest).build();

final PresignedCreateMultipartUploadRequest presignedCreateMultipartUploadRequest = s3Presigner
.presignCreateMultipartUpload(createMultipartUploadPresignRequest);

final URL presignedCreateMultipartUploadRequestURL = presignedCreateMultipartUploadRequest.url();
System.out.println("Create MPU URL: " + presignedCreateMultipartUploadRequestURL);

final HttpURLResponse presignedCreateMultipartUploadRequestResponse = this.makeRequest(SdkHttpMethod.POST,
presignedCreateMultipartUploadRequestURL, PRESIGNED_MPU_UR_CT, /* data */ null);

// final HttpURLResponse presignedCreateMultipartUploadRequestResponse = this.makeRequest(SdkHttpMethod.POST,
// presignedCreateMultipartUploadRequestURL, PRESIGNED_MPU_UR_CT, /* data */
// null);

final JsonNode presignedCreateMultipartUploadRequestResponseXml = this.fromXml(presignedCreateMultipartUploadRequestResponse.getBody());
final String multipartUploadId = presignedCreateMultipartUploadRequestResponseXml.get("UploadId").asText();

System.out.println("MPU ID: " + multipartUploadId);
// -->

// MPU Part
final int partNumber = 1;
final String partContent = "AAA" + this.createLongString(1024 * 1024 * 5);
final byte[] partContentBytes = partContent.getBytes(StandardCharsets.UTF_8);

System.out.printf("UploadPartRequest #%d: bucket=%s key=%s upload-id=%s%n", partNumber, bucket, key, multipartUploadId);

final UploadPartRequest uploadPartRequest = UploadPartRequest.builder().bucket(bucket).key(key).uploadId(multipartUploadId)
.partNumber(partNumber).build();

final UploadPartPresignRequest uploadPartPresignRequest = UploadPartPresignRequest.builder().signatureDuration(Duration.ofHours(1))
.uploadPartRequest(uploadPartRequest).build();

final PresignedUploadPartRequest presignedUploadPartRequest = s3Presigner.presignUploadPart(uploadPartPresignRequest);
final URL presignedUploadPartRequestURL = presignedUploadPartRequest.url();
System.out.printf("UploadPart #%d URL: %s%n", partNumber, presignedUploadPartRequestURL);

final HttpURLResponse presignedUploadPartRequest1Response = this.makeRequest(SdkHttpMethod.PUT, presignedUploadPartRequestURL,
/* contentType */ null, partContentBytes);
final String eTag = presignedUploadPartRequest1Response.getHeaders().get("ETag").get(0);
System.out.printf("UploadPart #%d ETag: %s%n", partNumber, eTag);

final CompletedPart completedPart = CompletedPart.builder().partNumber(partNumber).eTag(eTag).build();

final String eTagPart = completedPart.eTag();
// -->

// Commit MPU
final CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedPart).build();

System.out.printf("CompleteMultipartUploadRequest: bucket=%s key=%s multipart-id=%s%n", bucket, key, multipartUploadId);
final CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucket).key(key)
.uploadId(multipartUploadId).multipartUpload(completedMultipartUpload).build();

final CompleteMultipartUploadPresignRequest completeMultipartUploadPresignRequest = CompleteMultipartUploadPresignRequest.builder()
.signatureDuration(Duration.ofHours(1)).completeMultipartUploadRequest(completeMultipartUploadRequest).build();

final PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = s3Presigner
.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
final URL presignedCompleteMultipartUploadRequestURL = presignedCompleteMultipartUploadRequest.url();
System.out.printf("CompleteMultipartUpload URL: %s%n", presignedCompleteMultipartUploadRequestURL);

final Map parts = Collections.singletonMap(partNumber, eTagPart);
final String completeMultipartRequestPayload = this.buildCompleteMultipartRequestPayload(parts);
final byte[] completeMultipartRequestPayloadBytes = completeMultipartRequestPayload.getBytes(StandardCharsets.UTF_8);

System.out.println("Request payload: " + new String(completeMultipartRequestPayloadBytes, StandardCharsets.UTF_8));

String contentTypeComMPUResponse = "application/x-www-form-urlencoded";
// String contentTypeComMPUResponse = "application/octet-stream";

final HttpURLResponse completeMultipartUploadResponse = this.makeRequest(SdkHttpMethod.POST, presignedCompleteMultipartUploadRequestURL,
/* contentType */ contentTypeComMPUResponse, completeMultipartRequestPayloadBytes);

System.out.println("Status Code: " + completeMultipartUploadResponse.getStatusCode());
System.out.println("Response headers: " + completeMultipartUploadResponse.getHeaders());
System.out.println("\n [ Response body: ] " + completeMultipartUploadResponse.getBody());
// -->
}

private String buildCompleteMultipartRequestPayload(final Map parts) throws XMLStreamException
{
final StringWriter stringWriter = new StringWriter();
final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newFactory();
final XMLStreamWriter xmlWriter = xmlOutputFactory.createXMLStreamWriter(stringWriter);

xmlWriter.writeStartDocument();
xmlWriter.writeStartElement("CompleteMultipartUpload");
xmlWriter.writeDefaultNamespace("http://s3.amazonaws.com/doc/2006-03-01/");

for (final Map.Entry part : parts.entrySet())
{
xmlWriter.writeStartElement("Part");

xmlWriter.writeStartElement("ETag");
xmlWriter.writeCharacters(part.getValue());
xmlWriter.writeEndElement();

xmlWriter.writeStartElement("PartNumber");
xmlWriter.writeCharacters(part.getKey().toString());
xmlWriter.writeEndElement();

xmlWriter.writeEndElement();
}

xmlWriter.writeEndElement();
xmlWriter.writeEndDocument();

return stringWriter.toString().replace("", "");
}

private String createLongString(final int length)
{
final char[] randomChars = new char[length];
Arrays.fill(randomChars, 'z');

return new String(randomChars);
}

private JsonNode fromXml(final String xml) throws JsonProcessingException
{
return this.xmlMapper.readTree(xml);
}

private HttpURLResponse makeRequest(final SdkHttpMethod httpMethod, final URL url, final String contentType, final byte[] data) throws IOException
{
System.out.println("Making request to: " + url.toString());

final HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod(httpMethod.name());
httpURLConnection.setDefaultUseCaches(false);

if (contentType != null)
{
httpURLConnection.setRequestProperty("Content-Type", contentType);
}

if (data != null)
{
httpURLConnection.setDoOutput(true);

try (final InputStream requestPayloadInputStream = new ByteArrayInputStream(data);
final OutputStream httpOutputStream = httpURLConnection.getOutputStream())
{
IoUtils.copy(requestPayloadInputStream, httpOutputStream);
}
}

return new HttpURLResponse(httpURLConnection);
}

private static class HttpURLResponse
{

private final int statusCode;
private final Map> headers;
private final String body;

public HttpURLResponse(final HttpURLConnection httpURLConnection) throws IOException
{
this.statusCode = httpURLConnection.getResponseCode();
this.headers = httpURLConnection.getHeaderFields();

if (this.statusCode == HttpURLConnection.HTTP_OK)
{
try (final InputStream in = httpURLConnection.getInputStream(); final ByteArrayOutputStream out = new ByteArrayOutputStream())
{
IoUtils.copy(in, out);
final byte[] bodyBytes = out.toByteArray();
this.body = new String(bodyBytes, StandardCharsets.UTF_8);
}
} else
{
try (final InputStream in = httpURLConnection.getErrorStream(); final ByteArrayOutputStream out = new ByteArrayOutputStream())
{
IoUtils.copy(in, out);
final byte[] bodyBytes = out.toByteArray();
this.body = new String(bodyBytes, StandardCharsets.UTF_8);
}
}
}

public int getStatusCode()
{
return this.statusCode;
}

public Map> getHeaders()
{
return this.headers;
}

public String getBody()
{
return this.body;
}

}

}

### Possible Solution

_No response_

### Additional Information/Context

_No response_

### AWS Java SDK version used

2.29.1, 2.27.12 and some others

### JDK version used

1.8

### Operating System and version

Windows 10 and Ubuntu

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.