aws / aws/aws-sdk-cpp

S3 crt client timeouts and retry strategy are not working.

Offen
#2,971 15 Kommentare 0 Reaktionen 0 zugewiesene Personen Auf GitHub ansehen
bug p2
Vorherrschende Sprache
C++
Sterne
2.2k
Forks
1.2k
Ø Merge
4 T. 11 Std.
Gemergte PRs (30 T.)
13

Beschreibung

### Describe the bug

Below mentioned s3 crt client configs we set.
```
Aws::S3Crt::S3CrtClient *s3_crt_client;
Aws::S3Crt::Model::GetObjectRequest object_request;
Aws::SDKOptions options;
Aws::InitAPI(options);
Aws::S3Crt::ClientConfiguration config;
config.throughputTargetGbps = 1;
config.partSize = 10*1024*1024;
config.httpRequestTimeoutMs = 100;
config.connectTimeoutMs = 100;
config.requestTimeoutMs = 100;
s3_crt_client = Aws::New("test", config);
```
We are reading 25MB at a time and we are measuring how much time it is taking for getObject.
average of 500ms taken for each getObject
Even though we set all the timeouts s3 crt client is taking more time?
We even tried with no retry strategy, but still there is no use and the behaviour is same.
```
config.httpRequestTimeoutMs = 100;
config.connectTimeoutMs = 100;
config.requestTimeoutMs = 100;
std::shared_ptr retry;
retry.reset(Aws::New("test", 0, 25));
config.retryStrategy = retry;
```
GetObject() is not exiting even timeout is reached. One time we saw GetObject() for size 25MB took 10seconds.
Basically behavior is same even if we didn't set timeouts or assign custom retry strategy.
we are running this in a c5a4x large instance which is running in ap-south-1 region and s3 bucket is also in the same region and account.
Please tell whats wrong with this and suggest how to reduce time taken for GetObject().

### Expected Behavior

With zero retries GetObject should exit once timeout is reached.

### Current Behavior

With zero retries GetObject is not exiting even timeout is reached.

### Reproduction Steps

### zero retries

```
#include
#include

#include
#include
#include
#include
#include
#include

void usage(const char *bin)
{
printf("USAGE: %s \n", bin);
}

uint64_t get_curr_time_ms(void)
{
struct timespec ts;
uint64_t ts_64;
clock_gettime(CLOCK_MONOTONIC, &ts);
ts_64 = (uint64_t)((ts.tv_sec * 1000) + (ts.tv_nsec/1000000));
return ts_64;
}

int main (int argc, char* argv[])
{
char *s3_uri;
uint64_t read_size = UINT64_MAX;
uint64_t part_size;
char *buf = NULL;
uint64_t read_left;
uint64_t to_read;
uint64_t ret;
uint64_t iter = 0;
char *out_file = NULL;
int out_fd = -1;

if(argc < 5 || argc > 6)
{
usage(argv[0]);
return 0;
}

s3_uri = argv[1];
part_size = strtoull(argv[2], NULL, 10);
if(argc == 6) out_file = argv[3];
Aws::S3Crt::S3CrtClient *s3_crt_client;
Aws::S3Crt::Model::GetObjectRequest object_request;
Aws::SDKOptions options;
Aws::InitAPI(options);
Aws::S3Crt::ClientConfiguration config;
config.httpRequestTimeoutMs = 100;
config.connectTimeoutMs = 100;
config.requestTimeoutMs = 100;
std::shared_ptr retry;
retry.reset(Aws::New("test", 0, 25));
config.retryStrategy = retry;
config.throughputTargetGbps = 1;
config.partSize = 10*1024*1024;
s3_crt_client = Aws::New("test", config);

char bucket[128] = {0};
char key[128] = {0};
if(sscanf(s3_uri, "s3://%[^/]/%s", bucket, key) == 2)
{
Aws::S3Crt::Model::HeadObjectRequest head_object_request;
Aws::S3Crt::Model::HeadObjectOutcome outcome;
head_object_request.SetBucket(bucket);
head_object_request.SetKey(key);
outcome = s3_crt_client->HeadObject(head_object_request);
if(outcome.IsSuccess())
printf("Object found, size:%lu, etag %s\n", outcome.GetResult().GetContentLength(), outcome.GetResult().GetETag().c_str());
object_request.SetBucket(bucket);
object_request.SetKey(key);
}

if(out_file)
{
out_fd = open(out_file, O_CREAT | O_WRONLY);
if(out_fd <= 2)
{
printf("Error in opening %s in write mode", out_file);
return 0;
}
}
int64_t pos = 0;

buf = new char[part_size];
read_left = read_size;
while(read_left != 0)
{
ret = 0;
int64_t retry_count = -1;
to_read = part_size <= read_left ? part_size : read_left;
iter ++;
uint64_t start,end;
Aws::S3Crt::Model::GetObjectOutcome outcome;
object_request.SetRange(std::string("bytes=") + std::to_string(pos) + "-" + std::to_string(pos+to_read-1));
object_request.SetResponseStreamFactory(
[buf, to_read]()
{
std::unique_ptr
stream(Aws::New("test"));
stream->rdbuf()->pubsetbuf(static_cast(buf),
to_read);

return stream.release();
});
start = get_curr_time_ms();
outcome = s3_crt_client->GetObject(object_request);
end = get_curr_time_ms();
printf("Time taken for GetObject %lums\n",end-start);
if(outcome.IsSuccess())
{
ret = outcome.GetResult().GetContentLength();

if(ret <= to_read)
{
pos += ret;
//log_info("Testing data %02x %02x", ((uint8_t*)buf)[0], ((uint8_t*)buf)[1]);
}
else
{
printf("This should never happen, pos_:%lu, requested:%lu, read:%lu\n",
pos, to_read, ret);
break;
}
}
else
{
printf("Failed to read, pos_:%lu, requested:%lu, error:%s, error_type:%d, error_code:%d\n",
pos, to_read, outcome.GetError().GetMessage().c_str(), outcome.GetError().GetErrorType(), outcome.GetError().GetResponseCode()) ;
break;
}
if(ret == 0)
{
printf("Nothing read, read_left:%lubytes, iter:%lu\n", read_left, iter);
break;
}
else if(ret < to_read)
{
printf("Read %ldbytes instead of %ldbytes, iter:%lu\n", ret, to_read, iter);
}
else
{
printf("Read %ld bytes, iter:%lu\n", ret, iter);
}
read_left -= ret;

if(ret > 0 && out_fd > 2)
{
if(write(out_fd, buf, ret) != ret)
{
printf("Error in write!!! (%lu bytes)\n", ret);
return 0;
}
fdatasync(out_fd);
}
}

if(s3_crt_client)
{
Aws::Delete(s3_crt_client);
s3_crt_client = NULL;
}
Aws::ShutdownAPI(options);

if(buf) delete[] buf;
if(out_fd > 2) close(out_fd);
return 0;
}
```

### default strategy

```
#include
#include

#include
#include
#include
#include
#include
#include

void usage(const char *bin)
{
printf("USAGE: %s till end)> \n", bin);
}

uint64_t get_curr_time_ms(void)
{
struct timespec ts;
uint64_t ts_64;
clock_gettime(CLOCK_MONOTONIC, &ts);
ts_64 = (uint64_t)((ts.tv_sec * 1000) + (ts.tv_nsec/1000000));
return ts_64;
}

int main (int argc, char* argv[])
{
char *s3_uri;
uint64_t offset;
uint64_t read_size;
uint64_t part_size;
char *buf = NULL;
uint64_t read_left;
uint64_t to_read;
uint64_t ret;
uint64_t iter = 0;
char *out_file = NULL;
int out_fd = -1;

if(argc < 5 || argc > 6)
{
usage(argv[0]);
return 0;
}

s3_uri = argv[1];
offset = strtoull(argv[2], NULL, 10);
read_size = strtoull(argv[3], NULL, 10);
if(read_size == 0) read_size = UINT64_MAX;
part_size = strtoull(argv[4], NULL, 10);
if(argc == 6) out_file = argv[5];
Aws::S3Crt::S3CrtClient *s3_crt_client;
Aws::S3Crt::Model::GetObjectRequest object_request;
Aws::SDKOptions options;
Aws::InitAPI(options);
Aws::S3Crt::ClientConfiguration config;
config.httpRequestTimeoutMs = 100;
config.connectTimeoutMs = 100;
config.requestTimeoutMs = 100;
config.throughputTargetGbps = 1;
config.partSize = 10*1024*1024;
s3_crt_client = Aws::New("test", config);

char bucket[128] = {0};
char key[128] = {0};
if(sscanf(s3_uri, "s3://%[^/]/%s", bucket, key) == 2)
{
Aws::S3Crt::Model::HeadObjectRequest head_object_request;
Aws::S3Crt::Model::HeadObjectOutcome outcome;
head_object_request.SetBucket(bucket);
head_object_request.SetKey(key);
outcome = s3_crt_client->HeadObject(head_object_request);
if(outcome.IsSuccess())
printf("Object found, size:%lu, etag %s\n", outcome.GetResult().GetContentLength(), outcome.GetResult().GetETag().c_str());
object_request.SetBucket(bucket);
object_request.SetKey(key);
}

if(out_file)
{
out_fd = open(out_file, O_CREAT | O_WRONLY);
if(out_fd <= 2)
{
printf("Error in opening %s in write mode", out_file);
return 0;
}
}
int64_t pos = 0;

buf = new char[part_size];
read_left = read_size;
while(read_left != 0)
{
ret = 0;
int64_t retry_count = -1;
to_read = part_size <= read_left ? part_size : read_left;
iter ++;
uint64_t start,end;
Aws::S3Crt::Model::GetObjectOutcome outcome;
object_request.SetRange(std::string("bytes=") + std::to_string(pos) + "-" + std::to_string(pos+to_read-1));
object_request.SetResponseStreamFactory(
[buf, to_read]()
{
std::unique_ptr
stream(Aws::New("test"));
stream->rdbuf()->pubsetbuf(static_cast(buf),
to_read);

return stream.release();
});
start = get_curr_time_ms();
outcome = s3_crt_client->GetObject(object_request);
end = get_curr_time_ms();
printf("Time taken for GetObject %lums\n",end-start);
if(outcome.IsSuccess())
{
ret = outcome.GetResult().GetContentLength();

if(ret <= to_read)
{
pos += ret;
//log_info("Testing data %02x %02x", ((uint8_t*)buf)[0], ((uint8_t*)buf)[1]);
}
else
{
printf("This should never happen, pos_:%lu, requested:%lu, read:%lu\n",
pos, to_read, ret);
break;
}
}
else
{
printf("Failed to read, pos_:%lu, requested:%lu, error:%s, error_type:%d, error_code:%d\n",
pos, to_read, outcome.GetError().GetMessage().c_str(), outcome.GetError().GetErrorType(), outcome.GetError().GetResponseCode()) ;
break;
}
if(ret == 0)
{
printf("Nothing read, read_left:%lubytes, iter:%lu\n", read_left, iter);
break;
}
else if(ret < to_read)
{
printf("Read %ldbytes instead of %ldbytes, iter:%lu\n", ret, to_read, iter);
}
else
{
printf("Read %ld bytes, iter:%lu\n", ret, iter);
}
read_left -= ret;

if(ret > 0 && out_fd > 2)
{
if(write(out_fd, buf, ret) != ret)
{
printf("Error in write!!! (%lu bytes)\n", ret);
return 0;
}
fdatasync(out_fd);
}
}

if(s3_crt_client)
{
Aws::Delete(s3_crt_client);
s3_crt_client = NULL;
}
Aws::ShutdownAPI(options);

if(buf) delete[] buf;
if(out_fd > 2) close(out_fd);
return 0;
}
```

### Possible Solution

_No response_

### Additional Information/Context

_No response_

### AWS CPP SDK version used

1.11.269

### Compiler and Version used

gcc (GCC) 4.8.5

### Operating System and version

CentOS Linux and version 7

Beitragsleitfaden

Beitragsleitfaden öffnen

Rechercherichtung

Beginne damit, die bereitgestellten C++-Reproduktionen rund um Aws::S3Crt::S3CrtClient::GetObject, HeadObject, ClientConfiguration und DefaultRetryStrategy zu erstellen und auszuführen. Verfolge das Timeout- und Wiederholungsverhalten in den Einstiegspunkten des S3 CRT-Clients; als abgeschlossen gilt die Aufgabe, wenn die gemeldeten Timeout-Einstellungen und die Konfiguration mit null Wiederholungsversuchen die erwartete Beendigung von GetObject bewirken.

Vom Indexierungsmodell aus dem Issue-Text verfasst.

Bewertung

Tech-Stack
aws, cpp
Bereich
cloud
Issue-Typ
Bug
Schwierigkeit
4/5
Geschätzter Aufwand
3-5 Tage
Aktivitätsstatus
Veraltet
Klarheit
Muss geklärt werden
Anfängerfreundlichkeit
35/100

Neue Issues direkt in Ihr Postfach

Eine kurze Übersicht über anfängerfreundliche GitHub-Issues.