GoogleCloudPlatform / GoogleCloudPlatform/gsutil

Suddenly decoding error in cp gcs to s3 in gsutil python3 version

Open
#959 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
918
Forks
335
PR merge metrics
No merged PRs in 30d

Description

enviroment

gcloud version
Google Cloud SDK 276.0.0
bq 2.0.52
core 2020.01.10
gsutil 4.47

os Amazon Linux2

gsutil cp gs://XXXXX/XX.gz s3://XXXX/XX/

following error

Exception in thread Thread-2:iB]
Traceback (most recent call last):
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 196, in PerformDownload
decryption_tuple=self.decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 353, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1155, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1287, in _PerformDownload
use_chunks=False)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 481, in GetRange
additional_headers=additional_headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 415, in __GetChunk
retries=self.num_retries)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 356, in MakeRequest
max_retry_wait, total_wait_sec))
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/retry_util.py", line 84, in RetriesInDataTransferHandler
http_wrapper.RethrowExceptionHandler(retry_args)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 546, in NewRequest
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/httplib2/python3/httplib2/__init__.py", line 1514, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 453, in OverrideRequest
headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 690, in _conn_request
text_util.write_to_fd(self.stream, new_data)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/text_util.py", line 382, in write_to_fd
fd.write(six.ensure_text(data))
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/six/six.py", line 904, in ensure_text
return s.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte

Traceback (most recent call last):
File "/opt/google/google-cloud-sdk/platform/gsutil/gsutil", line 21, in
gsutil.RunMain()
File "/opt/google/google-cloud-sdk/platform/gsutil/gsutil.py", line 124, in RunMain
sys.exit(gslib.__main__.main())
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 433, in main
user_project=user_project)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 762, in _RunNamedCommandAndHandleExceptions
_HandleUnknownFailure(e)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 628, in _RunNamedCommandAndHandleExceptions
user_project=user_project)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command_runner.py", line 411, in RunNamedCommand
return_code = command_inst.RunCommand()
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 1206, in RunCommand
seek_ahead_iterator=seek_ahead_iterator)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 1515, in Apply
arg_checker, should_return_results, fail_on_error)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 1586, in _SequentialApply
worker_thread.PerformTask(task, self)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 2306, in PerformTask
results = task.func(cls, task.args, thread_state=self.thread_gsutil_api)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 796, in _CopyFuncWrapper
preserve_posix=cls.preserve_posix_attrs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 1006, in CopyFunc
preserve_posix=preserve_posix)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/copy_helper.py", line 3910, in PerformCopy
decryption_key=decryption_key)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/copy_helper.py", line 3428, in _CopyObjToObjDaisyChainMode
encryption_tuple=encryption_keywrapper)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 421, in UploadObjectResumable
gzip_encoded=gzip_encoded)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 1004, in UploadObjectResumable
size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 1094, in UploadObject
headers=headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 849, in _PerformSimpleUpload
rewind=True)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/storage_uri.py", line 694, in set_contents_from_file
rewind=rewind)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 1313, in set_contents_from_file
chunked_transfer=chunked_transfer, size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 765, in send_file
chunked_transfer=chunked_transfer, size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 967, in _send_file_internal
query_args=query_args
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/connection.py", line 684, in make_request
retry_handler=retry_handler
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/connection.py", line 1074, in make_request
retry_handler=retry_handler)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/connection.py", line 943, in _mexe
request.body, request.headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 843, in sender
chunk = fp.read(self.BufferSize)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 246, in read
raise self.download_exception # pylint: disable=raising-bad-type
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 196, in PerformDownload
decryption_tuple=self.decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 353, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1155, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1287, in _PerformDownload
use_chunks=False)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 481, in GetRange
additional_headers=additional_headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 415, in __GetChunk
retries=self.num_retries)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 356, in MakeRequest
max_retry_wait, total_wait_sec))
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/retry_util.py", line 84, in RetriesInDataTransferHandler
http_wrapper.RethrowExceptionHandler(retry_args)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 546, in NewRequest
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/httplib2/python3/httplib2/__init__.py", line 1514, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 453, in OverrideRequest
headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 690, in _conn_request
text_util.write_to_fd(self.stream, new_data)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/text_util.py", line 382, in write_to_fd
fd.write(six.ensure_text(data))
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/six/six.py", line 904, in ensure_text
return s.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte

why Suddenly ??

Contributor guide

Open the contributing guide

Research direction

Reproduce the gsutil cp command using the listed Google Cloud SDK 276.0.0, gsutil 4.47, Python 3.7, and Amazon Linux 2 versions. Start with gslib/gcs_json_media.py, gslib/utils/text_util.py, and gslib/daisy_chain_wrapper.py around the traceback; done means the GCS-to-S3 copy completes without the UnicodeDecodeError.

Written by the indexing model from the issue text.

Assessment

Tech stack
aws, google-cloud, python
Domain
cli, cloud
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
30/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.