GoogleCloudPlatform / GoogleCloudPlatform/gsutil
Suddenly decoding error in cp gcs to s3 in gsutil python3 version
- Dominant language
- Python
- Stars
- 918
- Forks
- 335
- PR merge metrics
- No merged PRs in 30d
Description
enviroment
gcloud version
Google Cloud SDK 276.0.0
bq 2.0.52
core 2020.01.10
gsutil 4.47
os Amazon Linux2
gsutil cp gs://XXXXX/XX.gz s3://XXXX/XX/
following error
Exception in thread Thread-2:iB]
Traceback (most recent call last):
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 196, in PerformDownload
decryption_tuple=self.decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 353, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1155, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1287, in _PerformDownload
use_chunks=False)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 481, in GetRange
additional_headers=additional_headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 415, in __GetChunk
retries=self.num_retries)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 356, in MakeRequest
max_retry_wait, total_wait_sec))
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/retry_util.py", line 84, in RetriesInDataTransferHandler
http_wrapper.RethrowExceptionHandler(retry_args)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 546, in NewRequest
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/httplib2/python3/httplib2/__init__.py", line 1514, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 453, in OverrideRequest
headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 690, in _conn_request
text_util.write_to_fd(self.stream, new_data)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/text_util.py", line 382, in write_to_fd
fd.write(six.ensure_text(data))
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/six/six.py", line 904, in ensure_text
return s.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte
Traceback (most recent call last):
File "/opt/google/google-cloud-sdk/platform/gsutil/gsutil", line 21, in
gsutil.RunMain()
File "/opt/google/google-cloud-sdk/platform/gsutil/gsutil.py", line 124, in RunMain
sys.exit(gslib.__main__.main())
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 433, in main
user_project=user_project)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 762, in _RunNamedCommandAndHandleExceptions
_HandleUnknownFailure(e)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/__main__.py", line 628, in _RunNamedCommandAndHandleExceptions
user_project=user_project)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command_runner.py", line 411, in RunNamedCommand
return_code = command_inst.RunCommand()
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 1206, in RunCommand
seek_ahead_iterator=seek_ahead_iterator)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 1515, in Apply
arg_checker, should_return_results, fail_on_error)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 1586, in _SequentialApply
worker_thread.PerformTask(task, self)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/command.py", line 2306, in PerformTask
results = task.func(cls, task.args, thread_state=self.thread_gsutil_api)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 796, in _CopyFuncWrapper
preserve_posix=cls.preserve_posix_attrs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/commands/cp.py", line 1006, in CopyFunc
preserve_posix=preserve_posix)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/copy_helper.py", line 3910, in PerformCopy
decryption_key=decryption_key)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/copy_helper.py", line 3428, in _CopyObjToObjDaisyChainMode
encryption_tuple=encryption_keywrapper)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 421, in UploadObjectResumable
gzip_encoded=gzip_encoded)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 1004, in UploadObjectResumable
size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 1094, in UploadObject
headers=headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/boto_translation.py", line 849, in _PerformSimpleUpload
rewind=True)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/storage_uri.py", line 694, in set_contents_from_file
rewind=rewind)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 1313, in set_contents_from_file
chunked_transfer=chunked_transfer, size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 765, in send_file
chunked_transfer=chunked_transfer, size=size)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 967, in _send_file_internal
query_args=query_args
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/connection.py", line 684, in make_request
retry_handler=retry_handler
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/connection.py", line 1074, in make_request
retry_handler=retry_handler)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/connection.py", line 943, in _mexe
request.body, request.headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/vendored/boto/boto/s3/key.py", line 843, in sender
chunk = fp.read(self.BufferSize)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 246, in read
raise self.download_exception # pylint: disable=raising-bad-type
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/usr/bin/.pyenv/versions/3.7.0/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/daisy_chain_wrapper.py", line 196, in PerformDownload
decryption_tuple=self.decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/cloud_api_delegator.py", line 353, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1155, in GetObjectMedia
decryption_tuple=decryption_tuple)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_api.py", line 1287, in _PerformDownload
use_chunks=False)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 481, in GetRange
additional_headers=additional_headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/transfer.py", line 415, in __GetChunk
retries=self.num_retries)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 356, in MakeRequest
max_retry_wait, total_wait_sec))
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/retry_util.py", line 84, in RetriesInDataTransferHandler
http_wrapper.RethrowExceptionHandler(retry_args)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 546, in NewRequest
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/oauth2client/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/httplib2/python3/httplib2/__init__.py", line 1514, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 453, in OverrideRequest
headers)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/gcs_json_media.py", line 690, in _conn_request
text_util.write_to_fd(self.stream, new_data)
File "/opt/google/google-cloud-sdk/platform/gsutil/gslib/utils/text_util.py", line 382, in write_to_fd
fd.write(six.ensure_text(data))
File "/opt/google/google-cloud-sdk/platform/gsutil/third_party/six/six.py", line 904, in ensure_text
return s.decode(encoding, errors)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte
why Suddenly ??
Contributor guide
Research direction
Reproduce the gsutil cp command using the listed Google Cloud SDK 276.0.0, gsutil 4.47, Python 3.7, and Amazon Linux 2 versions. Start with gslib/gcs_json_media.py, gslib/utils/text_util.py, and gslib/daisy_chain_wrapper.py around the traceback; done means the GCS-to-S3 copy completes without the UnicodeDecodeError.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- aws, google-cloud, python
- Domain
- cli, cloud
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 30/100