S3 integration tests still raise ThrottlingException even after catching the exception and adding retries.
s3 integration tests have been temporary disabled (see #40) and further investigation is needed.
2023-01-26 11:17:36.781878 \| controller \| TASK [s3 : Delete S3 bucket] ***************************************************
--
12158 | 2023-01-26 11:17:36.781884 \| controller \| task path: /home/zuul/.ansible/collections/ansible_collections/amazon/cloud/tests/integration/targets/s3/tasks/main.yml:242
12159 | 2023-01-26 11:17:45.376705 \| controller \| Using module file /home/zuul/.ansible/collections/ansible_collections/amazon/cloud/plugins/modules/s3_bucket.py
12160 | 2023-01-26 11:17:45.376748 \| controller \| Pipelining is enabled.
12161 | 2023-01-26 11:17:45.376756 \| controller \| <testhost> ESTABLISH LOCAL CONNECTION FOR USER: zuul
12162 | 2023-01-26 11:17:45.376763 \| controller \| <testhost> EXEC /bin/sh -c 'ANSIBLE_DEBUG_BOTOCORE_LOGS=True /home/zuul/venv/bin/python && sleep 0'
12163 | 2023-01-26 11:17:45.376770 \| controller \| The full traceback is:
12164 | 2023-01-26 11:17:45.376776 \| controller \| Traceback (most recent call last):
12165 | 2023-01-26 11:17:45.376784 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/cloud/plugins/module_utils/core.py", line 195, in list_resource_requests
12166 | 2023-01-26 11:17:45.376790 \| controller \| response = self.client.list_resource_requests(
12167 | 2023-01-26 11:17:45.376796 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/retries.py", line 96, in deciding_wrapper
12168 | 2023-01-26 11:17:45.376818 \| controller \| return retrying_wrapper(*args, **kwargs)
12169 | 2023-01-26 11:17:45.376826 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 115, in _retry_wrapper
12170 | 2023-01-26 11:17:45.376832 \| controller \| return _retry_func(
12171 | 2023-01-26 11:17:45.376838 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 65, in _retry_func
12172 | 2023-01-26 11:17:45.376844 \| controller \| return func()
12173 | 2023-01-26 11:17:45.376848 \| controller \| File "/home/zuul/venv/lib/python3.9/site-packages/botocore/client.py", line 391, in _api_call
12174 | 2023-01-26 11:17:45.376853 \| controller \| return self._make_api_call(operation_name, kwargs)
12175 | 2023-01-26 11:17:45.376858 \| controller \| File "/home/zuul/venv/lib/python3.9/site-packages/botocore/client.py", line 719, in _make_api_call
12176 | 2023-01-26 11:17:45.376863 \| controller \| raise error_class(parsed_response, operation_name)
12177 | 2023-01-26 11:17:45.376868 \| controller \| botocore.errorfactory.ThrottlingException: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded
12178 | 2023-01-26 11:17:45.376873 \| controller \| fatal: [testhost]: FAILED! => {
12179 | 2023-01-26 11:17:45.376879 \| controller \| "boto3_version": "1.20.0",
12180 | 2023-01-26 11:17:45.376885 \| controller \| "botocore_version": "1.23.0",
12181 | 2023-01-26 11:17:45.376890 \| controller \| "changed": false,
12182 | 2023-01-26 11:17:45.376895 \| controller \| "error": {
12183 | 2023-01-26 11:17:45.376900 \| controller \| "code": "ThrottlingException",
12184 | 2023-01-26 11:17:45.376905 \| controller \| "message": "Rate exceeded"
12185 | 2023-01-26 11:17:45.376910 \| controller \| },
12186 | 2023-01-26 11:17:45.376915 \| controller \| "invocation": {
12187 | 2023-01-26 11:17:45.376920 \| controller \| "module_args": {
12188 | 2023-01-26 11:17:45.376925 \| controller \| "accelerate_configuration": null,
12189 | 2023-01-26 11:17:45.376930 \| controller \| "access_control": null,
12190 | 2023-01-26 11:17:45.376936 \| controller \| "access_key": "ASIA6CCDWXDOKBFP3TXZ",
12191 | 2023-01-26 11:17:45.376940 \| controller \| "analytics_configurations": null,
12192 | 2023-01-26 11:17:45.376944 \| controller \| "aws_access_key": "ASIA6CCDWXDOKBFP3TXZ",
12193 | 2023-01-26 11:17:45.376949 \| controller \| "aws_ca_bundle": null,
12194 | 2023-01-26 11:17:45.376954 \| controller \| "aws_config": null,
12195 | 2023-01-26 11:17:45.376958 \| controller \| "aws_secret_key": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12196 | 2023-01-26 11:17:45.376963 \| controller \| "bucket_encryption": null,
12197 | 2023-01-26 11:17:45.376967 \| controller \| "bucket_name": "d8ea0ccd-31dd-548c-aa26-514258b6c748",
12198 | 2023-01-26 11:17:45.376971 \| controller \| "cors_configuration": null,
12199 | 2023-01-26 11:17:45.376976 \| controller \| "debug_botocore_endpoint_logs": true,
12200 | 2023-01-26 11:17:45.376980 \| controller \| "endpoint_url": null,
12201 | 2023-01-26 11:17:45.376984 \| controller \| "force": false,
12202 | 2023-01-26 11:17:45.377050 \| controller \| "intelligent_tiering_configurations": null,
12203 | 2023-01-26 11:17:45.377056 \| controller \| "inventory_configurations": null,
12204 | 2023-01-26 11:17:45.377062 \| controller \| "lifecycle_configuration": null,
12205 | 2023-01-26 11:17:45.377067 \| controller \| "logging_configuration": null,
12206 | 2023-01-26 11:17:45.377071 \| controller \| "metrics_configurations": null,
12207 | 2023-01-26 11:17:45.377076 \| controller \| "notification_configuration": null,
12208 | 2023-01-26 11:17:45.377080 \| controller \| "object_lock_configuration": null,
12209 | 2023-01-26 11:17:45.377085 \| controller \| "object_lock_enabled": null,
12210 | 2023-01-26 11:17:45.377089 \| controller \| "ownership_controls": null,
12211 | 2023-01-26 11:17:45.377094 \| controller \| "profile": null,
12212 | 2023-01-26 11:17:45.377098 \| controller \| "public_access_block_configuration": null,
12213 | 2023-01-26 11:17:45.377103 \| controller \| "purge_tags": true,
12214 | 2023-01-26 11:17:45.377107 \| controller \| "region": "us-east-1",
12215 | 2023-01-26 11:17:45.377112 \| controller \| "replication_configuration": null,
12216 | 2023-01-26 11:17:45.377117 \| controller \| "secret_key": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12217 | 2023-01-26 11:17:45.377121 \| controller \| "security_token": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12218 | 2023-01-26 11:17:45.377131 \| controller \| "session_token": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12219 | 2023-01-26 11:17:45.377137 \| controller \| "state": "absent",
12220 | 2023-01-26 11:17:45.377141 \| controller \| "tags": null,
12221 | 2023-01-26 11:17:45.377146 \| controller \| "validate_certs": true,
12222 | 2023-01-26 11:17:45.377151 \| controller \| "versioning_configuration": null,
12223 | 2023-01-26 11:17:45.377156 \| controller \| "wait": true,
12224 | 2023-01-26 11:17:45.377160 \| controller \| "wait_timeout": 320,
12225 | 2023-01-26 11:17:45.377165 \| controller \| "website_configuration": null
12226 | 2023-01-26 11:17:45.377170 \| controller \| }
12227 | 2023-01-26 11:17:45.377174 \| controller \| },
12228 | 2023-01-26 11:17:45.377179 \| controller \| "msg": "Failed to list resource requests: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded",
12229 | 2023-01-26 11:17:45.377184 \| controller \| "resource_actions": [
12230 | 2023-01-26 11:17:45.377189 \| controller \| "cloudcontrolapi:GetResource",
12231 | 2023-01-26 11:17:45.377193 \| controller \| "cloudcontrolapi:ListResourceRequests"
12232 | 2023-01-26 11:17:45.377198 \| controller \| ],
12233 | 2023-01-26 11:17:45.377202 \| controller \| "response_metadata": {
12234 | 2023-01-26 11:17:45.377207 \| controller \| "http_headers": {
12235 | 2023-01-26 11:17:45.377211 \| controller \| "content-length": "88",
12236 | 2023-01-26 11:17:45.377216 \| controller \| "content-type": "application/x-amz-json-1.0",
12237 | 2023-01-26 11:17:45.377221 \| controller \| "date": "Thu, 26 Jan 2023 11:17:45 GMT",
12238 | 2023-01-26 11:17:45.377225 \| controller \| "x-amzn-requestid": "512f6531-4f01-486e-86e9-164d5b59601c"
12239 | 2023-01-26 11:17:45.377230 \| controller \| },
12240 | 2023-01-26 11:17:45.377234 \| controller \| "http_status_code": 400,
12241 | 2023-01-26 11:17:45.377239 \| controller \| "max_attempts_reached": true,
12242 | 2023-01-26 11:17:45.377245 \| controller \| "request_id": "512f6531-4f01-486e-86e9-164d5b59601c",
12243 | 2023-01-26 11:17:45.377255 \| controller \| "retry_attempts": 4
12244 | 2023-01-26 11:17:45.377260 \| controller \| }
12245 | 2023-01-26 11:17:45.377265 \| controller \| }
12246 | 2023-01-26 11:17:45.377270 \| controller \|
2023-01-26 11:17:36.781878 \| controller \| TASK [s3 : Delete S3 bucket] ***************************************************
--
12158 | 2023-01-26 11:17:36.781884 \| controller \| task path: /home/zuul/.ansible/collections/ansible_collections/amazon/cloud/tests/integration/targets/s3/tasks/main.yml:242
12159 | 2023-01-26 11:17:45.376705 \| controller \| Using module file /home/zuul/.ansible/collections/ansible_collections/amazon/cloud/plugins/modules/s3_bucket.py
12160 | 2023-01-26 11:17:45.376748 \| controller \| Pipelining is enabled.
12161 | 2023-01-26 11:17:45.376756 \| controller \| <testhost> ESTABLISH LOCAL CONNECTION FOR USER: zuul
12162 | 2023-01-26 11:17:45.376763 \| controller \| <testhost> EXEC /bin/sh -c 'ANSIBLE_DEBUG_BOTOCORE_LOGS=True /home/zuul/venv/bin/python && sleep 0'
12163 | 2023-01-26 11:17:45.376770 \| controller \| The full traceback is:
12164 | 2023-01-26 11:17:45.376776 \| controller \| Traceback (most recent call last):
12165 | 2023-01-26 11:17:45.376784 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/cloud/plugins/module_utils/core.py", line 195, in list_resource_requests
12166 | 2023-01-26 11:17:45.376790 \| controller \| response = self.client.list_resource_requests(
12167 | 2023-01-26 11:17:45.376796 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/retries.py", line 96, in deciding_wrapper
12168 | 2023-01-26 11:17:45.376818 \| controller \| return retrying_wrapper(*args, **kwargs)
12169 | 2023-01-26 11:17:45.376826 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 115, in _retry_wrapper
12170 | 2023-01-26 11:17:45.376832 \| controller \| return _retry_func(
12171 | 2023-01-26 11:17:45.376838 \| controller \| File "/tmp/ansible_amazon.cloud.s3_bucket_payload_v4ib8xxx/ansible_amazon.cloud.s3_bucket_payload.zip/ansible_collections/amazon/aws/plugins/module_utils/cloud.py", line 65, in _retry_func
12172 | 2023-01-26 11:17:45.376844 \| controller \| return func()
12173 | 2023-01-26 11:17:45.376848 \| controller \| File "/home/zuul/venv/lib/python3.9/site-packages/botocore/client.py", line 391, in _api_call
12174 | 2023-01-26 11:17:45.376853 \| controller \| return self._make_api_call(operation_name, kwargs)
12175 | 2023-01-26 11:17:45.376858 \| controller \| File "/home/zuul/venv/lib/python3.9/site-packages/botocore/client.py", line 719, in _make_api_call
12176 | 2023-01-26 11:17:45.376863 \| controller \| raise error_class(parsed_response, operation_name)
12177 | 2023-01-26 11:17:45.376868 \| controller \| botocore.errorfactory.ThrottlingException: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded
12178 | 2023-01-26 11:17:45.376873 \| controller \| fatal: [testhost]: FAILED! => {
12179 | 2023-01-26 11:17:45.376879 \| controller \| "boto3_version": "1.20.0",
12180 | 2023-01-26 11:17:45.376885 \| controller \| "botocore_version": "1.23.0",
12181 | 2023-01-26 11:17:45.376890 \| controller \| "changed": false,
12182 | 2023-01-26 11:17:45.376895 \| controller \| "error": {
12183 | 2023-01-26 11:17:45.376900 \| controller \| "code": "ThrottlingException",
12184 | 2023-01-26 11:17:45.376905 \| controller \| "message": "Rate exceeded"
12185 | 2023-01-26 11:17:45.376910 \| controller \| },
12186 | 2023-01-26 11:17:45.376915 \| controller \| "invocation": {
12187 | 2023-01-26 11:17:45.376920 \| controller \| "module_args": {
12188 | 2023-01-26 11:17:45.376925 \| controller \| "accelerate_configuration": null,
12189 | 2023-01-26 11:17:45.376930 \| controller \| "access_control": null,
12190 | 2023-01-26 11:17:45.376936 \| controller \| "access_key": "ASIA6CCDWXDOKBFP3TXZ",
12191 | 2023-01-26 11:17:45.376940 \| controller \| "analytics_configurations": null,
12192 | 2023-01-26 11:17:45.376944 \| controller \| "aws_access_key": "ASIA6CCDWXDOKBFP3TXZ",
12193 | 2023-01-26 11:17:45.376949 \| controller \| "aws_ca_bundle": null,
12194 | 2023-01-26 11:17:45.376954 \| controller \| "aws_config": null,
12195 | 2023-01-26 11:17:45.376958 \| controller \| "aws_secret_key": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12196 | 2023-01-26 11:17:45.376963 \| controller \| "bucket_encryption": null,
12197 | 2023-01-26 11:17:45.376967 \| controller \| "bucket_name": "d8ea0ccd-31dd-548c-aa26-514258b6c748",
12198 | 2023-01-26 11:17:45.376971 \| controller \| "cors_configuration": null,
12199 | 2023-01-26 11:17:45.376976 \| controller \| "debug_botocore_endpoint_logs": true,
12200 | 2023-01-26 11:17:45.376980 \| controller \| "endpoint_url": null,
12201 | 2023-01-26 11:17:45.376984 \| controller \| "force": false,
12202 | 2023-01-26 11:17:45.377050 \| controller \| "intelligent_tiering_configurations": null,
12203 | 2023-01-26 11:17:45.377056 \| controller \| "inventory_configurations": null,
12204 | 2023-01-26 11:17:45.377062 \| controller \| "lifecycle_configuration": null,
12205 | 2023-01-26 11:17:45.377067 \| controller \| "logging_configuration": null,
12206 | 2023-01-26 11:17:45.377071 \| controller \| "metrics_configurations": null,
12207 | 2023-01-26 11:17:45.377076 \| controller \| "notification_configuration": null,
12208 | 2023-01-26 11:17:45.377080 \| controller \| "object_lock_configuration": null,
12209 | 2023-01-26 11:17:45.377085 \| controller \| "object_lock_enabled": null,
12210 | 2023-01-26 11:17:45.377089 \| controller \| "ownership_controls": null,
12211 | 2023-01-26 11:17:45.377094 \| controller \| "profile": null,
12212 | 2023-01-26 11:17:45.377098 \| controller \| "public_access_block_configuration": null,
12213 | 2023-01-26 11:17:45.377103 \| controller \| "purge_tags": true,
12214 | 2023-01-26 11:17:45.377107 \| controller \| "region": "us-east-1",
12215 | 2023-01-26 11:17:45.377112 \| controller \| "replication_configuration": null,
12216 | 2023-01-26 11:17:45.377117 \| controller \| "secret_key": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12217 | 2023-01-26 11:17:45.377121 \| controller \| "security_token": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12218 | 2023-01-26 11:17:45.377131 \| controller \| "session_token": "VALUE_SPECIFIED_IN_NO_LOG_PARAMETER",
12219 | 2023-01-26 11:17:45.377137 \| controller \| "state": "absent",
12220 | 2023-01-26 11:17:45.377141 \| controller \| "tags": null,
12221 | 2023-01-26 11:17:45.377146 \| controller \| "validate_certs": true,
12222 | 2023-01-26 11:17:45.377151 \| controller \| "versioning_configuration": null,
12223 | 2023-01-26 11:17:45.377156 \| controller \| "wait": true,
12224 | 2023-01-26 11:17:45.377160 \| controller \| "wait_timeout": 320,
12225 | 2023-01-26 11:17:45.377165 \| controller \| "website_configuration": null
12226 | 2023-01-26 11:17:45.377170 \| controller \| }
12227 | 2023-01-26 11:17:45.377174 \| controller \| },
12228 | 2023-01-26 11:17:45.377179 \| controller \| "msg": "Failed to list resource requests: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded",
12229 | 2023-01-26 11:17:45.377184 \| controller \| "resource_actions": [
12230 | 2023-01-26 11:17:45.377189 \| controller \| "cloudcontrolapi:GetResource",
12231 | 2023-01-26 11:17:45.377193 \| controller \| "cloudcontrolapi:ListResourceRequests"
12232 | 2023-01-26 11:17:45.377198 \| controller \| ],
12233 | 2023-01-26 11:17:45.377202 \| controller \| "response_metadata": {
12234 | 2023-01-26 11:17:45.377207 \| controller \| "http_headers": {
12235 | 2023-01-26 11:17:45.377211 \| controller \| "content-length": "88",
12236 | 2023-01-26 11:17:45.377216 \| controller \| "content-type": "application/x-amz-json-1.0",
12237 | 2023-01-26 11:17:45.377221 \| controller \| "date": "Thu, 26 Jan 2023 11:17:45 GMT",
12238 | 2023-01-26 11:17:45.377225 \| controller \| "x-amzn-requestid": "512f6531-4f01-486e-86e9-164d5b59601c"
12239 | 2023-01-26 11:17:45.377230 \| controller \| },
12240 | 2023-01-26 11:17:45.377234 \| controller \| "http_status_code": 400,
12241 | 2023-01-26 11:17:45.377239 \| controller \| "max_attempts_reached": true,
12242 | 2023-01-26 11:17:45.377245 \| controller \| "request_id": "512f6531-4f01-486e-86e9-164d5b59601c",
12243 | 2023-01-26 11:17:45.377255 \| controller \| "retry_attempts": 4
12244 | 2023-01-26 11:17:45.377260 \| controller \| }
12245 | 2023-01-26 11:17:45.377265 \| controller \| }
12246 | 2023-01-26 11:17:45.377270 \| controller \|
Tests pass without any error.