tests.system.test_session_api: test_transaction_execute_sql_dml_returning failed

[flaky-bot]

Note: #930 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 998b87a
buildURL: Build Status, Sponge
status: failed

Test output

args = (session: "projects/precise-truck-742/instances/google-cloud-1683781402287/databases/test_sessions_1683781523987/sessi..., \'[email protected]\') RETURNING contact_id, first_name, last_name, email"
params {
}
seqno: 2
request_options {
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1683781402287/databa...ZSYAnp0lEH8vxFg'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.55.0rc1 gax/2.11.0 gccl/3.33.0')], 'timeout': 3600.0}
result = <_MultiThreadedRendezvous of RPC that terminated with:
	status = StatusCode.ABORTED
	details = "Transaction was aborte...95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"
>
prefetch_first = True
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        result = callable_(*args, **kwargs)
        # Auto-fetching the first result causes PubSub client's streaming pull
        # to hang when re-opening the stream, thus we need examine the hacky
        # hidden flag to see if pre-fetching is disabled.
        # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
        prefetch_first = getattr(callable_, "_prefetch_first_result_", True)


      return _StreamingResponseIterator(


            result, prefetch_first_result=prefetch_first
        )

.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:162:

self = <google.api_core.grpc_helpers._StreamingResponseIterator object at 0x7fb4c41fbbb0>

wrapped = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.ABORTED

details = "Transaction was aborte...95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"


prefetch_first_result = True
def __init__(self, wrapped, prefetch_first_result=True):
    self._wrapped = wrapped

    # This iterator is used in a retry context, and returned outside after init.
    # gRPC will not throw an exception until the stream is consumed, so we need
    # to retrieve the first result, in order to fail, in order to trigger a retry.
    try:
        if prefetch_first_result:


          self._stored_first_result = next(self._wrapped)


.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:88:

self = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.ABORTED

details = "Transaction was aborte...95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"


def __next__(self):


  return self._next()


.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/grpc/_channel.py:475:

self = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.ABORTED

details = "Transaction was aborte...95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"


def _next(self) -> Any:
    with self._state.condition:
        if self._state.code is None:
            event_handler = _event_handler(self._state,
                                           self._response_deserializer)
            self._state.due.add(cygrpc.OperationType.receive_message)
            operating = self._call.operate(
                (cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS),),
                event_handler)
            if not operating:
                self._state.due.remove(cygrpc.OperationType.receive_message)
        elif self._state.code is grpc.StatusCode.OK:
            raise StopIteration()
        else:
            raise self

        def _response_ready():
            return (self._state.response is not None or
                    (cygrpc.OperationType.receive_message
                     not in self._state.due and
                     self._state.code is not None))

        _common.wait(self._state.condition.wait, _response_ready)
        if self._state.response is not None:
            response = self._state.response
            self._state.response = None
            return response
        elif cygrpc.OperationType.receive_message not in self._state.due:
            if self._state.code is grpc.StatusCode.OK:
                raise StopIteration()
            elif self._state.code is not None:


              raise self


E                   grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:

E                   	status = StatusCode.ABORTED

E                   	details = "Transaction was aborted."

E                   	debug_error_string = "UNKNOWN:Error received from peer ipv4:74.125.20.95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"

E                   >
.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/grpc/_channel.py:881: _MultiThreadedRendezvous
The above exception was the direct cause of the following exception:
sessions_database = <google.cloud.spanner_v1.database.Database object at 0x7fb4c4227fd0>

sessions_to_delete = [<google.cloud.spanner_v1.session.Session object at 0x7fb4c41fa1c0>]

database_dialect = <DatabaseDialect.POSTGRESQL: 2>
@_helpers.retry_mabye_conflict
@pytest.mark.skipif(
    _helpers.USE_EMULATOR, reason="Emulator does not support DML Returning."
)
def test_transaction_execute_sql_dml_returning(
    sessions_database, sessions_to_delete, database_dialect
):
    sd = _sample_data

    session = sessions_database.session()
    session.create()
    sessions_to_delete.append(session)

    with session.batch() as batch:
        batch.delete(sd.TABLE, sd.ALL)

    with session.transaction() as transaction:
        for row in sd.ROW_DATA:
            insert_statement = _generate_insert_returning_statement(
                row, database_dialect
            )
            results = transaction.execute_sql(insert_statement)


          returned = results.one()


tests/system/test_session_api.py:808:

google/cloud/spanner_v1/streamed.py:157: in one

answer = self.one_or_none()

google/cloud/spanner_v1/streamed.py:181: in one_or_none

answer = next(iterator)

google/cloud/spanner_v1/streamed.py:145: in iter

self._consume_next()

google/cloud/spanner_v1/streamed.py:117: in _consume_next

response = next(self._response_iterator)

google/cloud/spanner_v1/snapshot.py:86: in _restart_on_unavailable

iterator = method(request=request)

google/cloud/spanner_v1/services/spanner/client.py:1204: in execute_streaming_sql

response = rpc(

.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:113: in call

return wrapped_func(*args, **kwargs)

.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/google/api_core/timeout.py:120: in func_with_timeout

return func(*args, **kwargs)

args = (session: "projects/precise-truck-742/instances/google-cloud-1683781402287/databases/test_sessions_1683781523987/sessi..., '[email protected]') RETURNING contact_id, first_name, last_name, email"

params {

}

seqno: 2

request_options {

}

,)

kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1683781402287/databa...ZSYAnp0lEH8vxFg'), ('x-goog-api-client', 'gl-python/3.8.13 grpc/1.55.0rc1 gax/2.11.0 gccl/3.33.0')], 'timeout': 3600.0}

result = <_MultiThreadedRendezvous of RPC that terminated with:

status = StatusCode.ABORTED

details = "Transaction was aborte...95:443 {grpc_message:"Transaction was aborted.", grpc_status:10, created_time:"2023-05-11T05:06:12.741805841+00:00"}"


prefetch_first = True
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        result = callable_(*args, **kwargs)
        # Auto-fetching the first result causes PubSub client's streaming pull
        # to hang when re-opening the stream, thus we need examine the hacky
        # hidden flag to see if pre-fetching is disabled.
        # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
        prefetch_first = getattr(callable_, "_prefetch_first_result_", True)
        return _StreamingResponseIterator(
            result, prefetch_first_result=prefetch_first
        )
    except grpc.RpcError as exc:


      raise exceptions.from_grpc_error(exc) from exc


E           google.api_core.exceptions.Aborted: 409 Transaction was aborted.
.nox/prerelease_deps-3-8-database_dialect-postgresql/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:166: Aborted

[flaky-bot]

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

---

When run at the same commit (998b87a), this test passed in one build (Build Status, Sponge) and failed in another build (Build Status, Sponge).