Skip to content

tests.system.test_table_api: test_db_list_tables_reload failed #515

@flaky-bot

Description

@flaky-bot

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.


commit: bdd5f8b
buildURL: Build Status, Sponge
status: failed

Test output
target = functools.partial(>)
predicate = .if_exception_type_predicate at 0x7f6d8182d3a0>
sleep_generator = 
deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
          return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:


self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _done_or_raise(self, retry=DEFAULT_RETRY):
    """Check if the future is done and raise if it's not."""
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

    if not self.done(**kwargs):
      raise _OperationNotComplete()

E google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
      retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:


args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>

@general_helpers.wraps(func)
def retry_wrapped_func(*args, **kwargs):
    """A wrapper that calls target function with retry."""
    target = functools.partial(func, *args, **kwargs)
    sleep_generator = exponential_sleep_generator(
        self._initial, self._maximum, multiplier=self._multiplier
    )
  return retry_target(
        target,
        self._predicate,
        sleep_generator,
        self._deadline,
        on_error=on_error,
    )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:


target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)
predicate = <function if_exception_type..if_exception_type_predicate at 0x7f6d8182d3a0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>
deadline = 60, on_error = None

def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:
              six.raise_from(
                    exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ),
                    last_exc,
                )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:


value = None, from_value = _OperationNotComplete()

???
E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>), last exception:

:3: RetryError

During handling of the above exception, another exception occurred:

spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7f6d811926d0>
operation_timeout = 60, shared_instance_id = 'google-cloud-1628986419564'
instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"
display_name: "us-central1"
replicas {
locat...
replicas {
location: "us-central1"
type_: READ_WRITE
}
replicas {
location: "us-central1"
type_: READ_WRITE
}

existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"
config: "projects/precise-truck-742/instanceConfi...ated"
value: "1628986209"
}
labels {
key: "python-spanner-systests"
value: "true"
}
processing_units: 1000
, ...]

@pytest.fixture(scope="session")
def shared_instance(
    spanner_client,
    operation_timeout,
    shared_instance_id,
    instance_config,
    existing_instances,  # evalutate before creating one
):
    _helpers.cleanup_old_instances(spanner_client)

    if _helpers.CREATE_INSTANCE:
        create_time = str(int(time.time()))
        labels = {"python-spanner-systests": "true", "created": create_time}

        instance = spanner_client.instance(
            shared_instance_id, instance_config.name, labels=labels
        )
        created_op = _helpers.retry_429_503(instance.create)()
      created_op.result(operation_timeout)  # block until completion

tests/system/conftest.py:119:


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
self._blocking_poll(timeout=timeout, **kwargs)


self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
        retry_(self._done_or_raise)(**kwargs)
    except exceptions.RetryError:
      raise concurrent.futures.TimeoutError(
            "Operation did not complete within the designated " "timeout."
        )

E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

Metadata

Metadata

Assignees

Labels

🚨This issue needs some love.api: spannerIssues related to the googleapis/python-spanner API.flakybot: flakyTells the Flaky Bot not to close or comment on this issue.flakybot: issueAn issue filed by the Flaky Bot. Should not be added manually.priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions