Блог

Я пытаюсь получить результат запроса Google Bigquery во фрейме данных pandas (в Jupiter notebook).

Но каждый раз, когда я пытаюсь выполнить запрос, я получаю превышенный дедлайн: превышен дедлайн 504.

Это происходит не только для запросов в моем собственном проекте BQ, но и для других проектов. Я перепробовал множество вариантов выполнения запроса, как здесь:https://cloud.google.com/bigquery/docs/bigquery-storage-python-pandas

У кого-нибудь есть идея, как это исправить?

Запрос:

 %load_ext google.cloud.bigquery

 %%bigquery tax_forms --use_bqstorage_api
 ВЫБЕРИТЕ * ИЗ `bigquery-public-data.irs_990.irs_990_2012`

 ---------------------------------------------------------------------------
_MultiThreadedRendezvous                  Traceback (most recent call last)
~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coregrpc_helpers.py in error_remapped_callable(*args, **kwargs)
    149             prefetch_first = getattr(callable_, "_prefetch_first_result_", True)
--> 150             return _StreamingResponseIterator(result, prefetch_first_result=prefetch_first)
    151         except grpc.RpcError as exc:

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coregrpc_helpers.py in __init__(self, wrapped, prefetch_first_result)
     72             if prefetch_first_result:
---> 73                 self._stored_first_result = six.next(self._wrapped)
     74         except TypeError:

~AppDataLocalContinuumanaconda3libsite-packagesgrpc_channel.py in __next__(self)
    415     def __next__(self):
--> 416         return self._next()
    417 

~AppDataLocalContinuumanaconda3libsite-packagesgrpc_channel.py in _next(self)
    705                 elif self._state.code is not None:
--> 706                     raise self
    707 

_MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
    status = StatusCode.DEADLINE_EXCEEDED
    details = "Deadline Exceeded"
    debug_error_string = "{"created":"@1597838569.388000000","description":"Error received from peer ipv4:172.217.168.202:443","file":"src/core/lib/surface/call.cc","file_line":1062,"grpc_message":"Deadline Exceeded","grpc_status":4}"
>

The above exception was the direct cause of the following exception:

DeadlineExceeded                          Traceback (most recent call last)
<ipython-input-2-4fdaec7219df> in <module>
----> 1 get_ipython().run_cell_magic('bigquery', 'tax_forms --use_bqstorage_api', 'SELECT * FROM `bigquery-public-data.irs_990.irs_990_2012`n')

~AppDataLocalContinuumanaconda3libsite-packagesIPythoncoreinteractiveshell.py in run_cell_magic(self, magic_name, line, cell)
   2357             with self.builtin_trap:
   2358                 args = (magic_arg_s, cell)
-> 2359                 result = fn(*args, **kwargs)
   2360             return result
   2361 

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquerymagics.py in _cell_magic(line, query)
    589             )
    590         else:
--> 591             result = query_job.to_dataframe(bqstorage_client=bqstorage_client)
    592 
    593         if args.destination_var:

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigqueryjob.py in to_dataframe(self, bqstorage_client, dtypes, progress_bar_type, create_bqstorage_client, date_as_object)
   3381             progress_bar_type=progress_bar_type,
   3382             create_bqstorage_client=create_bqstorage_client,
-> 3383             date_as_object=date_as_object,
   3384         )
   3385 

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquerytable.py in to_dataframe(self, bqstorage_client, dtypes, progress_bar_type, create_bqstorage_client, date_as_object)
   1726                 progress_bar_type=progress_bar_type,
   1727                 bqstorage_client=bqstorage_client,
-> 1728                 create_bqstorage_client=create_bqstorage_client,
   1729             )
   1730 

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquerytable.py in to_arrow(self, progress_bar_type, bqstorage_client, create_bqstorage_client)
   1544             record_batches = []
   1545             for record_batch in self._to_arrow_iterable(
-> 1546                 bqstorage_client=bqstorage_client
   1547             ):
   1548                 record_batches.append(record_batch)

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquerytable.py in _to_page_iterable(self, bqstorage_download, tabledata_list_download, bqstorage_client)
   1433     ):
   1434         if bqstorage_client is not None:
-> 1435             for item in bqstorage_download():
   1436                 yield item
   1437             return

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquery_pandas_helpers.py in _download_table_bqstorage(project_id, table, bqstorage_client, preserve_order, selected_fields, page_to_item)
    723                     # Call result() on any finished threads to raise any
    724                     # exceptions encountered.
--> 725                     future.result()
    726 
    727                 try:

~AppDataLocalContinuumanaconda3libconcurrentfutures_base.py in result(self, timeout)
    426                 raise CancelledError()
    427             elif self._state == FINISHED:
--> 428                 return self.__get_result()
    429 
    430             self._condition.wait(timeout)

~AppDataLocalContinuumanaconda3libconcurrentfutures_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

~AppDataLocalContinuumanaconda3libconcurrentfuturesthread.py in run(self)
     55 
     56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
     58         except BaseException as exc:
     59             self.future.set_exception(exc)

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquery_pandas_helpers.py in _download_table_bqstorage_stream(download_state, bqstorage_client, session, stream, worker_queue, page_to_item)
    591         rowstream = bqstorage_client.read_rows(position).rows(session)
    592     else:
--> 593         rowstream = bqstorage_client.read_rows(stream.name).rows(session)
    594 
    595     for page in rowstream.pages:

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquery_storage_v1client.py in read_rows(self, name, offset, retry, timeout, metadata)
    120             retry=retry,
    121             timeout=timeout,
--> 122             metadata=metadata,
    123         )
    124         return reader.ReadRowsStream(

~AppDataLocalContinuumanaconda3libsite-packagesgooglecloudbigquery_storage_v1gapicbig_query_read_client.py in read_rows(self, read_stream, offset, retry, timeout, metadata)
    370 
    371         return self._inner_api_calls["read_rows"](
--> 372             request, retry=retry, timeout=timeout, metadata=metadata
    373         )
    374 

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coregapic_v1method.py in __call__(self, *args, **kwargs)
    143             kwargs["metadata"] = metadata
    144 
--> 145         return wrapped_func(*args, **kwargs)
    146 
    147 

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coreretry.py in retry_wrapped_func(*args, **kwargs)
    284                 sleep_generator,
    285                 self._deadline,
--> 286                 on_error=on_error,
    287             )
    288 

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coreretry.py in retry_target(target, predicate, sleep_generator, deadline, on_error)
    182     for sleep in sleep_generator:
    183         try:
--> 184             return target()
    185 
    186         # pylint: disable=broad-except

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coretimeout.py in func_with_timeout(*args, **kwargs)
    212             """Wrapped function that adds timeout."""
    213             kwargs["timeout"] = next(timeouts)
--> 214             return func(*args, **kwargs)
    215 
    216         return func_with_timeout

~AppDataLocalContinuumanaconda3libsite-packagesgoogleapi_coregrpc_helpers.py in error_remapped_callable(*args, **kwargs)
    150             return _StreamingResponseIterator(result, prefetch_first_result=prefetch_first)
    151         except grpc.RpcError as exc:
--> 152             six.raise_from(exceptions.from_grpc_error(exc), exc)
    153 
    154     return error_remapped_callable

~AppDataLocalContinuumanaconda3libsite-packagessix.py in raise_from(value, from_value)

DeadlineExceeded: 504 Deadline Exceeded
  

Дайте мне знать, если вам нужно узнать больше. Заранее спасибо.

Rutger

Оказалось, что это конфликт между пакетами Conda и pip.

Я решил проблему, переустановив все пакеты.