BUG: Euclid fsspec timeout error #204
Description
Run into this error in this build: https://github.com/Caltech-IPAC/irsa-tutorials/actions/runs/20293131309/job/58281330341
This is also not the first case that the traceback if way more complex than necessary because of the added ipython magic, I would recommend cleaning up the %%time usages from the notebooks as those doesn't really serve much purpose to the end users anyway. (it's been used in 3 notebooks at the time of writing this)
📖 Built tutorials/euclid_access/4_Euclid_intro_PHZ_catalog.md in 5.02 min.
⛔️ tutorials/euclid_access/Euclid_ERO.md An exception occurred during code execution, halting further execution:
To suppress this message, add rule: "code-cell-executes" to "error_rules" in your project config
📖 Built tutorials/euclid_access/Euclid_ERO.md in 5.73 min.
---------------------------------------------------------------------------
CancelledError Traceback (most recent call last)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/client_reqrep.py:539, in ClientResponse.start(self, connection)
538 protocol = self._protocol
--> 539 message, payload = await protocol.read() # type: ignore[union-attr]
540 except http.HttpProcessingError as exc:
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/streams.py:680, in DataQueue.read(self)
679 try:
--> 680 await self._waiter
681 except (asyncio.CancelledError, asyncio.TimeoutError):
CancelledError:
The above exception was the direct cause of the following exception:
TimeoutError Traceback (most recent call last)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/asyn.py:56, in _runner(event, coro, result, timeout)
55 try:
---> 56 result[0] = await coro
57 except Exception as ex:
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/implementations/http.py:692, in HTTPFile.async_fetch_range(self, start, end)
691 logger.debug(f"{self.url} : {headers['Range']}")
--> 692 r = await self.session.get(
693 self.fs.encode_url(self.url), headers=headers, **kwargs
694 )
695 async with r:
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/client.py:779, in ClientSession._request(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, server_hostname, proxy_headers, trace_request_ctx, read_bufsize, auto_decompress, max_line_size, max_field_size, middlewares)
778 try:
--> 779 resp = await handler(req)
780 # Client connector errors should not be retried
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/client.py:757, in ClientSession._request.<locals>._connect_and_send_request(req)
756 try:
--> 757 await resp.start(conn)
758 except BaseException:
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/client_reqrep.py:534, in ClientResponse.start(self, connection)
532 self._connection = connection
--> 534 with self._timer:
535 while True:
536 # read response
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/aiohttp/helpers.py:713, in TimerContext.__exit__(self, exc_type, exc_val, exc_tb)
712 return None
--> 713 raise asyncio.TimeoutError from exc_val
714 return None
TimeoutError:
The above exception was the direct cause of the following exception:
FSTimeoutError Traceback (most recent call last)
Cell In[12], line 1
----> 1 get_ipython().run_cell_magic('time', '', 'for ii,filt in tqdm(enumerate(filters)):\n products = summary_table[summary_table["filter"] == filt]["products"][0].split(";")\n\n for product in products:\n sel = np.where( (image_tab["energy_bandpassname"] == filt) & (image_tab["dataproduct_subtype"] == p
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/IPython/core/interactiveshell.py:2572, in InteractiveShell.run_cell_magic(self, magic_name, line, cell)
2570 with self.builtin_trap:
2571 args = (magic_arg_s, cell)
-> 2572 result = fn(*args, **kwargs)
2574 # The code below prevents the output from being displayed
2575 # when using magics with decorator @output_can_be_silenced
2576 # when the last Python token in the expression is a ';'.
2577 if getattr(fn, magic.MAGIC_OUTPUT_CAN_BE_SILENCED, False):
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/IPython/core/magics/execution.py:1447, in ExecutionMagics.time(self, line, cell, local_ns)
1445 if interrupt_occured:
1446 if exit_on_interrupt and captured_exception:
-> 1447 raise captured_exception
1448 return
1449 return out
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/IPython/core/magics/execution.py:1411, in ExecutionMagics.time(self, line, cell, local_ns)
1409 st = clock2()
1410 try:
-> 1411 exec(code, glob, local_ns)
1412 out = None
1413 # multi-line %%time case
File <timed exec>:8
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/nddata/utils.py:651, in Cutout2D.__init__(self, data, position, size, wcs, mode, fill_value, copy, limit_rounding_method)
647 # reverse position because extract_array and overlap_slices
648 # use (y, x), but keep the input position
649 pos_yx = position[::-1]
--> 651 cutout_data, input_position_cutout = extract_array(
652 data,
653 tuple(shape),
654 pos_yx,
655 mode=mode,
656 fill_value=fill_value,
657 return_position=True,
658 limit_rounding_method=limit_rounding_method,
659 )
660 if copy:
661 cutout_data = np.copy(cutout_data)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/nddata/utils.py:269, in extract_array(array_large, shape, position, mode, fill_value, return_position, limit_rounding_method)
260 raise ValueError("Valid modes are 'partial', 'trim', and 'strict'.")
262 large_slices, small_slices = overlap_slices(
263 array_large.shape,
264 shape,
(...) 267 limit_rounding_method=limit_rounding_method,
268 )
--> 269 extracted_array = array_large[large_slices]
270 if return_position:
271 new_position = [i - s.start for i, s in zip(position, large_slices)]
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/hdu/image.py:1044, in Section.__getitem__(self, key)
1042 data = self.hdu._get_scaled_image_data(offset, dims)
1043 else:
-> 1044 data = self._getdata(key)
1046 if return_scalar:
1047 data = data.item()
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/hdu/image.py:1064, in Section._getdata(self, keys)
1061 break
1062 # This should always break at some point if _getdata is called.
-> 1064 data = [self[keys[:idx] + (k,) + keys[idx + 1 :]] for k in ks]
1066 if any(isinstance(key, slice) or np.iterable(key) for key in keys[idx + 1 :]):
1067 # data contains multidimensional arrays; combine them.
1068 return np.array(data)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/hdu/image.py:1042, in Section.__getitem__(self, key)
1039 offset = self.hdu._data_offset + offset * abs(bitpix) // 8
1040 # Note: the actual file read operations are delegated to
1041 # `util._array_from_file` via `ImageHDU._get_scaled_image_data`
-> 1042 data = self.hdu._get_scaled_image_data(offset, dims)
1043 else:
1044 data = self._getdata(key)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/hdu/image.py:799, in _ImageBaseHDU._get_scaled_image_data(self, offset, shape)
792 """
793 Internal function for reading image data from a file and apply scale
794 factors to it. Normally this is used for the entire image, but it
795 supports alternate offset/shape for Section support.
796 """
797 code = BITPIX2DTYPE[self._orig_bitpix]
--> 799 raw_data = self._get_raw_data(shape, code, offset)
800 raw_data = raw_data.view(raw_data.dtype.newbyteorder(">"))
802 return self._scale_data(raw_data)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/hdu/base.py:552, in _BaseHDU._get_raw_data(self, shape, code, offset)
550 return np.ndarray(shape, dtype=code, buffer=self._buffer, offset=offset)
551 elif self._file:
--> 552 return self._file.readarray(offset=offset, dtype=code, shape=shape)
553 else:
554 return None
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/file.py:435, in _File.readarray(self, size, offset, dtype, shape)
433 count = reduce(operator.mul, shape)
434 self._file.seek(offset)
--> 435 data = _array_from_file(self._file, dtype, count)
436 return data.reshape(shape)
437 finally:
438 # Make sure we leave the file in the position we found it; on
439 # some platforms (e.g. Windows) mmaping a file handle can also
440 # reset its file pointer.
441 # Also for Windows when using mmap seek() may return weird
442 # negative values, which is fixed by calling tell() before.
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/astropy/io/fits/util.py:534, in _array_from_file(infile, dtype, count)
529 else:
530 # treat as file-like object with "read" method; this includes gzip file
531 # objects, because numpy.fromfile just reads the compressed bytes from
532 # their underlying file object, instead of the decompressed bytes
533 read_size = np.dtype(dtype).itemsize * count
--> 534 s = infile.read(read_size)
535 array = np.ndarray(buffer=s, dtype=dtype, shape=(count,))
536 # copy is needed because np.frombuffer returns a read-only view of the
537 # underlying buffer
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/implementations/http.py:644, in HTTPFile.read(self, length)
642 else:
643 length = min(self.size - self.loc, length)
--> 644 return super().read(length)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/spec.py:2122, in AbstractBufferedFile.read(self, length)
2119 if length == 0:
2120 # don't even bother calling fetch
2121 return b""
-> 2122 out = self.cache._fetch(self.loc, self.loc + length)
2124 logger.debug(
2125 "%s read: %i - %i %s",
2126 self,
(...) 2129 self.cache._log_stats(),
2130 )
2131 self.loc += len(out)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/caching.py:552, in BytesCache._fetch(self, start, end)
550 else:
551 self.total_requested_bytes += bend - self.end
--> 552 new = self.fetcher(self.end, bend)
553 self.cache = self.cache + new
555 self.end = self.start + len(self.cache)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/asyn.py:118, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
115 @functools.wraps(func)
116 def wrapper(*args, **kwargs):
117 self = obj or args[0]
--> 118 return sync(self.loop, func, *args, **kwargs)
File ~/work/irsa-tutorials/irsa-tutorials/.tox/py312-buildhtml/lib/python3.12/site-packages/fsspec/asyn.py:101, in sync(loop, func, timeout, *args, **kwargs)
98 return_result = result[0]
99 if isinstance(return_result, asyncio.TimeoutError):
100 # suppress asyncio.TimeoutError, raise FSTimeoutError
--> 101 raise FSTimeoutError from return_result
102 elif isinstance(return_result, BaseException):
103 raise return_result
FSTimeoutError:
📖 Built tutorials/euclid_access/5_Euclid_intro_SPE_catalog.md in 5.75 min.