bug: reading Delta Lake tables hits arrow_scan issue in 10.0.0 #10829
Description
What happened?
noticed trying to read Delta Lake tables was failing, it seems like it's related to upgrading to 10.0.0. simple reproduction after installing in a fresh virtual environment on main:
(ibis) cody@dkdcascend ibis % ipy
imporPython 3.12.8 (main, Jan 14 2025, 23:36:58) [Clang 19.1.6 ]
Type 'copyright', 'credits' or 'license' for more information
IPython 8.32.0 -- An enhanced Interactive Python. Type '?' for help.
[ins] In [1]: import ibis
i
[ins] In [2]: ibis.__version__
Out[2]: '10.0.0'
[ins] In [3]: ibis.options.interactive = True
[ins] In [4]: t = ibis.examples.penguins.fetch()
[ins] In [5]: t.to_delta("penguins.delta")
[ins] In [6]: t = ibis.read_delta("penguins.delta")
[ins] In [7]: t
Out[7]: ---------------------------------------------------------------------------
InvalidInputException Traceback (most recent call last)
File ~/code/ibis/.venv/lib/python3.12/site-packages/IPython/core/formatters.py:770, in PlainTextFormatter.__call__(self, obj)
763 stream = StringIO()
764 printer = pretty.RepresentationPrinter(stream, self.verbose,
765 self.max_width, self.newline,
766 max_seq_length=self.max_seq_length,
767 singleton_pprinters=self.singleton_printers,
768 type_pprinters=self.type_printers,
769 deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
771 printer.flush()
772 return stream.getvalue()
File ~/code/ibis/.venv/lib/python3.12/site-packages/IPython/lib/pretty.py:419, in RepresentationPrinter.pretty(self, obj)
408 return meth(obj, self, cycle)
409 if (
410 cls is not object
411 # check if cls defines __repr__
(...)
417 and callable(_safe_getattr(cls, "__repr__", None))
418 ):
--> 419 return _repr_pprint(obj, self, cycle)
421 return _default_pprint(obj, self, cycle)
422 finally:
File ~/code/ibis/.venv/lib/python3.12/site-packages/IPython/lib/pretty.py:794, in _repr_pprint(obj, p, cycle)
792 """A pprint that just redirects to the normal repr function."""
793 # Find newlines and replace them with p.break_()
--> 794 output = repr(obj)
795 lines = output.splitlines()
796 with p.group():
File ~/code/ibis/ibis/expr/types/core.py:83, in Expr.__repr__(self)
81 def __repr__(self) -> str:
82 if ibis.options.interactive:
---> 83 return _capture_rich_renderable(self)
84 else:
85 return self._noninteractive_repr()
File ~/code/ibis/ibis/expr/types/core.py:63, in _capture_rich_renderable(renderable)
61 console = Console(force_terminal=False)
62 with console.capture() as capture:
---> 63 console.print(renderable)
64 return capture.get().rstrip()
File ~/code/ibis/.venv/lib/python3.12/site-packages/rich/console.py:1705, in Console.print(self, sep, end, style, justify, overflow, no_wrap, emoji, markup, highlight, width, height, crop, soft_wrap, new_line_start, *objects)
1703 if style is None:
1704 for renderable in renderables:
-> 1705 extend(render(renderable, render_options))
1706 else:
1707 for renderable in renderables:
File ~/code/ibis/.venv/lib/python3.12/site-packages/rich/console.py:1306, in Console.render(self, renderable, options)
1304 renderable = rich_cast(renderable)
1305 if hasattr(renderable, "__rich_console__") and not isclass(renderable):
-> 1306 render_iterable = renderable.__rich_console__(self, _options)
1307 elif isinstance(renderable, str):
1308 text_renderable = self.render_str(
1309 renderable, highlight=_options.highlight, markup=_options.markup
1310 )
File ~/code/ibis/ibis/expr/types/core.py:106, in Expr.__rich_console__(self, console, options)
103 if opts.interactive:
104 from ibis.expr.types.pretty import to_rich
--> 106 rich_object = to_rich(self, console_width=console_width)
107 else:
108 rich_object = Text(self._noninteractive_repr())
File ~/code/ibis/ibis/expr/types/pretty.py:279, in to_rich(expr, max_rows, max_columns, max_length, max_string, max_depth, console_width)
275 return _to_rich_scalar(
276 expr, max_length=max_length, max_string=max_string, max_depth=max_depth
277 )
278 else:
--> 279 return _to_rich_table(
280 expr,
281 max_rows=max_rows,
282 max_columns=max_columns,
283 max_length=max_length,
284 max_string=max_string,
285 max_depth=max_depth,
286 console_width=console_width,
287 )
File ~/code/ibis/ibis/expr/types/pretty.py:358, in _to_rich_table(tablish, max_rows, max_columns, max_length, max_string, max_depth, console_width)
355 if orig_ncols > len(computed_cols):
356 table = table.select(*computed_cols)
--> 358 result = table.limit(max_rows + 1).to_pyarrow()
359 # Now format the columns in order, stopping if the console width would
360 # be exceeded.
361 col_info = []
File ~/code/ibis/ibis/expr/types/core.py:579, in Expr.to_pyarrow(self, params, limit, **kwargs)
551 @experimental
552 def to_pyarrow(
553 self,
(...)
557 **kwargs: Any,
558 ) -> pa.Table:
559 """Execute expression and return results in as a pyarrow table.
560
561 This method is eager and will execute the associated expression
(...)
577 A pyarrow table holding the results of the executed expression.
578 """
--> 579 return self._find_backend(use_default=True).to_pyarrow(
580 self, params=params, limit=limit, **kwargs
581 )
File ~/code/ibis/ibis/backends/duckdb/__init__.py:1314, in Backend.to_pyarrow(self, expr, params, limit, **kwargs)
1303 def to_pyarrow(
1304 self,
1305 expr: ir.Expr,
(...)
1310 **kwargs: Any,
1311 ) -> pa.Table:
1312 table = self._to_duckdb_relation(
1313 expr, params=params, limit=limit, **kwargs
-> 1314 ).arrow()
1315 return expr.__pyarrow_result__(table, data_mapper=DuckDBPyArrowData)
InvalidInputException: Invalid Input Error: arrow_scan: get_next failed(): IOError: Repetition level histogram size mismatch
[ins] In [8]: exit
(ibis) cody@dkdcascend ibis % uv pip list | grep pyarrow
pyarrow 19.0.0
pyarrow-hotfix 0.6
What version of ibis are you using?
main/10.0.0
What backend(s) are you using, if any?
duckdb
Relevant log output
Code of Conduct
- I agree to follow this project's Code of Conduct