We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent e7290a7 commit 4f5532dCopy full SHA for 4f5532d
benchmarks/collect_gil_bench.py
@@ -23,14 +23,13 @@
23
24
def run(n_batches: int = 8, batch_size: int = 1_000_000) -> None:
25
ctx = SessionContext()
26
- partitions = []
+ batches = []
27
for i in range(n_batches):
28
start = i * batch_size
29
arr = pa.array(range(start, start + batch_size))
30
- batch = pa.record_batch([arr], names=["a"])
31
- partitions.append([batch]) # Each batch in its own partition
+ batches.append(pa.record_batch([arr], names=["a"]))
32
33
- df = ctx.create_dataframe(partitions)
+ df = ctx.create_dataframe([batches])
34
35
start = time.perf_counter()
36
df.collect()
0 commit comments