Skip to content

Commit 695414d

Browse files
wangzeyu03claude
andcommitted
[python] Expose commit_mode and options on the public write_paimon
Actor mode was only reachable by importing write_paimon_dataset from pypaimon.write.ray_datasink; write_paimon, the documented entry point, had no way to select it or to pass dynamic table options. Forward both kwargs and document what actor mode trades away: a partial write stays committed if a later task fails, and the returned snapshot id is the table's latest rather than this write's own. Co-Authored-By: Claude Code <noreply@anthropic.com>
1 parent 81011f9 commit 695414d

2 files changed

Lines changed: 80 additions & 2 deletions

File tree

‎paimon-python/pypaimon/ray/ray_paimon.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,8 @@ def write_paimon(
545545
concurrency: Optional[int] = None,
546546
ray_remote_args: Optional[Dict[str, Any]] = None,
547547
hash_fixed_precluster: str = "auto",
548+
commit_mode: str = "two_phase",
549+
options: Optional[Dict[str, str]] = None,
548550
) -> "Optional[PaimonWriteResult]":
549551
"""Write a Ray Dataset to a Paimon table.
550552
@@ -567,10 +569,26 @@ def write_paimon(
567569
hash_fixed_precluster: Pre-clustering mode. ``"auto"`` follows
568570
table options, ``"off"`` disables it, and ``"map_groups"``
569571
explicitly enables HASH_FIXED grouping.
572+
commit_mode: ``"two_phase"`` (default) has the write tasks return
573+
their commit messages to the driver, which commits once after
574+
every task finishes, so the write is all-or-nothing and the
575+
returned snapshot is exactly the one it committed.
576+
``"actor"`` instead has each task commit through a singleton
577+
Ray actor, which serializes the commits and makes each task's
578+
rows visible as soon as that task returns -- at the cost of a
579+
partial write staying committed if a later task fails, and of
580+
the returned snapshot id being the table's latest rather than
581+
this write's own. Actor mode is append-only and rejects
582+
``overwrite``, ``hash_fixed_precluster="map_groups"`` and
583+
postpone-bucket tables.
584+
options: Optional dynamic table options applied for this write
585+
only, e.g. ``{"target-file-size": "256mb"}``.
570586
571587
Returns:
572-
Metadata for the exact committed snapshot, or ``None`` when no
573-
snapshot was committed.
588+
Metadata for the committed snapshot, or ``None`` when no snapshot
589+
was committed. With ``commit_mode="actor"`` this is the table's
590+
latest snapshot after the write, which a concurrent writer may
591+
have produced; see ``commit_mode`` above.
574592
"""
575593
_require_ray_data()
576594

@@ -587,4 +605,6 @@ def write_paimon(
587605
concurrency=concurrency,
588606
ray_remote_args=ray_remote_args,
589607
hash_fixed_precluster=hash_fixed_precluster,
608+
commit_mode=commit_mode,
609+
options=options,
590610
)

‎paimon-python/pypaimon/tests/ray_actor_commit_test.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,64 @@ def test_two_phase_mode_applies_options(self):
296296
)
297297
self.assertEqual(len(self._read_all(table)), 3)
298298

299+
def test_public_write_paimon_reaches_actor_mode(self):
300+
"""``commit_mode`` and ``options`` must be reachable through the
301+
documented public entry point, not only through the internal
302+
``write_paimon_dataset``. ``file.format`` doubles as proof that
303+
both kwargs really traversed the wrapper rather than being
304+
accepted and dropped.
305+
"""
306+
from pypaimon.ray.ray_paimon import write_paimon
307+
308+
table = self._create_append_table('test_public_actor')
309+
310+
ds = ray.data.from_arrow(
311+
pa.Table.from_pydict(
312+
{'id': [1, 2, 3], 'value': ['a', 'b', 'c']}, schema=SCHEMA
313+
)
314+
)
315+
316+
result = write_paimon(
317+
ds, 'default.test_public_actor', {'warehouse': self.warehouse},
318+
commit_mode='actor',
319+
options={'file.format': 'orc'},
320+
)
321+
322+
self.assertEqual(result.row_count, 3)
323+
324+
table_dir = os.path.join(
325+
self.warehouse, 'default.db', 'test_public_actor'
326+
)
327+
self.assertTrue(
328+
glob.glob(os.path.join(table_dir, '**', '*.orc'), recursive=True),
329+
f'no .orc data files under {table_dir}',
330+
)
331+
self.assertFalse(
332+
glob.glob(os.path.join(table_dir, '**', '*.parquet'), recursive=True),
333+
'file.format=orc did not survive the public wrapper',
334+
)
335+
self.assertEqual(len(self._read_all(table)), 3)
336+
337+
def test_public_write_paimon_rejects_unknown_commit_mode(self):
338+
"""The wrapper must forward ``commit_mode`` to the validation that
339+
rejects it, rather than silently defaulting to two_phase.
340+
"""
341+
from pypaimon.ray.ray_paimon import write_paimon
342+
343+
self._create_append_table('test_public_bad_mode')
344+
345+
ds = ray.data.from_arrow(
346+
pa.Table.from_pydict({'id': [1], 'value': ['a']}, schema=SCHEMA)
347+
)
348+
349+
with self.assertRaises(ValueError) as ctx:
350+
write_paimon(
351+
ds, 'default.test_public_bad_mode',
352+
{'warehouse': self.warehouse},
353+
commit_mode='nonsense',
354+
)
355+
self.assertIn('nonsense', str(ctx.exception))
356+
299357
# ------------------------------------------------------------------
300358
# Commit dedup (actor replay / whole-task retry)
301359
# ------------------------------------------------------------------

0 commit comments

Comments
 (0)