Skip to content

Commit b11f902

Browse files
avenkateshhaclaude
andcommitted
fix(xtoken): mark alignment pairs over tokenizer padding invalid
TokenAligner aligns the padded student and teacher sequences, so it also pairs padding with padding. _drop_padding reset the padded positions' chunk ids to -1 but left pair_valid True for every pair. CSD's per-step chunk count, sum(pair_valid * sample_mask), therefore counted padding pairs: on a padded text batch of ten documents it counted 7515 pairs for 2330 chunks holding real tokens, which shrank the KD term (and its logged value) by a batch-dependent factor. When both attention masks are given, _drop_padding now marks a pair invalid if it has tokens on both sides and all of its student or all of its teacher tokens are padding. Pairs that are one-sided by alignment, such as the student-only start-of-text pair, keep their flag. The CSD chunk count then equals the number of chunks that hold real tokens. X-Token losses are unchanged: their chunk mask already requires real tokens on both sides, so every pair this marks invalid was already excluded. CSD's per-chunk terms are unchanged too; only its chunk count, and so the KD scale, moves. Under dynamic_loss_scaling the training loss is unchanged, while the logged KD values grow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Adithya Hanasoge <avenkateshha@nvidia.com>
1 parent 9bdc5d3 commit b11f902

3 files changed

Lines changed: 89 additions & 8 deletions

File tree

‎nemo_rl/algorithms/x_token/token_aligner.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,8 @@ class AlignmentBatch:
252252
V2 can shard on dim 0 without knowing about cross-tokenizer specifics.
253253
254254
Attributes:
255-
pair_valid: ``[B, max_pairs]`` bool. False on padding entries.
255+
pair_valid: ``[B, max_pairs]`` bool. False on padding entries and on
256+
pairs whose student or teacher tokens are all tokenizer padding.
256257
pair_is_correct: ``[B, max_pairs]`` bool. True when decoded student
257258
and teacher span texts are equal, when an NFC-validated native
258259
repair matches them, or when both spans are their tokenizer's
@@ -547,22 +548,41 @@ def _drop_padding(
547548
student_attention_mask: torch.Tensor | None,
548549
teacher_attention_mask: torch.Tensor | None,
549550
) -> None:
550-
"""Strip tokenizer padding out of the chunk-id tensors.
551+
"""Strip tokenizer padding out of the chunk ids and invalidate padding pairs.
551552
552553
Mutates ``batch`` in place. For every position the attention mask
553554
marks as padding, reset ``*_chunk_id`` to ``-1``. Gating per position
554555
(rather than trimming a contiguous span) keeps this correct under either
555-
left- or right-padding. A pair whose tokens are entirely padding on
556-
one side then has size 0 there and is dropped by
557-
:func:`nemo_rl.algorithms.x_token.loss_utils.valid_chunk_mask`; a pair
558-
straddling the real/pad boundary shrinks to its real tokens.
556+
left- or right-padding; a pair straddling the real/pad boundary shrinks
557+
to its real tokens. When both masks are given, a pair with tokens on
558+
both sides whose student or teacher tokens are all padding becomes
559+
invalid in ``pair_valid``. A pair with no tokens on one side (one-sided
560+
by alignment) keeps its flag.
559561
"""
562+
max_pairs = batch.pair_valid.shape[1]
563+
two_sided = TokenAligner._pairs_with_tokens(
564+
batch.student_chunk_id, max_pairs
565+
) & TokenAligner._pairs_with_tokens(batch.teacher_chunk_id, max_pairs)
560566
if student_attention_mask is not None:
561567
s_pad = student_attention_mask == 0
562568
batch.student_chunk_id[s_pad] = -1
563569
if teacher_attention_mask is not None:
564570
t_pad = teacher_attention_mask == 0
565571
batch.teacher_chunk_id[t_pad] = -1
572+
if student_attention_mask is None or teacher_attention_mask is None:
573+
return
574+
two_sided_real = TokenAligner._pairs_with_tokens(
575+
batch.student_chunk_id, max_pairs
576+
) & TokenAligner._pairs_with_tokens(batch.teacher_chunk_id, max_pairs)
577+
batch.pair_valid &= two_sided_real | ~two_sided
578+
579+
@staticmethod
580+
def _pairs_with_tokens(chunk_id: torch.Tensor, max_pairs: int) -> torch.Tensor:
581+
"""``[B, max_pairs]`` bool: True where the pair owns a position of ``chunk_id``."""
582+
# Column 0 counts the -1 sentinel; column p + 1 counts pair p.
583+
counts = torch.zeros((chunk_id.shape[0], max_pairs + 1), dtype=torch.long)
584+
counts.scatter_add_(1, chunk_id + 1, torch.ones_like(chunk_id))
585+
return counts[:, 1:] > 0
566586

567587
@staticmethod
568588
def _pairs_to_batch(

‎tests/unit/algorithms/x_token/test_token_aligner.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,43 @@ def test_align_padding_positions_receive_sentinel_chunk_id():
288288
assert bool((t_sizes[chunk_mask] > 0).all())
289289

290290

291+
def test_align_pair_valid_drops_padding_pairs():
292+
"""In a right-padded batch with rows of different lengths, the pairs over
293+
padding are invalid, while real two-sided pairs and the student-only
294+
start-of-text pair (one-sided by alignment) stay valid.
295+
"""
296+
vocab = {0: "<pad>", 7: "<bos>", 1: "Hello", 2: "Ġworld", 3: "!"}
297+
aligner = _make_aligner(vocab, vocab, bos_token_id=7, pad_token_id=0)
298+
299+
# Only the student starts with <bos>; row 1 is two tokens shorter.
300+
student_ids = torch.tensor([[7, 1, 2, 3], [7, 1, 0, 0]], dtype=torch.long)
301+
teacher_ids = torch.tensor([[1, 2, 3], [1, 0, 0]], dtype=torch.long)
302+
student_offsets = torch.tensor(
303+
[[[0, 0], [0, 5], [5, 11], [11, 12]], [[0, 0], [0, 5], [0, 0], [0, 0]]],
304+
dtype=torch.long,
305+
)
306+
teacher_offsets = torch.tensor(
307+
[[[0, 5], [5, 11], [11, 12]], [[0, 5], [0, 0], [0, 0]]], dtype=torch.long
308+
)
309+
310+
batch = aligner.align(
311+
student_ids,
312+
teacher_ids,
313+
student_offsets=student_offsets,
314+
teacher_offsets=teacher_offsets,
315+
student_attention_mask=student_ids.ne(0),
316+
teacher_attention_mask=teacher_ids.ne(0),
317+
)
318+
319+
# Pair 0 is <bos> alone; row 1's pairs 2-3 pair padding with padding.
320+
assert batch.student_chunk_id.tolist() == [[0, 1, 2, 3], [0, 1, -1, -1]]
321+
assert batch.teacher_chunk_id.tolist() == [[1, 2, 3], [1, -1, -1]]
322+
assert batch.pair_valid.tolist() == [
323+
[True, True, True, True],
324+
[True, True, False, False],
325+
]
326+
327+
291328
def test_module_exposes_alignment_kernel():
292329
"""The offset kernel is reachable from the package namespace (callers use
293330
it directly instead of poking at name-mangled internals).

‎tests/unit/data/test_cross_tokenizer_collate.py‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,8 @@ def test_long_text_truncates_trailing_tokens_not_drops_sample(self):
404404
# Captured from the collator before the optional overlength and CE-turn
405405
# behaviors existed; the default batch must stay bitwise identical. Its first
406406
# row fills each side's context, so padding to the longest row keeps the
407-
# full-context width.
407+
# full-context width. ``alignment_0_pair_valid`` was recaptured when pairs
408+
# over padding became invalid.
408409
_TEXT_SNAPSHOT_CHECKSUMS = {
409410
"input_ids": 3661199532,
410411
"input_lengths": 868467641,
@@ -414,7 +415,7 @@ def test_long_text_truncates_trailing_tokens_not_drops_sample(self):
414415
"teacher_0_input_ids": 548984487,
415416
"teacher_0_input_lengths": 137384594,
416417
"teacher_0_token_mask": 3787835366,
417-
"alignment_0_pair_valid": 64817739,
418+
"alignment_0_pair_valid": 2277715625,
418419
"alignment_0_pair_is_correct": 13516310,
419420
"alignment_0_student_chunk_id": 1888554585,
420421
"alignment_0_teacher_chunk_id": 1481747482,
@@ -797,6 +798,29 @@ def test_batch_matches_full_context_padding_at_real_positions(self):
797798
for key in ("alignment_0_pair_valid", "alignment_0_pair_is_correct"):
798799
assert torch.equal(new[key][row, pairs], old[key][row, pairs]), key
799800

801+
def test_pair_valid_counts_real_chunks(self):
802+
"""``pair_valid`` counts the pairs holding real tokens, including the
803+
student-only start-of-text pair, and none of the pairs over padding.
804+
"""
805+
batch = [_datum(text, idx=i) for i, text in enumerate(self._TEXTS)]
806+
out = _text_collator(
807+
ctx_length_student=16,
808+
ctx_length_teachers=[64, 16],
809+
student=_word_tokenizer(bos=True),
810+
add_special_tokens_in_text=True,
811+
)(batch)
812+
real = 0
813+
for row in range(len(batch)):
814+
student = out["alignment_0_student_chunk_id"][row]
815+
teacher = out["alignment_0_teacher_chunk_id"][row]
816+
real += len(
817+
set(student[student >= 0].tolist())
818+
| set(teacher[teacher >= 0].tolist())
819+
)
820+
# One pair per word plus each row's start-of-text pair.
821+
assert real == 8 + 3
822+
assert out["alignment_0_pair_valid"].sum().item() == real
823+
800824

801825
class TestCollatorPadTokenFallback:
802826
def test_missing_pad_token_set_from_eos(self):

0 commit comments

Comments
 (0)