Test runner added plus fix(es)

tests/generators/fork_choice_generated/instantiators/helpers.py

@@ -19,7 +19,10 @@
     output_store_checks,
     run_on_attestation,
     run_on_attester_slashing,
-    run_on_block
+    run_on_block,
+    get_block_file_name,
+    get_attestation_file_name,
+    get_attester_slashing_file_name,
 )
 from .debug_helpers import print_head
 
@@ -304,6 +307,37 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
+def _add_block(spec, store, signed_block, test_steps):
+    """
+    Helper method to add a block, when it's unknown whether it's valid or not
+    """
+    yield get_block_file_name(signed_block), signed_block
+    try:
+        run_on_block(spec, store, signed_block)
+        valid = True
+    except AssertionError:
+        valid = False
+
+    test_steps.append({'block': get_block_file_name(signed_block), 'valid': valid})
+
+    if valid:
+        # An on_block step implies receiving block's attestations
+        for attestation in signed_block.message.body.attestations:
+            try:
+                run_on_attestation(spec, store, attestation, is_from_block=True, valid=True)
+            except AssertionError:
+                # ignore possible faults, if the block is valud
+                pass
+
+        # An on_block step implies receiving block's attester slashings
+        for attester_slashing in signed_block.message.body.attester_slashings:
+            try:
+                run_on_attester_slashing(spec, store, attester_slashing, valid=True)
+            except AssertionError:
+                # ignore possible faults, if the block is valud
+                pass
+
+
 @filter_out_duplicate_messages
 def yield_fork_choice_test_events(spec, store, test_data: FCTestData, test_events: list, debug: bool):
     # Yield meta
@@ -314,6 +348,18 @@ def yield_fork_choice_test_events(spec, store, test_data: FCTestData, test_event
     yield 'anchor_state', test_data.anchor_state
     yield 'anchor_block', test_data.anchor_block
 
+    for message in test_data.blocks:
+        block = message.payload
+        yield get_block_file_name(block), block.encode_bytes()
+
+    for message in test_data.atts:
+        attestation = message.payload
+        yield get_attestation_file_name(attestation), attestation.encode_bytes()
+
+    for message in test_data.slashings:
+        attester_slashing = message.payload
+        yield get_attester_slashing_file_name(attester_slashing), attester_slashing.encode_bytes()
+
     test_steps = []
 
     def try_add_mesage(runner, message):
@@ -323,34 +369,41 @@ def try_add_mesage(runner, message):
         except AssertionError:
             return False
 
+    # record initial tick
+    on_tick_and_append_step(spec, store, store.time, test_steps, checks_with_viable_for_head_weights=True)
+
     for event in test_events:
         event_kind = event[0]
         if event_kind == 'tick':
             _, time, _ = event
             if time > store.time:
-                on_tick_and_append_step(spec, store, time, test_steps)
+                on_tick_and_append_step(spec, store, time, test_steps, checks_with_viable_for_head_weights=True)
                 assert store.time == time
         elif event_kind == 'block':
             _, signed_block, valid = event
             if valid is None:
-                valid = try_add_mesage(run_on_block, signed_block)
-            yield from add_block(spec, store, signed_block, test_steps, valid=valid)
-
-            block_root = signed_block.message.hash_tree_root()
-            if valid:
-                assert store.blocks[block_root] == signed_block.message
+                yield from _add_block(spec, store, signed_block, test_steps)
             else:
-                assert block_root not in store.blocks.values()
+                yield from add_block(spec, store, signed_block, test_steps, valid=valid)
+
+                block_root = signed_block.message.hash_tree_root()
+                if valid:
+                    assert store.blocks[block_root] == signed_block.message
+                else:
+                    assert block_root not in store.blocks.values()
+            output_store_checks(spec, store, test_steps, with_viable_for_head_weights=True)
         elif event_kind == 'attestation':
             _, attestation, valid = event
             if valid is None:
                 valid = try_add_mesage(run_on_attestation, attestation)
             yield from add_attestation(spec, store, attestation, test_steps, valid=valid)
+            output_store_checks(spec, store, test_steps, with_viable_for_head_weights=True)
         elif event_kind == 'attester_slashing':
             _, attester_slashing, valid = event
             if valid is None:
                 valid = try_add_mesage(run_on_attester_slashing, attester_slashing)
             yield from add_attester_slashing(spec, store, attester_slashing, test_steps, valid=valid)
+            output_store_checks(spec, store, test_steps, with_viable_for_head_weights=True)
         else:
             raise ValueError('Unknown event ' + str(event_kind))
 

tests/generators/fork_choice_generated/scheduler.py

@@ -25,9 +25,9 @@ def __init__(self, message, is_attestation, is_from_block=False):
 
 
 class MessageScheduler:
-    def __init__(self, spec, anchor_state, anchor_block):
+    def __init__(self, spec, store):
         self.spec = spec
-        self.store = spec.get_forkchoice_store(anchor_state, anchor_block)
+        self.store = store
         self.message_queue = []
 
     def is_early_message(self, item: QueueItem) -> bool:
@@ -42,30 +42,45 @@ def drain_queue(self, ) -> list[QueueItem]:
         self.message_queue.clear()
         return messages
 
-    def process_queue(self):
+    def process_queue(self) -> tuple[bool, list]:
+        applied_events = []
         updated = False
         for item in self.drain_queue():
             if self.is_early_message(item):
                 self.enque_message(item)
             else:
                 if item.is_attestation:
-                    self.process_attestation(item.message)
+                    if self.process_attestation(item.message):
+                        applied_events.append(('attestation', item.message, True))
                 else:
-                    updated |= self.process_block(item.message)
-        return updated
+                    updated_, events_ = self.process_block(item.message, recovery=True)
+                    if updated_:
+                        updated = True
+                        applied_events.extend(events_)
+                        assert ('block', item.message, True) in events_
+        return updated, applied_events
 
-    def purge_queue(self):
-        while self.process_queue():
-            pass
+    def purge_queue(self) -> list:
+        applied_events = []
+        while True:
+            updated, events = self.process_queue()
+            applied_events.extend(events)
+            if updated:
+                continue
+            else:
+                return applied_events
 
-    def process_tick(self, time):
+    def process_tick(self, time) -> list:
+        applied_events = []
         SECONDS_PER_SLOT = self.spec.config.SECONDS_PER_SLOT
         assert time >= self.store.time
         tick_slot = (time - self.store.genesis_time) // SECONDS_PER_SLOT
         while self.spec.get_current_slot(self.store) < tick_slot:
             previous_time = self.store.genesis_time + (self.spec.get_current_slot(self.store) + 1) * SECONDS_PER_SLOT
             self.spec.on_tick(self.store, previous_time)
-            self.purge_queue()
+            applied_events.append(('tick', previous_time, self.spec.get_current_slot(self.store) < tick_slot))
+            applied_events.extend(self.purge_queue())
+        return applied_events
 
     def process_attestation(self, attestation, is_from_block=False):
         try:
@@ -91,16 +106,18 @@ def process_block_messages(self, signed_block):
         for attester_slashing in block.body.attester_slashings:
             self.process_slashing(attester_slashing)
 
-    def process_block(self, signed_block):
+    def process_block(self, signed_block, recovery=False) -> tuple[bool, list]:
+        applied_events = []
         try:
             self.spec.on_block(self.store, signed_block)
             valid = True
+            applied_events.append(('block', signed_block, recovery))
         except AssertionError:
             item = QueueItem(signed_block, False)
             if self.is_early_message(item):
                 self.enque_message(item)
             valid = False
         if valid:
-            self.purge_queue()
+            applied_events.extend(self.purge_queue())
             self.process_block_messages(signed_block)
-        return valid
+        return valid, applied_events

tests/generators/fork_choice_generated/standard/test_gen.yaml

@@ -14,8 +14,8 @@ shuffling_test:
   test_type: block_tree
   instances: standard/block_tree_other.yaml # 8
   seed: 6673
-  nr_variations: 1
-  nr_mutations: 20
+  nr_variations: 4
+  nr_mutations: 63
 attester_slashing_test:
   test_type: block_tree
   instances: standard/block_tree_other.yaml # 8

tests/generators/fork_choice_generated/test_provider.py

@@ -15,7 +15,7 @@
 from instantiators.block_cover import yield_block_cover_test_case, yield_block_cover_test_data
 from instantiators.block_tree import yield_block_tree_test_case, yield_block_tree_test_data
 from instantiators.helpers import FCTestData, make_events, yield_fork_choice_test_events, filter_out_duplicate_messages
-from mutation_operators import mutate_test_vector
+from instantiators.mutation_operators import MutationOps
 import random
 
 
@@ -52,13 +52,19 @@ def mutation_case_fn(self):
         test_data = list(self.call_instantiator(test_data_only=True))[0][2]
         events = make_events(spec, test_data)
         store = spec.get_forkchoice_store(test_data.anchor_state, test_data.anchor_block)
+        start_time = store.time
+        seconds_per_slot = spec.config.SECONDS_PER_SLOT
+
         if mut_seed is None:
             return (spec_test(yield_fork_choice_test_events))(
                 spec, store, test_data, events, self.debug, generator_mode=True, bls_active=self.bls_active)
         else:
             test_vector = events_to_test_vector(events)
-            mutated_vector, = list(mutate_test_vector(random.Random(mut_seed), test_vector, 1, debug=self.debug))
-            #mutated_tc.meta['mutation_seed'] = mut_seed
+            mops = MutationOps(start_time, seconds_per_slot)
+            mutated_vector, mutations = mops.rand_mutations(test_vector, 4, random.Random(mut_seed))
+
+            test_data.meta['mut_seed'] = mut_seed
+            test_data.meta['mutations'] = mutations
 
             mutated_events = test_vector_to_events(mutated_vector)
 
@@ -128,6 +134,8 @@ def test_vector_to_events(test_vector):
 
 @filter_out_duplicate_messages
 def yield_test_parts(spec, store, test_data: FCTestData, events):
+        record_recovery_messages = True
+
         for k,v in test_data.meta.items():
             yield k, 'meta', v
 
@@ -136,35 +144,88 @@ def yield_test_parts(spec, store, test_data: FCTestData, events):
 
         for message in test_data.blocks:
             block = message.payload
-            yield get_block_file_name(block), block.encode_bytes()
+            yield get_block_file_name(block), block
 
         for message in test_data.atts:
             attestation = message.payload
-            yield get_attestation_file_name(attestation), attestation.encode_bytes()
+            yield get_attestation_file_name(attestation), attestation
 
         for message in test_data.slashings:
             attester_slashing = message.payload
-            yield get_attester_slashing_file_name(attester_slashing), attester_slashing.encode_bytes()
+            yield get_attester_slashing_file_name(attester_slashing), attester_slashing
 
-        anchor_state = test_data.anchor_state
-        anchor_block = test_data.anchor_block
         test_steps = []
-        scheduler = MessageScheduler(spec, anchor_state, anchor_block)
+        scheduler = MessageScheduler(spec, store)
+
+        # record first tick
+        on_tick_and_append_step(spec, store, store.time, test_steps)
 
         for (kind, data, _) in events:
             if kind == 'tick':
                 time = data
                 if time > store.time:
-                    scheduler.process_tick(time)
-                    on_tick_and_append_step(spec, store, time, test_steps)
-
-                    # output checks after applying buffered messages, since they affect store state
-                    output_store_checks(spec, store, test_steps)
+                    applied_events = scheduler.process_tick(time)
+                    if record_recovery_messages:
+                        for (event_kind, event_data, recovery) in applied_events:
+                            if event_kind == 'tick':
+                                test_steps.append({'tick': int(event_data)})
+                            elif event_kind == 'block':
+                                assert recovery
+                                _block_id = get_block_file_name(event_data)
+                                print('recovered block', _block_id)
+                                test_steps.append({'block': _block_id, 'valid': True, 'recovery': True})
+                            elif event_kind == 'attestation':
+                                assert recovery
+                                _attestation_id = get_attestation_file_name(event_data)
+                                if _attestation_id not in test_data.atts:
+                                    yield _attestation_id, event_data
+                                print('recovered attestation', _attestation_id)
+                                test_steps.append({'attestation': _attestation_id, 'valid': True, 'recovery': True})
+                            else:
+                                assert False
+                    else:
+                        assert False
+                    if time > store.time:
+                        # inside a slot
+                        on_tick_and_append_step(spec, store, time, test_steps)
+                    else:
+                        assert time == store.time
+                        output_store_checks(spec, store, test_steps)
             elif kind == 'block':
                 block = data
                 block_id = get_block_file_name(block)
-                valid = scheduler.process_block(block)
-                test_steps.append({'block': block_id, 'valid': valid})
+                valid, applied_events = scheduler.process_block(block)
+                if record_recovery_messages:
+                    if valid:
+                        for (event_kind, event_data, recovery) in applied_events:
+                            if event_kind == 'block':
+                                _block_id = get_block_file_name(event_data)
+                                if recovery:
+                                    print('recovered block', _block_id)
+                                    test_steps.append({'block': _block_id, 'valid': True, 'recovery': True})
+                                else:
+                                    test_steps.append({'block': _block_id, 'valid': True})
+                            elif event_kind == 'attestation':
+                                _attestation_id = get_attestation_file_name(event_data)
+                                if recovery:
+                                    print('recovered attestation', _attestation_id)
+                                    if _attestation_id not in test_data.atts:
+                                        yield _attestation_id, event_data
+                                    test_steps.append({'attestation': _attestation_id, 'valid': True, 'recovery': True})
+                                else:
+                                    assert False
+                                    test_steps.append({'attestation': _attestation_id, 'valid': True})
+                            else:
+                                assert False
+                    else:
+                        assert len(applied_events) == 0
+                        test_steps.append({'block': block_id, 'valid': valid})
+                else:
+                    assert False
+                    test_steps.append({'block': block_id, 'valid': valid})
+                block_root = block.message.hash_tree_root()
+                assert valid == (block_root in store.blocks)
+
                 output_store_checks(spec, store, test_steps)
             elif kind == 'attestation':
                 attestation = data
@@ -181,8 +242,7 @@ def yield_test_parts(spec, store, test_data: FCTestData, events):
             else:
                 raise ValueError(f'not implemented {kind}')
         next_slot_time = store.genesis_time + (spec.get_current_slot(store) + 1) * spec.config.SECONDS_PER_SLOT
-        # on_tick_and_append_step(spec, store, next_slot_time, test_steps, checks_with_viable_for_head_weights=True)
-        on_tick_and_append_step(spec, store, next_slot_time, test_steps)
+        on_tick_and_append_step(spec, store, next_slot_time, test_steps, checks_with_viable_for_head_weights=True)
 
         yield 'steps', test_steps