Skip to content

Feature/trace update #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tvorogme merged 3 commits into from
Aug 21, 2025
Merged

Feature/trace update #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37b71cf
Fix trace multi-run cache, add account color schema feature
tvorogme Aug 21, 2025
272b38d
Add warn_if in color scheme
tvorogme Aug 21, 2025
0ea2b96
Add allow_extra_tx for traces
tvorogme Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion diff_colored.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@
"out_msgs": "alarm"
},
"total_fees": {
"grams": "alarm",
"grams": {
"amount": {
"value": {
"warn_if": "abs_diff == -1 else skip"
}
}
},
"other": "alarm"
},
"state_update": "alarm",
Expand Down Expand Up @@ -65,5 +71,11 @@
"aborted": "alarm",
"destroyed": "alarm"
}
},
"account": {},
"trace": {
"allow_extra_tx": [
{"op": "0x1", "with_bounced": true}
]
}
}
137 changes: 114 additions & 23 deletions src/tonemuso/diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@

from deepdiff import DeepDiff
from tonpy.autogen.block import Transaction, ShardAccount
from loguru import logger
import json

import re

def make_json_dumpable(obj):
"""
Expand Down Expand Up @@ -73,37 +72,129 @@ def unpack_path(path):

def get_colored_diff(diff, color_schema, root='transaction'):
max_level = 'skip'
# Prepare quick lookup for numeric value changes by affected path string
diff_dict = diff.to_dict() if hasattr(diff, 'to_dict') else {}
values_changed = diff_dict.get('values_changed', {}) if isinstance(diff_dict, dict) else {}

log = {
'affected_paths': list(diff.affected_paths),
'colors': {}
}
for path in diff.affected_paths:
path = unpack_path(path)
for affected_path_str in diff.affected_paths:
path_list = unpack_path(affected_path_str)

if root not in color_schema:
raise ValueError(f"{root} is not in color schema")

current_root = color_schema[root]
for p in path:
traversed_ok = True
for i, p in enumerate(path_list):
if p not in current_root:
logger.warning(f"[COLOR_SCHEMA] {p} is not in list, full path: {path}, but affected")
logger.warning(f"[COLOR_SCHEMA: {root}] {p} is not in list, full path: {path_list}, but affected")
max_level = 'alarm'
log['colors']['__'.join(path)] = 'alarm'
log['colors']['__'.join(path_list)] = 'alarm'
traversed_ok = False
break
else:
if isinstance(current_root[p], str):
level = current_root[p]
if level == 'warn':
if max_level != 'alarm':
max_level = 'warn'
log['colors']['__'.join(path)] = 'warn'
elif level == 'alarm':
max_level = 'alarm'
log['colors']['__'.join(path)] = 'alarm'
elif level == 'skip':
pass
else:
logger.warning(f"[COLOR_SCHEMA] {p} has invalid color level")
break
node = current_root[p]
# Leaf processing
if isinstance(node, str):
level = node
if level == 'warn':
if max_level != 'alarm':
max_level = 'warn'
log['colors']['__'.join(path_list)] = 'warn'
elif level == 'alarm':
max_level = 'alarm'
log['colors']['__'.join(path_list)] = 'alarm'
elif level == 'skip':
log['colors']['__'.join(path_list)] = 'skip'
else:
current_root = current_root[p]
logger.warning(f"[COLOR_SCHEMA] {p} has invalid color level")
break
elif isinstance(node, dict) and ('warn_if' in node):
# Evaluate warn_if rule only for numeric value changes
change = values_changed.get(affected_path_str)
try:
old_v = change.get('old_value') if isinstance(change, dict) else None
new_v = change.get('new_value') if isinstance(change, dict) else None
except Exception:
old_v, new_v = None, None
applied = False
diff_v = None
# Precompute diff if both are ints
if isinstance(old_v, int) and isinstance(new_v, int):
diff_v = new_v - old_v
else:
logger.warning(f"[COLOR_SCHEMA] {p} has non-numeric value change, but warn_if rule is defined")
log['colors']['__'.join(path_list)] = 'alarm'
break

rule = node.get('warn_if', '')
# Conservative parser: supports patterns like "diff > 3 else skip", "abs_diff >= 5 else skip" or "new_value > 100 else warn"
m = re.match(r"\s*(diff|abs_diff|new_value)\s*(==|!=|>=|<=|>|<)\s*(-?\d+)\s*else\s*(skip|warn|alarm)\s*$", str(rule))
if m:
lhs, op, num_s, else_action = m.groups()
num = int(num_s)
# Determine comparable value based on lhs
can_compare = False
val = None
if lhs == 'diff' and isinstance(old_v, int) and isinstance(new_v, int):
val = diff_v
can_compare = True
elif lhs == 'abs_diff' and isinstance(old_v, int) and isinstance(new_v, int):
val = abs(diff_v)
can_compare = True
elif lhs == 'new_value' and isinstance(new_v, int):
val = new_v
can_compare = True
if can_compare:
cond = False
if op == '==':
cond = (val == num)
elif op == '!=':
cond = (val != num)
elif op == '>':
cond = (val > num)
elif op == '<':
cond = (val < num)
elif op == '>=':
cond = (val >= num)
elif op == '<=':
cond = (val <= num)
else:
logger.warning(f"[COLOR_SCHEMA] {p} has invalid operator")
log['colors']['__'.join(path_list)] = 'alarm'
break
if cond:
# warn
if max_level != 'alarm':
max_level = 'warn'
log['colors']['__'.join(path_list)] = 'warn'
else:
# else action
if else_action == 'alarm':
max_level = 'alarm'
log['colors']['__'.join(path_list)] = 'alarm'
elif else_action == 'warn':
if max_level != 'alarm':
max_level = 'warn'
log['colors']['__'.join(path_list)] = 'warn'
elif else_action == 'skip':
log['colors']['__'.join(path_list)] = 'skip'
else:
logger.warning(f"[COLOR_SCHEMA] {p} has invalid else action")
log['colors']['__'.join(path_list)] = 'alarm'
break
applied = True
if not applied:
logger.warning(f"[COLOR_SCHEMA] {p} rule is not applied, but affected")
log['colors']['__'.join(path_list)] = 'alarm'
break
break
else:
# Traverse deeper
current_root = node
# end for path components

return max_level, log

Expand Down
69 changes: 67 additions & 2 deletions src/tonemuso/emulation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,14 +195,22 @@ def emulate_tx_step(block: Dict[str, Any],

# Always include secondary emulator info if available
account_diff_dict = None
account_color_level = None
account_color_log = None
unchanged_emulator_tx_hash = None
sa_diff = None
try:
if em2 is not None:
unchanged_emulator_tx_hash = em2.transaction.get_hash() if em2.transaction is not None else None
sa_diff = get_shard_account_diff(em.account.to_cell(), em2.account.to_cell())
account_diff_dict = {'data': make_json_dumpable(sa_diff.to_dict()),
'account_emulator_tx_hash_match': unchanged_emulator_tx_hash == tx[
'tx'].get_hash()}
'account_emulator_tx_hash_match': unchanged_emulator_tx_hash == tx['tx'].get_hash()}
# If color schema contains account root, evaluate colored diff for account as well
if color_schema is not None and isinstance(color_schema, dict):
try:
account_color_level, account_color_log = get_colored_diff(sa_diff, color_schema, root='account')
except Exception:
account_color_level, account_color_log = None, None
except Exception as ee:
logger.error(f"UNCHANGED EMULATOR ERROR: {ee}")

Expand Down Expand Up @@ -237,6 +245,9 @@ def emulate_tx_step(block: Dict[str, Any],
'fail_reason': "color_schema_alarm"}
if account_diff_dict is not None:
err_obj['account_diff'] = account_diff_dict
if account_color_log is not None:
err_obj['account_color_schema_log'] = account_color_log
err_obj['account_color_level'] = account_color_level
if unchanged_emulator_tx_hash is not None:
err_obj['unchanged_emulator_tx_hash'] = unchanged_emulator_tx_hash
out.append(err_obj)
Expand All @@ -247,10 +258,64 @@ def emulate_tx_step(block: Dict[str, Any],
warn_obj = {'mode': 'warning'}
if account_diff_dict is not None:
warn_obj['account_diff'] = account_diff_dict
if account_color_log is not None:
warn_obj['account_color_schema_log'] = account_color_log
warn_obj['account_color_level'] = account_color_level
if unchanged_emulator_tx_hash is not None:
warn_obj['unchanged_emulator_tx_hash'] = unchanged_emulator_tx_hash
out.append(warn_obj)

# If transaction comparison produced no issues, still perform colored check for account diff (if em2 present)
if color_schema is not None and em2 is not None:
try:
# Compute account diff between after-states of em1 and em2
sa_diff2 = get_shard_account_diff(em.account.to_cell(), em2.account.to_cell())
if isinstance(color_schema, dict):
root_key2 = 'account' if 'account' in color_schema else ('transaction' if 'transaction' in color_schema else None)
if root_key2 is not None:
try:
acc_level2, acc_log2 = get_colored_diff(sa_diff2, color_schema, root=root_key2)
except Exception:
acc_level2, acc_log2 = None, None
if acc_level2 == 'alarm' and go_as_success:
go_as_success = False
# Build error object similar to tx colored alarm
err_obj = {
'mode': 'error',
'fail_reason': 'account_color_schema_alarm',
'expected': tx['tx'].get_hash(),
'got': em.transaction.get_hash() if em.transaction is not None else None,
'account_diff': {
'data': make_json_dumpable(sa_diff2.to_dict()),
'account_emulator_tx_hash_match': (em2.transaction.get_hash() if em2.transaction is not None else None) == tx['tx'].get_hash()
},
'account_color_schema_log': acc_log2,
'account_color_level': acc_level2
}
try:
err_obj['unchanged_emulator_tx_hash'] = em2.transaction.get_hash() if em2.transaction is not None else None
except Exception:
pass
out.append(err_obj)
elif acc_level2 == 'warn' and go_as_success:
go_as_success = False
warn_obj = {
'mode': 'warning',
'account_diff': {
'data': make_json_dumpable(sa_diff2.to_dict()),
'account_emulator_tx_hash_match': (em2.transaction.get_hash() if em2.transaction is not None else None) == tx['tx'].get_hash()
},
'account_color_schema_log': acc_log2,
'account_color_level': acc_level2
}
try:
warn_obj['unchanged_emulator_tx_hash'] = em2.transaction.get_hash() if em2.transaction is not None else None
except Exception:
pass
out.append(warn_obj)
except Exception as e:
logger.error(f"ACCOUNT COLORED CHECK ERROR: {e}")

# Update account states for next transaction
try:
new_state_em1 = em.account.to_cell()
Expand Down
36 changes: 25 additions & 11 deletions src/tonemuso/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@

from tonpy.blockscanner.blockscanner import *
from tonpy import begin_cell
from collections import Counter
from collections import Counter, OrderedDict, defaultdict
import json
import os
import argparse
import base64
import requests
from multiprocessing import get_context

from tonpy.utils.shard_account import get_empty_shard_account
from tqdm import tqdm
from tonemuso.diff import get_diff, get_colored_diff, make_json_dumpable, get_shard_account_diff
from queue import Empty as QueueEmpty
from tonemuso.trace_runner import TraceOrderedRunner
from tonemuso.utils import b64_to_hex, hex_to_b64
from tonpy.autogen.block import Block, BlockInfo
from tonpy.autogen.block import Block, BlockInfo, ShardAccount

LOGLEVEL = int(os.getenv("EMUSO_LOGLEVEL", 1))
COLOR_SCHEMA = str(os.getenv("COLOR_SCHEMA_PATH", ''))
Expand Down Expand Up @@ -463,17 +465,22 @@ def _build_preindex(raw_chunks):
blocks = {}
tx_index = {}
before_states = {}
default_initial_state = {}
# Per-block, per-account default initial states
default_initial_state = defaultdict(OrderedDict)
for block, initial_account_state, txs in raw_chunks:
blk = block['block_id']
block_key = (blk.id.workchain, blk.id.shard, blk.id.seqno, blk.root_hash)
blocks[block_key] = block
if block_key not in default_initial_state:
default_initial_state[block_key] = initial_account_state
for tx in txs:
try:
txh = tx['tx'].get_hash().upper()
tx_index[txh] = (block_key, tx)
# compute account address for this tx and assign default state
tx_tlb = Transaction().cell_unpack(tx['tx'], True)
account_address = int(tx_tlb.account_addr, 2)
account_addr = Address(f"{block_key[0]}:{hex(account_address).upper()[2:].zfill(64)}")
if account_addr not in default_initial_state[block_key]:
default_initial_state[block_key][account_addr] = initial_account_state
if 'before_state_em1' in tx and tx['before_state_em1'] is not None:
before_states[txh] = tx['before_state_em1']
except Exception:
Expand Down Expand Up @@ -534,6 +541,7 @@ def _build_preindex(raw_chunks):
total_new = 0
total_missed = 0
trace_success_count = 0
trace_warning_count = 0
trace_unsuccess_count = 0

def _count_modes_tree(node):
Expand Down Expand Up @@ -572,16 +580,17 @@ def dfs(n):
not_presented = entry.get('not_presented') or []
total_missed += len(not_presented)

# Per-trace success means: all nodes are 'success' (no warnings/errors/new/missed) and no not_presented
if emu and c.get('warnings', 0) == 0 and c.get('unsuccess', 0) == 0 and c.get('new',
0) == 0 and c.get(
'missed', 0) == 0 and len(not_presented) == 0:
# Per-trace status from entry.final_status
fstatus = (entry.get('final_status') or '').lower()
if fstatus == 'success':
trace_success_count += 1
elif fstatus == 'warning' or fstatus == 'warnings':
trace_warning_count += 1
else:
trace_unsuccess_count += 1

logger.warning(
f"Final emulator status: {total_success} success, {total_unsuccess} unsuccess, {total_warnings} warnings, {total_new} new, {total_missed} missed; traces: {trace_success_count} success, {trace_unsuccess_count} unsuccess")
f"Final emulator status: {total_success} success, {total_unsuccess} unsuccess, {total_warnings} warnings, {total_new} new, {total_missed} missed; traces: {trace_success_count} success, {trace_warning_count} warnings, {trace_unsuccess_count} unsuccess")
except Exception as e:
logger.error(f"Failed to compute final status from traces emulation: {e}")

Expand Down Expand Up @@ -786,8 +795,13 @@ def dfs(n):
# Add also transactions from original trace that are not presented in emulated tree
not_presented = trace_entries[0].get('not_presented') or []
missed_cnt += len(not_presented)
# Per-trace status summary (only one trace here)
status = (trace_entries[0].get('final_status') or '').lower()
t_succ = 1 if status == 'success' else 0
t_warn = 1 if status == 'warning' or status == 'warnings' else 0
t_uns = 1 if status not in ('success', 'warning', 'warnings') else 0
logger.warning(
f"Final emulator status: {success} success, {unsuccess} unsuccess, {warnings} warnings, {new_cnt} new, {missed_cnt} missed")
f"Final emulator status: {success} success, {unsuccess} unsuccess, {warnings} warnings, {new_cnt} new, {missed_cnt} missed; traces: {t_succ} success, {t_warn} warnings, {t_uns} unsuccess")
# Skip the default final log below by returning early
return
except Exception as e:
Expand Down
Loading
Loading