From 789e107a8d979c03418da6379eb747fa8911655d Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 12:32:20 -0500 Subject: [PATCH 01/15] fix unicode patches during mutation --- slither/core/source_mapping/source_mapping.py | 2 +- slither/tools/mutator/mutators/AOR.py | 2 +- slither/tools/mutator/mutators/ASOR.py | 2 +- slither/tools/mutator/mutators/BOR.py | 2 +- slither/tools/mutator/mutators/CR.py | 2 +- slither/tools/mutator/mutators/FHR.py | 2 +- slither/tools/mutator/mutators/LIR.py | 4 ++-- slither/tools/mutator/mutators/LOR.py | 2 +- slither/tools/mutator/mutators/MIA.py | 2 +- slither/tools/mutator/mutators/MVIE.py | 4 ++-- slither/tools/mutator/mutators/MVIV.py | 4 ++-- slither/tools/mutator/mutators/MWA.py | 2 +- slither/tools/mutator/mutators/ROR.py | 2 +- slither/tools/mutator/mutators/RR.py | 10 ++++++---- slither/tools/mutator/mutators/SBR.py | 4 ++-- slither/tools/mutator/mutators/UOR.py | 2 +- 16 files changed, 25 insertions(+), 23 deletions(-) diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index 355aa5538..61545c8f5 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -79,7 +79,7 @@ def content(self) -> str: # If the compilation unit was not initialized, it means that the set_offset was never called # on the corresponding object, which should not happen assert self.compilation_unit - return self.compilation_unit.core.source_code[self.filename.absolute][self.start : self.end] + return self.compilation_unit.core.source_code[self.filename.absolute].encode("utf-8")[self.start : self.end].decode("utf-8") @property def content_hash(self) -> str: diff --git a/slither/tools/mutator/mutators/AOR.py b/slither/tools/mutator/mutators/AOR.py index f86e7d3d4..d74c57371 100644 --- a/slither/tools/mutator/mutators/AOR.py +++ b/slither/tools/mutator/mutators/AOR.py @@ -74,7 +74,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true diff --git a/slither/tools/mutator/mutators/ASOR.py b/slither/tools/mutator/mutators/ASOR.py index 2ff403b38..18beb63c5 100644 --- a/slither/tools/mutator/mutators/ASOR.py +++ b/slither/tools/mutator/mutators/ASOR.py @@ -48,7 +48,7 @@ def _mutate(self) -> Dict: if op != ir.expression: start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true diff --git a/slither/tools/mutator/mutators/BOR.py b/slither/tools/mutator/mutators/BOR.py index a8720a4b6..fed5319ec 100644 --- a/slither/tools/mutator/mutators/BOR.py +++ b/slither/tools/mutator/mutators/BOR.py @@ -31,7 +31,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true diff --git a/slither/tools/mutator/mutators/CR.py b/slither/tools/mutator/mutators/CR.py index ebf93bf18..2c1ebc4d9 100644 --- a/slither/tools/mutator/mutators/CR.py +++ b/slither/tools/mutator/mutators/CR.py @@ -23,7 +23,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: new_str = "//" + old_str diff --git a/slither/tools/mutator/mutators/FHR.py b/slither/tools/mutator/mutators/FHR.py index 028c1916c..c874b489d 100644 --- a/slither/tools/mutator/mutators/FHR.py +++ b/slither/tools/mutator/mutators/FHR.py @@ -22,7 +22,7 @@ def _mutate(self) -> Dict: for function in self.contract.functions_and_modifiers_declared: start = function.source_mapping.start stop = start + function.source_mapping.content.find("{") - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = function.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in function_header_replacements: diff --git a/slither/tools/mutator/mutators/LIR.py b/slither/tools/mutator/mutators/LIR.py index fc621829f..d2ad3e5b3 100644 --- a/slither/tools/mutator/mutators/LIR.py +++ b/slither/tools/mutator/mutators/LIR.py @@ -36,7 +36,7 @@ def _mutate(self) -> Dict: # pylint: disable=too-many-branches # Get the string start = variable.source_mapping.start stop = start + variable.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in literal_replacements: @@ -67,7 +67,7 @@ def _mutate(self) -> Dict: # pylint: disable=too-many-branches literal_replacements.append("-1") start = variable.source_mapping.start stop = start + variable.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for new_value in literal_replacements: diff --git a/slither/tools/mutator/mutators/LOR.py b/slither/tools/mutator/mutators/LOR.py index 2d1535b1a..64285ed77 100644 --- a/slither/tools/mutator/mutators/LOR.py +++ b/slither/tools/mutator/mutators/LOR.py @@ -29,7 +29,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true diff --git a/slither/tools/mutator/mutators/MIA.py b/slither/tools/mutator/mutators/MIA.py index f29569f63..07946dc55 100644 --- a/slither/tools/mutator/mutators/MIA.py +++ b/slither/tools/mutator/mutators/MIA.py @@ -17,7 +17,7 @@ def _mutate(self) -> Dict: # Get the string start = node.expression.source_mapping.start stop = start + node.expression.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true and false diff --git a/slither/tools/mutator/mutators/MVIE.py b/slither/tools/mutator/mutators/MVIE.py index ce51792ff..983b7cf15 100644 --- a/slither/tools/mutator/mutators/MVIE.py +++ b/slither/tools/mutator/mutators/MVIE.py @@ -24,7 +24,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: @@ -44,7 +44,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: diff --git a/slither/tools/mutator/mutators/MVIV.py b/slither/tools/mutator/mutators/MVIV.py index f9e51c553..d51414a7d 100644 --- a/slither/tools/mutator/mutators/MVIV.py +++ b/slither/tools/mutator/mutators/MVIV.py @@ -24,7 +24,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: @@ -43,7 +43,7 @@ def _mutate(self) -> Dict: if variable.initialized and isinstance(variable.expression, Literal): start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: diff --git a/slither/tools/mutator/mutators/MWA.py b/slither/tools/mutator/mutators/MWA.py index 9682f10ca..1ba56b3e1 100644 --- a/slither/tools/mutator/mutators/MWA.py +++ b/slither/tools/mutator/mutators/MWA.py @@ -18,7 +18,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: if not isinstance(node.expression, UnaryOperation): diff --git a/slither/tools/mutator/mutators/ROR.py b/slither/tools/mutator/mutators/ROR.py index 9daae0663..aea209f4b 100644 --- a/slither/tools/mutator/mutators/ROR.py +++ b/slither/tools/mutator/mutators/ROR.py @@ -36,7 +36,7 @@ def _mutate(self) -> Dict: # Get the string start = ir.expression.source_mapping.start stop = start + ir.expression.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true diff --git a/slither/tools/mutator/mutators/RR.py b/slither/tools/mutator/mutators/RR.py index ba76d657f..d37f2e9b6 100644 --- a/slither/tools/mutator/mutators/RR.py +++ b/slither/tools/mutator/mutators/RR.py @@ -21,9 +21,11 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] - line_no = node.source_mapping.lines - if not line_no[0] in self.dont_mutate_line: + old_str = node.source_mapping.content + line_no = node.source_mapping.lines[0] + if not line_no in self.dont_mutate_line: + if node.type == NodeType.RETURN and not old_str.lstrip().startswith("return"): + continue # skip the return declarations in fn signatures if not old_str.lstrip().startswith("revert"): new_str = "revert()" create_patch_with_line( @@ -33,6 +35,6 @@ def _mutate(self) -> Dict: stop, old_str, new_str, - line_no[0], + line_no, ) return result diff --git a/slither/tools/mutator/mutators/SBR.py b/slither/tools/mutator/mutators/SBR.py index efbda4877..18cea69d9 100644 --- a/slither/tools/mutator/mutators/SBR.py +++ b/slither/tools/mutator/mutators/SBR.py @@ -64,7 +64,7 @@ def _mutate(self) -> Dict: # Get the string start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in solidity_rules: @@ -89,7 +89,7 @@ def _mutate(self) -> Dict: if node: start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in solidity_rules: diff --git a/slither/tools/mutator/mutators/UOR.py b/slither/tools/mutator/mutators/UOR.py index f427c2fbf..bd22f1f37 100644 --- a/slither/tools/mutator/mutators/UOR.py +++ b/slither/tools/mutator/mutators/UOR.py @@ -29,7 +29,7 @@ def _mutate(self) -> Dict: continue start = node.source_mapping.start stop = start + node.source_mapping.length - old_str = self.in_file_str[start:stop] + old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: if ( From e31a88196d3a2fe6f33ee88cb606a61163981586 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 12:34:33 -0500 Subject: [PATCH 02/15] fix mutation logs & RR mutator --- slither/tools/mutator/__main__.py | 15 +++++++++++---- slither/tools/mutator/mutators/RR.py | 3 ++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/slither/tools/mutator/__main__.py b/slither/tools/mutator/__main__.py index d97806e28..f71ef26b5 100644 --- a/slither/tools/mutator/__main__.py +++ b/slither/tools/mutator/__main__.py @@ -167,7 +167,10 @@ def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too # get all the contracts as a list from given codebase sol_file_list: List[str] = get_sol_file_list(Path(args.codebase), paths_to_ignore_list) - logger.info(blue("Preparing to mutate files:\n- " + "\n- ".join(sol_file_list))) + if not contract_names: + logger.info(blue("Preparing to mutate files:\n- " + "\n- ".join(sol_file_list))) + else: + logger.info(blue("Preparing to mutate contracts:\n- " + "\n- ".join(contract_names))) # folder where backup files and uncaught mutants are saved if output_dir is None: @@ -240,7 +243,8 @@ def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too # perform mutations on {target_contract} in file {file_name} # setup placeholder val to signal whether we need to skip if no target_contract is found - target_contract = "SLITHER_SKIP_MUTATIONS" if contract_names else "" + skip_flag = "SLITHER_SKIP_MUTATIONS" + target_contract = skip_flag if contract_names else "" try: # loop through all contracts in file_name for compilation_unit_of_main_file in sl.compilation_units: @@ -258,8 +262,7 @@ def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too ) continue - if target_contract == "SLITHER_SKIP_MUTATIONS": - logger.debug(f"Skipping mutations in {filename}") + if target_contract == skip_flag: continue # TODO: find a more specific way to omit interfaces @@ -334,6 +337,10 @@ def main() -> None: # pylint: disable=too-many-statements,too-many-branches,too # transfer and delete the backup files transfer_and_delete(files_dict) + if target_contract == skip_flag: + logger.debug(f"No target contracts found in {filename}, skipping") + continue + # log results for this file logger.info(blue(f"Done mutating {target_contract}.")) if total_mutant_counts[0] > 0: diff --git a/slither/tools/mutator/mutators/RR.py b/slither/tools/mutator/mutators/RR.py index d37f2e9b6..174daa671 100644 --- a/slither/tools/mutator/mutators/RR.py +++ b/slither/tools/mutator/mutators/RR.py @@ -3,7 +3,6 @@ from slither.tools.mutator.utils.patch import create_patch_with_line from slither.tools.mutator.mutators.abstract_mutator import AbstractMutator - class RR(AbstractMutator): # pylint: disable=too-few-public-methods NAME = "RR" HELP = "Revert Replacement" @@ -15,8 +14,10 @@ def _mutate(self) -> Dict: for node in function.nodes: if node.type not in ( NodeType.ENTRYPOINT, + NodeType.IF, NodeType.ENDIF, NodeType.ENDLOOP, + NodeType.PLACEHOLDER, ): # Get the string start = node.source_mapping.start From 8b9590dc96e12959a5c743b79af953282b127a06 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 13:26:03 -0500 Subject: [PATCH 03/15] skip assembly in BOR mutator --- slither/tools/mutator/mutators/BOR.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/slither/tools/mutator/mutators/BOR.py b/slither/tools/mutator/mutators/BOR.py index fed5319ec..d6638daf8 100644 --- a/slither/tools/mutator/mutators/BOR.py +++ b/slither/tools/mutator/mutators/BOR.py @@ -35,7 +35,10 @@ def _mutate(self) -> Dict: line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true - new_str = f"{old_str.split(ir.type.value)[0]}{op.value}{old_str.split(ir.type.value)[1]}" + halves = old_str.split(ir.type.value) + if len(halves) != 2: + continue # skip if assembly + new_str = f"{halves[0]}{op.value}{halves[1]}" create_patch_with_line( result, self.in_file, From 2295939b63103d96bab291829db7ac66d0dda59f Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 15:21:58 -0500 Subject: [PATCH 04/15] fix bugs in other mutators --- slither/tools/mutator/mutators/AOR.py | 6 ++++-- slither/tools/mutator/mutators/FHR.py | 2 +- slither/tools/mutator/mutators/MVIE.py | 4 ++-- slither/tools/mutator/mutators/ROR.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/mutators/AOR.py b/slither/tools/mutator/mutators/AOR.py index d74c57371..a9e816f9e 100644 --- a/slither/tools/mutator/mutators/AOR.py +++ b/slither/tools/mutator/mutators/AOR.py @@ -77,8 +77,10 @@ def _mutate(self) -> Dict: old_str = node.source_mapping.content line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: - # Replace the expression with true - new_str = f"{old_str.split(ir.type.value)[0]}{op.value}{old_str.split(ir.type.value)[1]}" + halves = old_str.split(ir.type.value) + if len(halves) != 2: + continue # skip if assembly + new_str = f"{halves[0]}{op.value}{halves[1]}" create_patch_with_line( result, self.in_file, diff --git a/slither/tools/mutator/mutators/FHR.py b/slither/tools/mutator/mutators/FHR.py index c874b489d..c847542ca 100644 --- a/slither/tools/mutator/mutators/FHR.py +++ b/slither/tools/mutator/mutators/FHR.py @@ -22,7 +22,7 @@ def _mutate(self) -> Dict: for function in self.contract.functions_and_modifiers_declared: start = function.source_mapping.start stop = start + function.source_mapping.content.find("{") - old_str = node.source_mapping.content + old_str = function.source_mapping.content line_no = function.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in function_header_replacements: diff --git a/slither/tools/mutator/mutators/MVIE.py b/slither/tools/mutator/mutators/MVIE.py index 983b7cf15..be0a80daf 100644 --- a/slither/tools/mutator/mutators/MVIE.py +++ b/slither/tools/mutator/mutators/MVIE.py @@ -24,7 +24,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = node.source_mapping.content + old_str = variable.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: @@ -44,7 +44,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = node.source_mapping.content + old_str = variable.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: diff --git a/slither/tools/mutator/mutators/ROR.py b/slither/tools/mutator/mutators/ROR.py index aea209f4b..8c13ce672 100644 --- a/slither/tools/mutator/mutators/ROR.py +++ b/slither/tools/mutator/mutators/ROR.py @@ -40,7 +40,7 @@ def _mutate(self) -> Dict: line_no = node.source_mapping.lines if not line_no[0] in self.dont_mutate_line: # Replace the expression with true - new_str = f"{old_str.split(ir.type.value)[0]} {op.value} {old_str.split(ir.type.value)[1]}" + new_str = f"{old_str.split(ir.type.value)[0]}{op.value}{old_str.split(ir.type.value)[1]}" create_patch_with_line( result, self.in_file, From 5b03e0a153abf66b7ede75ec92a6ef622bfb4d85 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 15:23:00 -0500 Subject: [PATCH 05/15] try/catch mutant generation --- slither/tools/mutator/mutators/abstract_mutator.py | 11 ++++++++--- .../tools/mutator/utils/testing_generated_mutant.py | 4 +--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 2e99466d9..21ce52ebe 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -6,6 +6,7 @@ from slither.formatters.utils.patches import apply_patch, create_diff from slither.tools.mutator.utils.testing_generated_mutant import test_patch from slither.core.declarations import Contract +from slither.utils.colors import red logger = logging.getLogger("Slither-Mutate") @@ -70,13 +71,17 @@ def __init__( # pylint: disable=too-many-arguments @abc.abstractmethod def _mutate(self) -> Dict: - """TODO Documentation""" + """Abstract placeholder, will be overwritten by each mutator""" return {} # pylint: disable=too-many-branches def mutate(self) -> Tuple[List[int], List[int], List[int]]: - # call _mutate function from different mutators - (all_patches) = self._mutate() + all_patches: Dict = {} + try: + # call _mutate function from different mutators + (all_patches) = self._mutate() + except Exception as e: + logger.error(red("%s mutator failed in %s: %s"), self.NAME, self.contract.name, str(e)) if "patches" not in all_patches: logger.debug("No patches found by %s", self.NAME) return [0, 0, 0], [0, 0, 0], self.dont_mutate_line diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index d484ff68f..d5725485b 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -109,9 +109,7 @@ def test_patch( # pylint: disable=too-many-arguments create_mutant_file(output_folder, file, generator_name) logger.info( - red( - f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" - ) + f"[{generator_name}] Line {patch['line_number']}: '{patch['old_string']}' ==> '{patch['new_string']}' --> UNCAUGHT" ) reset_file(file) From 87fb412ea0e87dceeceb7ff8c20a76a556da363a Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 15:42:13 -0500 Subject: [PATCH 06/15] fix formatting --- slither/core/source_mapping/source_mapping.py | 6 +++++- slither/tools/mutator/mutators/AOR.py | 2 +- slither/tools/mutator/mutators/BOR.py | 2 +- slither/tools/mutator/mutators/RR.py | 7 +++++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index 61545c8f5..59d59ad11 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -79,7 +79,11 @@ def content(self) -> str: # If the compilation unit was not initialized, it means that the set_offset was never called # on the corresponding object, which should not happen assert self.compilation_unit - return self.compilation_unit.core.source_code[self.filename.absolute].encode("utf-8")[self.start : self.end].decode("utf-8") + return ( + self.compilation_unit.core.source_code[self.filename.absolute] + .encode("utf-8")[self.start : self.end] + .decode("utf-8") + ) @property def content_hash(self) -> str: diff --git a/slither/tools/mutator/mutators/AOR.py b/slither/tools/mutator/mutators/AOR.py index a9e816f9e..1e9008efe 100644 --- a/slither/tools/mutator/mutators/AOR.py +++ b/slither/tools/mutator/mutators/AOR.py @@ -79,7 +79,7 @@ def _mutate(self) -> Dict: if not line_no[0] in self.dont_mutate_line: halves = old_str.split(ir.type.value) if len(halves) != 2: - continue # skip if assembly + continue # skip if assembly new_str = f"{halves[0]}{op.value}{halves[1]}" create_patch_with_line( result, diff --git a/slither/tools/mutator/mutators/BOR.py b/slither/tools/mutator/mutators/BOR.py index d6638daf8..2926cf1c0 100644 --- a/slither/tools/mutator/mutators/BOR.py +++ b/slither/tools/mutator/mutators/BOR.py @@ -37,7 +37,7 @@ def _mutate(self) -> Dict: # Replace the expression with true halves = old_str.split(ir.type.value) if len(halves) != 2: - continue # skip if assembly + continue # skip if assembly new_str = f"{halves[0]}{op.value}{halves[1]}" create_patch_with_line( result, diff --git a/slither/tools/mutator/mutators/RR.py b/slither/tools/mutator/mutators/RR.py index 174daa671..1bcc12e40 100644 --- a/slither/tools/mutator/mutators/RR.py +++ b/slither/tools/mutator/mutators/RR.py @@ -3,6 +3,7 @@ from slither.tools.mutator.utils.patch import create_patch_with_line from slither.tools.mutator.mutators.abstract_mutator import AbstractMutator + class RR(AbstractMutator): # pylint: disable=too-few-public-methods NAME = "RR" HELP = "Revert Replacement" @@ -25,8 +26,10 @@ def _mutate(self) -> Dict: old_str = node.source_mapping.content line_no = node.source_mapping.lines[0] if not line_no in self.dont_mutate_line: - if node.type == NodeType.RETURN and not old_str.lstrip().startswith("return"): - continue # skip the return declarations in fn signatures + if node.type == NodeType.RETURN and not old_str.lstrip().startswith( + "return" + ): + continue # skip the return declarations in fn signatures if not old_str.lstrip().startswith("revert"): new_str = "revert()" create_patch_with_line( From af8e5cfb05d28a13669e54c9c118fb77c75daf98 Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 27 Jan 2025 15:52:29 -0500 Subject: [PATCH 07/15] bugfixes & fix pylint --- slither/tools/mutator/mutators/LIR.py | 4 ++-- slither/tools/mutator/mutators/MVIV.py | 4 ++-- slither/tools/mutator/mutators/abstract_mutator.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/slither/tools/mutator/mutators/LIR.py b/slither/tools/mutator/mutators/LIR.py index d2ad3e5b3..46d816e26 100644 --- a/slither/tools/mutator/mutators/LIR.py +++ b/slither/tools/mutator/mutators/LIR.py @@ -36,7 +36,7 @@ def _mutate(self) -> Dict: # pylint: disable=too-many-branches # Get the string start = variable.source_mapping.start stop = start + variable.source_mapping.length - old_str = node.source_mapping.content + old_str = variable.source_mapping.content line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for value in literal_replacements: @@ -67,7 +67,7 @@ def _mutate(self) -> Dict: # pylint: disable=too-many-branches literal_replacements.append("-1") start = variable.source_mapping.start stop = start + variable.source_mapping.length - old_str = node.source_mapping.content + old_str = variable.source_mapping.content line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: for new_value in literal_replacements: diff --git a/slither/tools/mutator/mutators/MVIV.py b/slither/tools/mutator/mutators/MVIV.py index d51414a7d..1e14d1177 100644 --- a/slither/tools/mutator/mutators/MVIV.py +++ b/slither/tools/mutator/mutators/MVIV.py @@ -24,7 +24,7 @@ def _mutate(self) -> Dict: # Get the string start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = node.source_mapping.content + old_str = variable.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.node_initialization.source_mapping.lines if not line_no[0] in self.dont_mutate_line: @@ -43,7 +43,7 @@ def _mutate(self) -> Dict: if variable.initialized and isinstance(variable.expression, Literal): start = variable.source_mapping.start stop = variable.expression.source_mapping.start - old_str = node.source_mapping.content + old_str = variable.source_mapping.content new_str = old_str[: old_str.find("=")] line_no = variable.source_mapping.lines if not line_no[0] in self.dont_mutate_line: diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 21ce52ebe..1c7274407 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -77,6 +77,7 @@ def _mutate(self) -> Dict: # pylint: disable=too-many-branches def mutate(self) -> Tuple[List[int], List[int], List[int]]: all_patches: Dict = {} + # pylint: disable=broad-exception-caught try: # call _mutate function from different mutators (all_patches) = self._mutate() From 9fbb9aa4c519391667b610091cab17ad10af7724 Mon Sep 17 00:00:00 2001 From: bohendo Date: Wed, 5 Feb 2025 15:20:59 -0500 Subject: [PATCH 08/15] revert byte conversion in Source.content --- slither/core/source_mapping/source_mapping.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index 59d59ad11..355aa5538 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -79,11 +79,7 @@ def content(self) -> str: # If the compilation unit was not initialized, it means that the set_offset was never called # on the corresponding object, which should not happen assert self.compilation_unit - return ( - self.compilation_unit.core.source_code[self.filename.absolute] - .encode("utf-8")[self.start : self.end] - .decode("utf-8") - ) + return self.compilation_unit.core.source_code[self.filename.absolute][self.start : self.end] @property def content_hash(self) -> str: From 84011e4a52b2c465b2d383b93ce9dbbd9c83d222 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 7 Feb 2025 12:27:28 -0500 Subject: [PATCH 09/15] fix source_mapping.content to handle byte offsets --- slither/core/source_mapping/source_mapping.py | 11 ++++++++-- slither/tools/documentation/__main__.py | 22 +++++++++---------- slither/tools/flattening/flattening.py | 7 ++++++ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index 355aa5538..afa8e83a9 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -73,13 +73,20 @@ def content(self) -> str: """ Return the txt content of the Source - Returns: + Use this property instead of eg source_code[start:end] + Above will return incorrect content if source_code contains any unicode + because self.start and self.end are byte offsets, not char offsets + Returns: str """ # If the compilation unit was not initialized, it means that the set_offset was never called # on the corresponding object, which should not happen assert self.compilation_unit - return self.compilation_unit.core.source_code[self.filename.absolute][self.start : self.end] + return ( + self.compilation_unit.core.source_code[self.filename.absolute] + .encode("utf-8")[self.start : self.end] + .decode("utf-8") + ) @property def content_hash(self) -> str: diff --git a/slither/tools/documentation/__main__.py b/slither/tools/documentation/__main__.py index 0244dd6c6..161b2724b 100644 --- a/slither/tools/documentation/__main__.py +++ b/slither/tools/documentation/__main__.py @@ -154,15 +154,15 @@ def _handle_function( ): return overwrite prompt = "Create a natpsec documentation for this solidity code with only notice and dev.\n" - src_mapping = function.source_mapping - content = function.compilation_unit.core.source_code[src_mapping.filename.absolute] - start = src_mapping.start - end = src_mapping.start + src_mapping.length - prompt += content[start:end] - - use_tab = _use_tab(content[start - 1]) - if use_tab is None and src_mapping.starting_column > 1: - logger.info(f"Non standard space indentation found {content[start - 1:end]}") + srcmap = function.source_mapping + src = compilation_unit.core.source_code[srcmap.filename.absolute] + first_char_index = len(src.encode("utf-8")[:srcmap.start].decode("utf-8")) # convert byte offset to char offset + prev_char = src[prev_char_index - 1] + prompt += srcmap.content + + use_tab = _use_tab(prev_char) + if use_tab is None and srcmap.starting_column > 1: + logger.info(f"Non standard indentation found: '{first_char}'") if overwrite: logger.info("Disable overwrite to avoid mistakes") overwrite = False @@ -189,7 +189,7 @@ def _handle_function( if logging_file: codex.log_codex(logging_file, "A: " + str(answer)) - answer_processed = _handle_codex(answer, src_mapping.starting_column, use_tab, force) + answer_processed = _handle_codex(answer, srcmap.starting_column, use_tab, force) if answer_processed: break @@ -201,7 +201,7 @@ def _handle_function( if not answer_processed: return overwrite - create_patch(all_patches, src_mapping.filename.absolute, start, start, "", answer_processed) + create_patch(all_patches, srcmap.filename.absolute, srcmap.start, srcmap.start, "", answer_processed) return overwrite diff --git a/slither/tools/flattening/flattening.py b/slither/tools/flattening/flattening.py index 182333d3f..ace14d09d 100644 --- a/slither/tools/flattening/flattening.py +++ b/slither/tools/flattening/flattening.py @@ -107,6 +107,7 @@ def _get_source_code( :return: """ src_mapping = contract.source_mapping + # TODO: this needs to be encoded before it gets indexed! content = self._compilation_unit.core.source_code[src_mapping.filename.absolute] start = src_mapping.start end = src_mapping.start + src_mapping.length @@ -211,6 +212,12 @@ def _get_source_code( to_patch.sort(key=lambda x: x.index, reverse=True) + # Note: foundry and solc and everything else return srcmap offsets per-byte + # and it seems the rest of slither operates on bytes also + # it might just be the mutator and flattener that are incorrectly applying offsets directly to strings + # I think I just need to do the following (and similar for mutations) + # content = content.encode("utf-8")[start:end].decode("utf-8") + content = content[start:end] for patch in to_patch: patch_type = patch.patch_type From 172cdbb1eeeac09a52657f6b0317929a5efbbbb0 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 7 Feb 2025 14:18:29 -0500 Subject: [PATCH 10/15] fix source_code access in documentation + flattening tools --- slither/tools/documentation/__main__.py | 8 ++--- slither/tools/flattening/flattening.py | 45 ++++++++++++------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/slither/tools/documentation/__main__.py b/slither/tools/documentation/__main__.py index 161b2724b..417e4f09a 100644 --- a/slither/tools/documentation/__main__.py +++ b/slither/tools/documentation/__main__.py @@ -155,14 +155,14 @@ def _handle_function( return overwrite prompt = "Create a natpsec documentation for this solidity code with only notice and dev.\n" srcmap = function.source_mapping - src = compilation_unit.core.source_code[srcmap.filename.absolute] - first_char_index = len(src.encode("utf-8")[:srcmap.start].decode("utf-8")) # convert byte offset to char offset - prev_char = src[prev_char_index - 1] + src = function.compilation_unit.core.source_code[srcmap.filename.absolute] + first_char_index = len(src.encode("utf8")[:srcmap.start].decode("utf8")) # convert byte offset to char offset + prev_char = src[first_char_index - 1] prompt += srcmap.content use_tab = _use_tab(prev_char) if use_tab is None and srcmap.starting_column > 1: - logger.info(f"Non standard indentation found: '{first_char}'") + logger.info(f"Non standard indentation found: '{prev_char}'") if overwrite: logger.info("Disable overwrite to avoid mistakes") overwrite = False diff --git a/slither/tools/flattening/flattening.py b/slither/tools/flattening/flattening.py index ace14d09d..858a83c1f 100644 --- a/slither/tools/flattening/flattening.py +++ b/slither/tools/flattening/flattening.py @@ -108,9 +108,7 @@ def _get_source_code( """ src_mapping = contract.source_mapping # TODO: this needs to be encoded before it gets indexed! - content = self._compilation_unit.core.source_code[src_mapping.filename.absolute] - start = src_mapping.start - end = src_mapping.start + src_mapping.length + src_bytes = self._compilation_unit.core.source_code[src_mapping.filename.absolute] to_patch = [] # interface must use external @@ -125,8 +123,8 @@ def _get_source_code( + f.parameters_src().source_mapping.length ) attributes_end = f.returns_src().source_mapping.start - attributes = content[attributes_start:attributes_end] - regex = re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes) + attributes = src_bytes[attributes_start:attributes_end] + regex = re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes.decode("utf8")) if regex: to_patch.append( Patch( @@ -135,13 +133,13 @@ def _get_source_code( ) ) else: - raise SlitherException(f"External keyword not found {f.name} {attributes}") + raise SlitherException(f"External keyword not found {f.name} {attributes.decode("utf8")}") for var in f.parameters: if var.location == "calldata": calldata_start = var.source_mapping.start calldata_end = calldata_start + var.source_mapping.length - calldata_idx = content[calldata_start:calldata_end].find(" calldata ") + calldata_idx = src_bytes[calldata_start:calldata_end].find(" calldata ") to_patch.append( Patch( calldata_start + calldata_idx + 1, @@ -159,11 +157,11 @@ def _get_source_code( + f.parameters_src().source_mapping["length"] ) attributes_end = f.returns_src().source_mapping["start"] - attributes = content[attributes_start:attributes_end] + attributes = src_bytes[attributes_start:attributes_end] regex = ( - re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes) + re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes.decode("utf8")) if visibility == "external" - else re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", attributes) + else re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", attributes.decode("utf8")) ) if regex: to_patch.append( @@ -176,7 +174,7 @@ def _get_source_code( ) else: raise SlitherException( - f"{visibility} keyword not found {f.name} {attributes}" + f"{visibility} keyword not found {f.name} {attributes.decode("utf8")}" ) if self._private_to_internal: @@ -184,8 +182,8 @@ def _get_source_code( if variable.visibility == "private": attributes_start = variable.source_mapping.start attributes_end = attributes_start + variable.source_mapping.length - attributes = content[attributes_start:attributes_end] - regex = re.search(r" private ", attributes) + attributes = src_bytes[attributes_start:attributes_end] + regex = re.search(r" private ", attributes.decode("utf8")) if regex: to_patch.append( Patch( @@ -195,7 +193,7 @@ def _get_source_code( ) else: raise SlitherException( - f"private keyword not found {variable.name} {attributes}" + f"private keyword not found {variable.name} {attributes.decode("utf8")}" ) if self._remove_assert: @@ -216,28 +214,29 @@ def _get_source_code( # and it seems the rest of slither operates on bytes also # it might just be the mutator and flattener that are incorrectly applying offsets directly to strings # I think I just need to do the following (and similar for mutations) - # content = content.encode("utf-8")[start:end].decode("utf-8") + # content = content.encode("utf8")[start:end].decode("utf8") - content = content[start:end] + content = src_mapping.content.encode("utf8") + start = src_mapping.start for patch in to_patch: patch_type = patch.patch_type index = patch.index index = index - start if patch_type == "public_to_external": - content = content[:index] + "public" + content[index + len("external") :] + content = content[:index].decode("utf8") + "public" + content[index + len("external") :].decode("utf8") elif patch_type == "external_to_internal": - content = content[:index] + "internal" + content[index + len("external") :] + content = content[:index].decode("utf8") + "internal" + content[index + len("external") :].decode("utf8") elif patch_type == "public_to_internal": - content = content[:index] + "internal" + content[index + len("public") :] + content = content[:index].decode("utf8") + "internal" + content[index + len("public") :].decode("utf8") elif patch_type == "private_to_internal": - content = content[:index] + "internal" + content[index + len("private") :] + content = content[:index].decode("utf8") + "internal" + content[index + len("private") :].decode("utf8") elif patch_type == "calldata_to_memory": - content = content[:index] + "memory" + content[index + len("calldata") :] + content = content[:index].decode("utf8") + "memory" + content[index + len("calldata") :].decode("utf8") else: assert patch_type == "line_removal" - content = content[:index] + " // " + content[index:] + content = content[:index].decode("utf8") + " // " + content[index:].decode("utf8") - self._source_codes[contract] = content + self._source_codes[contract] = content.decode("utf8") def _pragmas(self) -> str: """ From 738aa96f050d6796a3b6e32bfdd142753153329f Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 7 Feb 2025 14:18:59 -0500 Subject: [PATCH 11/15] clean up utf8 strings --- slither/analyses/evm/convert.py | 8 ++++---- slither/core/source_mapping/source_mapping.py | 4 ++-- slither/detectors/source/rtlo.py | 6 +++--- slither/formatters/attributes/const_functions.py | 2 +- slither/formatters/functions/external_function.py | 6 +++--- slither/formatters/variables/unchanged_state_variables.py | 2 +- slither/formatters/variables/unused_state_variables.py | 6 +++--- slither/slithir/convert.py | 2 +- slither/tools/mutator/mutators/abstract_mutator.py | 1 - slither/tools/mutator/utils/testing_generated_mutant.py | 8 ++++---- slither/tools/read_storage/__main__.py | 2 +- slither/utils/function.py | 4 ++-- slither/utils/output.py | 2 +- slither/utils/standard_libraries.py | 2 +- slither/visitors/expression/constants_folding.py | 2 +- 15 files changed, 28 insertions(+), 29 deletions(-) diff --git a/slither/analyses/evm/convert.py b/slither/analyses/evm/convert.py index fe294798e..cb01fbb85 100644 --- a/slither/analyses/evm/convert.py +++ b/slither/analyses/evm/convert.py @@ -133,12 +133,12 @@ def _get_evm_instructions_node(node_info): contract_file = ( node_info["slither"] .source_code[node_info["contract"].source_mapping.filename.absolute] - .encode("utf-8") + .encode("utf8") ) # Get evm instructions corresponding to node's source line number node_source_line = ( - contract_file[0 : node_info["node"].source_mapping.start].count("\n".encode("utf-8")) + 1 + contract_file[0 : node_info["node"].source_mapping.start].count("\n".encode("utf8")) + 1 ) node_pcs = contract_pcs.get(node_source_line, []) node_ins = [] @@ -169,7 +169,7 @@ def generate_source_to_evm_ins_mapping(evm_instructions, srcmap_runtime, slither """ source_to_evm_mapping = {} - file_source = slither.source_code[filename].encode("utf-8") + file_source = slither.source_code[filename].encode("utf8") prev_mapping = [] for idx, mapping in enumerate(srcmap_runtime): @@ -193,7 +193,7 @@ def generate_source_to_evm_ins_mapping(evm_instructions, srcmap_runtime, slither # See https://github.com/ethereum/solidity/issues/6119#issuecomment-467797635 continue - line_number = file_source[0 : int(offset)].count("\n".encode("utf-8")) + 1 + line_number = file_source[0 : int(offset)].count("\n".encode("utf8")) + 1 # Append evm instructions to the corresponding source line number # Note: Some evm instructions in mapping are not necessarily in program execution order diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index afa8e83a9..fbb7279a2 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -84,8 +84,8 @@ def content(self) -> str: assert self.compilation_unit return ( self.compilation_unit.core.source_code[self.filename.absolute] - .encode("utf-8")[self.start : self.end] - .decode("utf-8") + .encode("utf8")[self.start : self.end] + .decode("utf8") ) @property diff --git a/slither/detectors/source/rtlo.py b/slither/detectors/source/rtlo.py index b020f69f9..b9bc7aa74 100644 --- a/slither/detectors/source/rtlo.py +++ b/slither/detectors/source/rtlo.py @@ -57,16 +57,16 @@ class RightToLeftOverride(AbstractDetector): WIKI_RECOMMENDATION = "Special control characters must not be allowed." - RTLO_CHARACTER_ENCODED = "\u202e".encode("utf-8") + RTLO_CHARACTER_ENCODED = "\u202e".encode("utf8") STANDARD_JSON = False def _detect(self) -> List[Output]: results = [] - pattern = re.compile(".*\u202e.*".encode("utf-8")) + pattern = re.compile(".*\u202e.*".encode("utf8")) for filename, source in self.slither.source_code.items(): # Attempt to find all RTLO characters in this source file. - original_source_encoded = source.encode("utf-8") + original_source_encoded = source.encode("utf8") start_index = 0 # Keep searching all file contents for the character. diff --git a/slither/formatters/attributes/const_functions.py b/slither/formatters/attributes/const_functions.py index feb404f7b..3ce5c001e 100644 --- a/slither/formatters/attributes/const_functions.py +++ b/slither/formatters/attributes/const_functions.py @@ -43,7 +43,7 @@ def _patch( in_file_str = compilation_unit.core.source_code[in_file].encode("utf8") old_str_of_interest = in_file_str[modify_loc_start:modify_loc_end] # Find the keywords view|pure|constant and remove them - m = re.search("(view|pure|constant)", old_str_of_interest.decode("utf-8")) + m = re.search("(view|pure|constant)", old_str_of_interest.decode("utf8")) if m: create_patch( result, diff --git a/slither/formatters/functions/external_function.py b/slither/formatters/functions/external_function.py index 66dbc4367..de60fa7a7 100644 --- a/slither/formatters/functions/external_function.py +++ b/slither/formatters/functions/external_function.py @@ -32,16 +32,16 @@ def _patch( old_str_of_interest = in_file_str[modify_loc_start:modify_loc_end] # Search for 'public' keyword which is in-between the function name and modifier name (if present) # regex: 'public' could have spaces around or be at the end of the line - m = re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", old_str_of_interest.decode("utf-8")) + m = re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", old_str_of_interest.decode("utf8")) if m is None: # No visibility specifier exists; public by default. create_patch( result, in_file, # start after the function definition's closing paranthesis - modify_loc_start + len(old_str_of_interest.decode("utf-8").split(")")[0]) + 1, + modify_loc_start + len(old_str_of_interest.decode("utf8").split(")")[0]) + 1, # end is same as start because we insert the keyword `external` at that location - modify_loc_start + len(old_str_of_interest.decode("utf-8").split(")")[0]) + 1, + modify_loc_start + len(old_str_of_interest.decode("utf8").split(")")[0]) + 1, "", " external", ) # replace_text is `external` diff --git a/slither/formatters/variables/unchanged_state_variables.py b/slither/formatters/variables/unchanged_state_variables.py index c7c8bf003..cb880d997 100644 --- a/slither/formatters/variables/unchanged_state_variables.py +++ b/slither/formatters/variables/unchanged_state_variables.py @@ -41,7 +41,7 @@ def _patch( # pylint: disable=too-many-arguments old_str_of_interest = in_file_str[modify_loc_start:modify_loc_end] # Add keyword `constant` before the variable name (new_str_of_interest, num_repl) = re.subn( - match_text, replace_text, old_str_of_interest.decode("utf-8"), 1 + match_text, replace_text, old_str_of_interest.decode("utf8"), 1 ) if num_repl != 0: create_patch( diff --git a/slither/formatters/variables/unused_state_variables.py b/slither/formatters/variables/unused_state_variables.py index 90009c7f1..5b1b391eb 100644 --- a/slither/formatters/variables/unused_state_variables.py +++ b/slither/formatters/variables/unused_state_variables.py @@ -22,8 +22,8 @@ def _patch( in_file_str = compilation_unit.core.source_code[in_file].encode("utf8") old_str_of_interest = in_file_str[modify_loc_start:] old_str = ( - old_str_of_interest.decode("utf-8").partition(";")[0] - + old_str_of_interest.decode("utf-8").partition(";")[1] + old_str_of_interest.decode("utf8").partition(";")[0] + + old_str_of_interest.decode("utf8").partition(";")[1] ) create_patch( @@ -31,7 +31,7 @@ def _patch( in_file, int(modify_loc_start), # Remove the entire declaration until the semicolon - int(modify_loc_start + len(old_str_of_interest.decode("utf-8").partition(";")[0]) + 1), + int(modify_loc_start + len(old_str_of_interest.decode("utf8").partition(";")[0]) + 1), old_str, "", ) diff --git a/slither/slithir/convert.py b/slither/slithir/convert.py index cc27eb286..8951707ff 100644 --- a/slither/slithir/convert.py +++ b/slither/slithir/convert.py @@ -196,7 +196,7 @@ def _fits_under_byte(val: Union[int, str]) -> List[str]: size = len(hex_val) // 2 return [f"bytes{size}"] # val is a str - length = len(val.encode("utf-8")) + length = len(val.encode("utf8")) return [f"bytes{f}" for f in range(length, 33)] + ["bytes"] diff --git a/slither/tools/mutator/mutators/abstract_mutator.py b/slither/tools/mutator/mutators/abstract_mutator.py index 1c7274407..cfeee9759 100644 --- a/slither/tools/mutator/mutators/abstract_mutator.py +++ b/slither/tools/mutator/mutators/abstract_mutator.py @@ -47,7 +47,6 @@ def __init__( # pylint: disable=too-many-arguments self.output_folder = output_folder self.contract = contract_instance self.in_file = self.contract.source_mapping.filename.absolute - self.in_file_str = self.contract.compilation_unit.core.source_code[self.in_file] self.dont_mutate_line = dont_mutate_line # total revert/comment/tweak mutants that were generated and compiled self.total_mutant_counts = [0, 0, 0] diff --git a/slither/tools/mutator/utils/testing_generated_mutant.py b/slither/tools/mutator/utils/testing_generated_mutant.py index d5725485b..2ea3314e6 100644 --- a/slither/tools/mutator/utils/testing_generated_mutant.py +++ b/slither/tools/mutator/utils/testing_generated_mutant.py @@ -75,8 +75,8 @@ def run_test_cmd( # If tests fail in verbose-mode, print both stdout and stderr for easier debugging if verbose: - logger.info(yellow(result.stdout.decode("utf-8"))) - logger.info(red(result.stderr.decode("utf-8"))) + logger.info(yellow(result.stdout.decode("utf8"))) + logger.info(red(result.stderr.decode("utf8"))) return False @@ -96,12 +96,12 @@ def test_patch( # pylint: disable=too-many-arguments function to verify whether each patch is caught by tests returns: 0 (uncaught), 1 (caught), or 2 (compilation failure) """ - with open(file, "r", encoding="utf-8") as filepath: + with open(file, "r", encoding="utf8") as filepath: content = filepath.read() # Perform the replacement based on the index values replaced_content = content[: patch["start"]] + patch["new_string"] + content[patch["end"] :] # Write the modified content back to the file - with open(file, "w", encoding="utf-8") as filepath: + with open(file, "w", encoding="utf8") as filepath: filepath.write(replaced_content) if compile_generated_mutant(file, mappings): diff --git a/slither/tools/read_storage/__main__.py b/slither/tools/read_storage/__main__.py index 3baa5d351..23b98764c 100644 --- a/slither/tools/read_storage/__main__.py +++ b/slither/tools/read_storage/__main__.py @@ -169,7 +169,7 @@ def main() -> None: print(srs.table) if args.json: - with open(args.json, "w", encoding="utf-8") as file: + with open(args.json, "w", encoding="utf8") as file: slot_infos_json = srs.to_json() json.dump(slot_infos_json, file, indent=4) diff --git a/slither/utils/function.py b/slither/utils/function.py index 64c098bfd..ede9e470b 100644 --- a/slither/utils/function.py +++ b/slither/utils/function.py @@ -10,7 +10,7 @@ def get_function_id(sig: str) -> int: (int) """ digest = keccak.new(digest_bits=256) - digest.update(sig.encode("utf-8")) + digest.update(sig.encode("utf8")) return int("0x" + digest.hexdigest()[:8], 16) @@ -23,5 +23,5 @@ def get_event_id(sig: str) -> int: (int) """ digest = keccak.new(digest_bits=256) - digest.update(sig.encode("utf-8")) + digest.update(sig.encode("utf8")) return int("0x" + digest.hexdigest(), 16) diff --git a/slither/utils/output.py b/slither/utils/output.py index 176b250e3..59bfa1872 100644 --- a/slither/utils/output.py +++ b/slither/utils/output.py @@ -411,7 +411,7 @@ def __init__( self._markdown_root = markdown_root id_txt = "".join(_convert_to_id(d) for d in info) - self._data["id"] = hashlib.sha3_256(id_txt.encode("utf-8")).hexdigest() + self._data["id"] = hashlib.sha3_256(id_txt.encode("utf8")).hexdigest() if standard_format: to_add = [i for i in info if not isinstance(i, str)] diff --git a/slither/utils/standard_libraries.py b/slither/utils/standard_libraries.py index 525771187..82defd51a 100644 --- a/slither/utils/standard_libraries.py +++ b/slither/utils/standard_libraries.py @@ -68,7 +68,7 @@ def is_openzeppelin(contract: "Contract") -> bool: def is_openzeppelin_strict(contract: "Contract") -> bool: - source_hash = sha1(contract.source_mapping.content.encode("utf-8")).hexdigest() + source_hash = sha1(contract.source_mapping.content.encode("utf8")).hexdigest() return source_hash in oz_hashes diff --git a/slither/visitors/expression/constants_folding.py b/slither/visitors/expression/constants_folding.py index ddadb77a1..65574df67 100644 --- a/slither/visitors/expression/constants_folding.py +++ b/slither/visitors/expression/constants_folding.py @@ -276,7 +276,7 @@ def _post_call_expression(self, expression: expressions.CallExpression) -> None: args = [get_val(arg) for arg in expression.arguments] if called.name == "keccak256(bytes)": digest = keccak.new(digest_bits=256) - digest.update(str(args[0]).encode("utf-8")) + digest.update(str(args[0]).encode("utf8")) set_val(expression, digest.digest()) else: raise NotConstant From b3a226e10ee9f1242a35290b21e20a60c3420d07 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 7 Feb 2025 14:28:23 -0500 Subject: [PATCH 12/15] bugfix Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 17697385e..6c6955fa7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PY_MODULE := slither TEST_MODULE := tests ALL_PY_SRCS := $(shell find $(PY_MODULE) -name '*.py') \ - $(shell find test -name '*.py') + $(shell find tests -name '*.py') # Optionally overridden by the user, if they're using a virtual environment manager. VENV ?= env @@ -85,4 +85,4 @@ package: $(VENV)/pyvenv.cfg .PHONY: edit edit: - $(EDITOR) $(ALL_PY_SRCS) \ No newline at end of file + $(EDITOR) $(ALL_PY_SRCS) From 8dee964d1f4471b4ab45dc1655450c99bd3ea0e8 Mon Sep 17 00:00:00 2001 From: bohendo Date: Fri, 7 Feb 2025 14:53:17 -0500 Subject: [PATCH 13/15] fix formatting --- slither/core/source_mapping/source_mapping.py | 4 +- slither/tools/documentation/__main__.py | 8 ++- slither/tools/flattening/flattening.py | 61 ++++++++++++------- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/slither/core/source_mapping/source_mapping.py b/slither/core/source_mapping/source_mapping.py index fbb7279a2..7a6cdae41 100644 --- a/slither/core/source_mapping/source_mapping.py +++ b/slither/core/source_mapping/source_mapping.py @@ -84,8 +84,8 @@ def content(self) -> str: assert self.compilation_unit return ( self.compilation_unit.core.source_code[self.filename.absolute] - .encode("utf8")[self.start : self.end] - .decode("utf8") + .encode("utf8")[self.start : self.end] + .decode("utf8") ) @property diff --git a/slither/tools/documentation/__main__.py b/slither/tools/documentation/__main__.py index 417e4f09a..dccdabba4 100644 --- a/slither/tools/documentation/__main__.py +++ b/slither/tools/documentation/__main__.py @@ -156,7 +156,9 @@ def _handle_function( prompt = "Create a natpsec documentation for this solidity code with only notice and dev.\n" srcmap = function.source_mapping src = function.compilation_unit.core.source_code[srcmap.filename.absolute] - first_char_index = len(src.encode("utf8")[:srcmap.start].decode("utf8")) # convert byte offset to char offset + first_char_index = len( + src.encode("utf8")[: srcmap.start].decode("utf8") + ) # convert byte offset to char offset prev_char = src[first_char_index - 1] prompt += srcmap.content @@ -201,7 +203,9 @@ def _handle_function( if not answer_processed: return overwrite - create_patch(all_patches, srcmap.filename.absolute, srcmap.start, srcmap.start, "", answer_processed) + create_patch( + all_patches, srcmap.filename.absolute, srcmap.start, srcmap.start, "", answer_processed + ) return overwrite diff --git a/slither/tools/flattening/flattening.py b/slither/tools/flattening/flattening.py index 858a83c1f..518e36717 100644 --- a/slither/tools/flattening/flattening.py +++ b/slither/tools/flattening/flattening.py @@ -107,8 +107,9 @@ def _get_source_code( :return: """ src_mapping = contract.source_mapping - # TODO: this needs to be encoded before it gets indexed! - src_bytes = self._compilation_unit.core.source_code[src_mapping.filename.absolute] + src_bytes = self._compilation_unit.core.source_code[src_mapping.filename.absolute].encode( + "utf8" + ) to_patch = [] # interface must use external @@ -123,8 +124,8 @@ def _get_source_code( + f.parameters_src().source_mapping.length ) attributes_end = f.returns_src().source_mapping.start - attributes = src_bytes[attributes_start:attributes_end] - regex = re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes.decode("utf8")) + attributes = src_bytes[attributes_start:attributes_end].decode("utf8") + regex = re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes) if regex: to_patch.append( Patch( @@ -133,7 +134,7 @@ def _get_source_code( ) ) else: - raise SlitherException(f"External keyword not found {f.name} {attributes.decode("utf8")}") + raise SlitherException(f"External keyword not found {f.name} {attributes}") for var in f.parameters: if var.location == "calldata": @@ -157,11 +158,11 @@ def _get_source_code( + f.parameters_src().source_mapping["length"] ) attributes_end = f.returns_src().source_mapping["start"] - attributes = src_bytes[attributes_start:attributes_end] + attributes = src_bytes[attributes_start:attributes_end].decode("utf8") regex = ( - re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes.decode("utf8")) + re.search(r"((\sexternal)\s+)|(\sexternal)$|(\)external)$", attributes) if visibility == "external" - else re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", attributes.decode("utf8")) + else re.search(r"((\spublic)\s+)|(\spublic)$|(\)public)$", attributes) ) if regex: to_patch.append( @@ -174,7 +175,7 @@ def _get_source_code( ) else: raise SlitherException( - f"{visibility} keyword not found {f.name} {attributes.decode("utf8")}" + f"{visibility} keyword not found {f.name} {attributes}" ) if self._private_to_internal: @@ -182,8 +183,8 @@ def _get_source_code( if variable.visibility == "private": attributes_start = variable.source_mapping.start attributes_end = attributes_start + variable.source_mapping.length - attributes = src_bytes[attributes_start:attributes_end] - regex = re.search(r" private ", attributes.decode("utf8")) + attributes = src_bytes[attributes_start:attributes_end].decode("utf8") + regex = re.search(r" private ", attributes) if regex: to_patch.append( Patch( @@ -193,7 +194,7 @@ def _get_source_code( ) else: raise SlitherException( - f"private keyword not found {variable.name} {attributes.decode("utf8")}" + f"private keyword not found {variable.name} {attributes}" ) if self._remove_assert: @@ -210,12 +211,6 @@ def _get_source_code( to_patch.sort(key=lambda x: x.index, reverse=True) - # Note: foundry and solc and everything else return srcmap offsets per-byte - # and it seems the rest of slither operates on bytes also - # it might just be the mutator and flattener that are incorrectly applying offsets directly to strings - # I think I just need to do the following (and similar for mutations) - # content = content.encode("utf8")[start:end].decode("utf8") - content = src_mapping.content.encode("utf8") start = src_mapping.start for patch in to_patch: @@ -223,15 +218,35 @@ def _get_source_code( index = patch.index index = index - start if patch_type == "public_to_external": - content = content[:index].decode("utf8") + "public" + content[index + len("external") :].decode("utf8") + content = ( + content[:index].decode("utf8") + + "public" + + content[index + len("external") :].decode("utf8") + ) elif patch_type == "external_to_internal": - content = content[:index].decode("utf8") + "internal" + content[index + len("external") :].decode("utf8") + content = ( + content[:index].decode("utf8") + + "internal" + + content[index + len("external") :].decode("utf8") + ) elif patch_type == "public_to_internal": - content = content[:index].decode("utf8") + "internal" + content[index + len("public") :].decode("utf8") + content = ( + content[:index].decode("utf8") + + "internal" + + content[index + len("public") :].decode("utf8") + ) elif patch_type == "private_to_internal": - content = content[:index].decode("utf8") + "internal" + content[index + len("private") :].decode("utf8") + content = ( + content[:index].decode("utf8") + + "internal" + + content[index + len("private") :].decode("utf8") + ) elif patch_type == "calldata_to_memory": - content = content[:index].decode("utf8") + "memory" + content[index + len("calldata") :].decode("utf8") + content = ( + content[:index].decode("utf8") + + "memory" + + content[index + len("calldata") :].decode("utf8") + ) else: assert patch_type == "line_removal" content = content[:index].decode("utf8") + " // " + content[index:].decode("utf8") From 78fe705920ae0b79277447c4065b34a867d689df Mon Sep 17 00:00:00 2001 From: bohendo Date: Mon, 10 Feb 2025 11:41:34 -0500 Subject: [PATCH 14/15] bugfix naming_convention formatter --- slither/formatters/naming_convention/naming_convention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slither/formatters/naming_convention/naming_convention.py b/slither/formatters/naming_convention/naming_convention.py index 8b3c40df4..56fdf86b7 100644 --- a/slither/formatters/naming_convention/naming_convention.py +++ b/slither/formatters/naming_convention/naming_convention.py @@ -339,7 +339,7 @@ def _is_var_declaration(slither: SlitherCompilationUnit, filename: str, start: i :return: """ v = "var " - return slither.core.source_code[filename][start : start + len(v)] == v + return slither.core.source_code[filename].encode("utf8")[start : start + len(v)].decode("utf8") == v def _explore_type( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches From 277f011a7b62e1c1f04980810a9ae3371a980d9e Mon Sep 17 00:00:00 2001 From: bohendo <8527067+bohendo@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:40:39 -0500 Subject: [PATCH 15/15] fix formatting Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- slither/formatters/naming_convention/naming_convention.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/slither/formatters/naming_convention/naming_convention.py b/slither/formatters/naming_convention/naming_convention.py index 56fdf86b7..ebb6b7f5b 100644 --- a/slither/formatters/naming_convention/naming_convention.py +++ b/slither/formatters/naming_convention/naming_convention.py @@ -339,7 +339,10 @@ def _is_var_declaration(slither: SlitherCompilationUnit, filename: str, start: i :return: """ v = "var " - return slither.core.source_code[filename].encode("utf8")[start : start + len(v)].decode("utf8") == v + return ( + slither.core.source_code[filename].encode("utf8")[start : start + len(v)].decode("utf8") + == v + ) def _explore_type( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches