Merge pull request #66 from nelsonjd/iss-65

Iss 65 - write active edges as transactions

paramFiles/small8/degree.csv

@@ -1,6 +1,6 @@
 Count,In-degree,Out-degree
-0,0,1
+1,1,0
 1,4,1
 2,2,3
-3,0,3
-4,2,0
+2,0,3
+2,2,0

scripts/transaction_graph_generator.py

@@ -302,29 +302,6 @@ def hub_nodes(self):
         return nodes
 
 
-    def add_normal_sar_edges(self, ratio=1.0):
-        """Add extra edges from normal accounts to SAR accounts to adjust transaction graph features
-        :param ratio: Ratio of the number of edges to be added from normal accounts to SAR accounts
-        compared to the number of total SAR accounts
-        """
-        sar_flags = nx.get_node_attributes(self.g, IS_SAR_KEY)
-        orig_candidates = [n for n in self.hubs if not sar_flags.get(n, False)]  # Normal
-        bene_candidates = [n for n, sar in sar_flags.items() if sar]  # SAR
-        num = int(len(bene_candidates) * ratio)
-        if num <= 0:
-            return
-
-        num_origs = len(orig_candidates)
-        print("Number of orig/bene candidates: %d/%d" % (num_origs, len(bene_candidates)))
-        orig_list = random.choices(orig_candidates, k=num)
-        bene_list = random.choices(bene_candidates, k=num)
-        for i in range(num):
-            _orig = orig_list[i]
-            _bene = bene_list[i]
-            self.g.add_edge(_orig, _bene)
-            self.add_edge_info(_orig, _bene)
-        logger.info("Added %d edges from normal accounts to sar accounts" % num)
-
     def check_account_exist(self, aid):
         """Validate an existence of a specified account. If absent, it raises KeyError.
         :param aid: Account ID
@@ -612,6 +589,13 @@ def load_edgelist(self, members, csv_name):
         self.add_subgraph(members, topology)
 
 
+    def mark_active_edges(self):
+        nx.set_edge_attributes(self.g, 'active', False)
+        for normal_model in self.normal_models:
+            subgraph = self.g.subgraph(normal_model.node_ids)
+            nx.set_edge_attributes(subgraph, 'active', True)
+
+
     def load_normal_models(self):
         """Load a Normal Model parameter file
         """
@@ -1247,7 +1231,8 @@ def write_transaction_list(self):
                 attr = e[2]
                 tid = attr['edge_id']
                 tx_type = random.choice(self.tx_types)
-                writer.writerow([tid, src, dst, tx_type])
+                if attr['active']:
+                    writer.writerow([tid, src, dst, tx_type])
         logger.info("Exported %d transactions to %s" % (self.g.number_of_edges(), tx_file))
 
     def write_alert_account_list(self):
@@ -1334,16 +1319,8 @@ def count__patterns(self, threshold=2):
 
     _conf_file = argv[1]
     _sim_name = argv[2] if argc >= 3 else None
-    _ratio = float(argv[3]) if argc >= 4 else 0.0
 
-    # if len(argv) >= 3:
-    #     _sim_name = argv[2]
-    # else:
-    #     _sim_name = None
-    # if len(argv) >= 4:
-    #     _ratio = float(argv[3])
-    # else:
-    #     _ratio = 0.0
+
 
     # Validation option for graph contractions
     deg_param = os.getenv("DEGREE")
@@ -1364,7 +1341,7 @@ def count__patterns(self, threshold=2):
     txg.build_normal_models()
     txg.set_main_acct_candidates()
     txg.load_alert_patterns()  # Load a parameter CSV file for AML typology subgraphs
-    txg.add_normal_sar_edges(_ratio)
+    txg.mark_active_edges()
 
     if degree_threshold > 0:
         logger.info("Added alert transaction patterns")

tests/fixtures/conf.py

@@ -0,0 +1,31 @@
+CONFIG = {
+    'general': {
+        'simulation_name': 'sample',
+        'total_steps': 100
+    },
+    'default': {
+
+    },
+    'input': {
+        "directory": "paramFiles/small8",
+        "schema": "schema.json",
+        "accounts": "accounts.csv",
+        "alert_patterns": "alertPatterns.csv",
+        "normal_models": "normalModels.csv",
+        "degree": "degree.csv",
+        "transaction_type": "transactionType.csv",
+        "is_aggregated_accounts": True
+    },
+    "temporal": {
+        "directory": "tmp",
+        "transactions": "transactions.csv",
+        "accounts": "accounts.csv",
+        "alert_members": "alert_members.csv",
+        "normal_models": "normal_models.csv"
+    },
+    "graph_generator": {
+        "degree_threshold": 3,
+        "high_risk_countries": "",
+        "high_risk_business": ""
+    },
+}

tests/transaction_graph_generator_test.py

@@ -1,8 +1,11 @@
 import unittest
 
-from transaction_graph_generator import get_degrees
+from transaction_graph_generator import TransactionGenerator, get_degrees
 from transaction_graph_generator import get_in_and_out_degrees
 from transaction_graph_generator import directed_configuration_model
+import networkx as nx
+from fixtures.conf import CONFIG
+from amlsim.normal_model import NormalModel
 
 
 class TransactionGraphGeneratorTests(unittest.TestCase):
@@ -113,7 +116,33 @@ def test_directed_configuration_model_self_loops(self):
         self.assertEqual(G.selfloop_edges(), [])
 
 
-
+    def test_mark_active_edges_marks_default_as_false(self):
+        G = nx.DiGraph()
+        G.add_nodes_from([1, 2, 3])
+        G.add_edge(2, 3)
+
+        txg = TransactionGenerator(CONFIG)
+        txg.g = G
+        txg.mark_active_edges()
+        self.assertEqual(txg.g[2][3]['active'], False)
+
+
+    def test_mark_active_edges_marks_real_path_as_active(self):
+        G = nx.DiGraph()
+        G.add_nodes_from([1, 2, 3])
+        G.add_edge(2, 3)
+        G.add_edge(1, 2)
+
+        txg = TransactionGenerator(CONFIG)
+        txg.g = G
+        txg.normal_models = [
+            NormalModel(
+                1, 'single', {2,3}, 2
+            )
+        ]
+        txg.mark_active_edges()
+        self.assertEqual(txg.g[2][3]['active'], True)
+        self.assertEqual(txg.g[1][2]['active'], False)
 
 
 if __name__ == ' main ':