From c2d97ab8f11d491333c88328d02437c6d99e2011 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jan 2024 19:25:14 -0600 Subject: [PATCH 1/5] Fix issue where table isn't passed into recursive call Signed-off-by: Andrew Brain --- augur/application/db/session.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index f1d1e64dd0..a943f92fe8 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -240,6 +240,9 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update) self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + else: self.logger.error("Unable to insert and return data in 10 attempts") return None From ef169bec07bf055b418ace9b75d06ced76110dd3 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jan 2024 19:42:36 -0600 Subject: [PATCH 2/5] Add 5 second delay before we retry Signed-off-by: Andrew Brain --- augur/application/db/session.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index a943f92fe8..e4a358f346 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -197,6 +197,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s if(len(data) == 1): raise e else: + time.sleep(5) first_half = data[:len(data)//2] second_half = data[len(data)//2:] @@ -234,6 +235,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s if(len(data) == 1): raise e else: + time.sp first_half = data[:len(data)//2] second_half = data[len(data)//2:] From 888dd23350817bf985b6829d61ecd04cb17e3eb2 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jan 2024 19:43:07 -0600 Subject: [PATCH 3/5] Change to 3 second delay Signed-off-by: Andrew Brain --- augur/application/db/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index e4a358f346..0d98999de2 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -197,7 +197,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s if(len(data) == 1): raise e else: - time.sleep(5) + time.sleep(3) first_half = data[:len(data)//2] second_half = data[len(data)//2:] From eb3887feb2dbf7076571064583d3b1d2091ac21a Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jan 2024 19:57:33 -0600 Subject: [PATCH 4/5] Fix issues suggested by linter Signed-off-by: Andrew Brain --- augur/application/db/session.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 0d98999de2..5d11b3b11c 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -196,13 +196,13 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s #self.logger.info(e) if(len(data) == 1): raise e - else: - time.sleep(3) - first_half = data[:len(data)//2] - second_half = data[len(data)//2:] + + time.sleep(3) + first_half = data[:len(data)//2] + second_half = data[len(data)//2:] - self.insert_data(first_half, table,natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert data in 10 attempts") @@ -232,18 +232,15 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s raise e except Exception as e: - if(len(data) == 1): + if len(data) == 1: raise e - else: - time.sp - first_half = data[:len(data)//2] - second_half = data[len(data)//2:] - - self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update) + + time.sleep(3) + first_half = data[:len(data)//2] + second_half = data[len(data)//2:] - self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half, table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert and return data in 10 attempts") From 51c58e6ef280814ab7efe0363841d3279fd11ee8 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 22 Jan 2024 19:58:58 -0600 Subject: [PATCH 5/5] Fix other issues suggested by linter Signed-off-by: Andrew Brain --- augur/application/db/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 5d11b3b11c..22379ad050 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -194,7 +194,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s except Exception as e: #self.logger.info(e) - if(len(data) == 1): + if len(data) == 1: raise e time.sleep(3)