Fix retry insert data issue

augur/application/db/session.py

@@ -1,29 +1,29 @@
 import os
 import re
 import time
 import sys
 import random
 import logging
 from sqlalchemy.orm import Session
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import OperationalError
 
 from typing import Optional, List, Union
 from psycopg2.errors import DeadlockDetected
 
 # from augur.tasks.util.random_key_auth import RandomKeyAuth
 from augur.application.db.engine import EngineConnection
 from augur.tasks.util.worker_util import remove_duplicate_dicts, remove_duplicates_by_uniques
 
 
 def remove_null_characters_from_string(string):
 
     if string:
         return string.replace("\x00", "\uFFFD")
 
     return string
 
 def remove_null_characters_from_strings_in_dict(data, fields):
 
     for field in fields:
 
@@ -40,14 +40,14 @@
 
     return data
 
 def remove_null_characters_from_list_of_dicts(data_list, fields):
 
     for value in data_list:
         value = remove_null_characters_from_strings_in_dict(value, fields)
 
     return data_list
 
 class DatabaseSession(Session):
 
     def __init__(self, logger, engine=None, from_msg=None, **kwargs):
 
@@ -58,7 +58,7 @@
 
         if self.engine is None:
             self.logger.debug("Passing engine will be required soon")
             from augur.application.db.engine import DatabaseEngine
 
             self.engine_created = True
 
@@ -194,14 +194,15 @@
 
                 except Exception as e:
                     #self.logger.info(e)
-                    if(len(data) == 1):
+                    if len(data) == 1:
                         raise e
-                    else:
-                        first_half = data[:len(data)//2]
-                        second_half = data[len(data)//2:]
+
+                    time.sleep(3)
+                    first_half = data[:len(data)//2]
+                    second_half = data[len(data)//2:]
 
-                        self.insert_data(first_half, table,natural_keys, return_columns, string_fields, on_conflict_update)
-                        self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update)
+                    self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update)
+                    self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update)
 
             else:
                 self.logger.error("Unable to insert data in 10 attempts")
@@ -231,14 +232,15 @@
                 raise e
 
             except Exception as e:
-                if(len(data) == 1):
+                if len(data) == 1:
                     raise e
-                else:
-                    first_half = data[:len(data)//2]
-                    second_half = data[len(data)//2:]
+
+                time.sleep(3)
+                first_half = data[:len(data)//2]
+                second_half = data[len(data)//2:]
 
-                    self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update)
-                    self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update)
+                self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update)
+                self.insert_data(second_half, table, natural_keys, return_columns, string_fields, on_conflict_update)
 
         else:
             self.logger.error("Unable to insert and return data in 10 attempts")