Updated key rotation to be more random

augur/application/cli/backend.py

@@ -170,21 +170,21 @@ def determine_worker_processes(ratio,maximum):
         sleep_time += 6
 
         #60% of estimate, Maximum value of 45
-        core_num_processes = determine_worker_processes(.6, 80)
+        core_num_processes = determine_worker_processes(.6, 45)
         logger.info(f"Starting core worker processes with concurrency={core_num_processes}")
         core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={core_num_processes} -n core:{uuid.uuid4().hex}@%h"
         process_list.append(subprocess.Popen(core_worker.split(" ")))
         sleep_time += 6
 
         #20% of estimate, Maximum value of 25
-        secondary_num_processes = determine_worker_processes(.2, 26)
+        secondary_num_processes = determine_worker_processes(.2, 25)
         logger.info(f"Starting secondary worker processes with concurrency={secondary_num_processes}")
         secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={secondary_num_processes} -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
         process_list.append(subprocess.Popen(secondary_worker.split(" ")))
         sleep_time += 6
 
         #15% of estimate, Maximum value of 20
-        facade_num_processes = determine_worker_processes(.2, 40)
+        facade_num_processes = determine_worker_processes(.2, 20)
         logger.info(f"Starting facade worker processes with concurrency={facade_num_processes}")
         facade_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={facade_num_processes} -n facade:{uuid.uuid4().hex}@%h -Q facade"
 

augur/tasks/github/issues/tasks.py

@@ -5,6 +5,7 @@
 
 from sqlalchemy.exc import IntegrityError
 
+from augur.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
 
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask
@@ -29,16 +30,29 @@ def collect_issues(repo_git : str) -> int:
 
         augur_db = manifest.augur_db
 
+        logger.info(f'this is the manifest.key_auth value: {str(manifest.key_auth)}')
+
         try:
 
             query = augur_db.session.query(Repo).filter(Repo.repo_git == repo_git)
             repo_obj = execute_session_query(query, 'one')
             repo_id = repo_obj.repo_id
 
+            #try this
+            # the_key = manifest.key_auth
+            # try: 
+            #     randomon = GithubApiKeyHandler(augur_db.session)
+            #     the_key = randomon.get_random_key()
+            #     logger.info(f'The Random Key {the_key}')
+            # except Exception as e: 
+            #     logger.info(f'error: {e}')
+            #     the_key = manifest.key_auth
+            #     pass 
+
             owner, repo = get_owner_repo(repo_git)
 
             issue_data = retrieve_all_issue_data(repo_git, logger, manifest.key_auth)
-
+            #issue_data = retrieve_all_issue_data(repo_git, logger, the_key)
 
             if issue_data:
                 total_issues = len(issue_data)

augur/tasks/github/util/github_api_key_handler.py

@@ -7,6 +7,7 @@
 from augur.tasks.util.redis_list import RedisList
 from augur.application.db.session import DatabaseSession
 from augur.application.config import AugurConfig
+from sqlalchemy import func 
 
 
 class NoValidKeysError(Exception):
@@ -39,7 +40,7 @@ def __init__(self, session: DatabaseSession):
 
         self.keys = self.get_api_keys()
 
-        # self.logger.debug(f"Retrieved {len(self.keys)} github api keys for use")
+        self.logger.info(f"Retrieved {len(self.keys)} github api keys for use")
 
     def get_random_key(self):
         """Retrieves a random key from the list of keys
@@ -71,9 +72,11 @@ def get_api_keys_from_database(self) -> List[str]:
         from augur.application.db.models import WorkerOauth
 
         select = WorkerOauth.access_token
+        # randomizing the order at db time
+        #select.order_by(func.random())
         where = [WorkerOauth.access_token != self.config_key, WorkerOauth.platform == 'github']
 
-        return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).all()]
+        return [key_tuple[0] for key_tuple in self.session.query(select).filter(*where).order_by(func.random()).all()]
 
 
     def get_api_keys(self) -> List[str]:
@@ -130,6 +133,18 @@ def get_api_keys(self) -> List[str]:
         if not valid_keys:
             raise NoValidKeysError("No valid github api keys found in the config or worker oauth table")
 
+
+        # shuffling the keys so not all processes get the same keys in the same order
+        valid_now = valid_keys
+        try: 
+            self.logger.debug(f'valid keys before shuffle: {valid_keys}')
+            valid_keys = random.sample(valid_keys, len(valid_keys))
+            self.logger.debug(f'valid keys AFTER shuffle: {valid_keys}')
+        except Exception as e: 
+            self.logger.debug(f'{e}')
+            valid_keys = valid_now
+            pass 
+
         return valid_keys
 
     def is_bad_api_key(self, client: httpx.Client, oauth_key: str) -> bool:

augur/tasks/github/util/github_random_key_auth.py

@@ -3,6 +3,7 @@
 from augur.tasks.util.random_key_auth import RandomKeyAuth
 from augur.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
 from augur.application.db.session import DatabaseSession
+import random 
 
 
 class GithubRandomKeyAuth(RandomKeyAuth):
@@ -16,6 +17,7 @@ def __init__(self, session: DatabaseSession, logger):
 
         # gets the github api keys from the database via the GithubApiKeyHandler
         github_api_keys = GithubApiKeyHandler(session).keys
+        github_api_keys = random.sample(github_api_keys, len(github_api_keys))
 
         if not github_api_keys:
             print("Failed to find github api keys. This is usually because your key has expired")

augur/tasks/start_tasks.py

@@ -261,16 +261,16 @@ def augur_collection_monitor():
         enabled_phase_names = get_enabled_phase_names_from_config(session.logger, session)
 
         if primary_repo_collect_phase.__name__ in enabled_phase_names:
-            start_primary_collection(session, max_repo=40)
+            start_primary_collection(session, max_repo=30)
 
         if secondary_repo_collect_phase.__name__ in enabled_phase_names:
             start_secondary_collection(session, max_repo=10)
 
         if facade_phase.__name__ in enabled_phase_names:
-            start_facade_collection(session, max_repo=30)
+            start_facade_collection(session, max_repo=20)
 
         if machine_learning_phase.__name__ in enabled_phase_names:
-            start_ml_collection(session,max_repo=5)
+            start_ml_collection(session,max_repo=1)
 
 # have a pipe of 180
 

augur/tasks/util/random_key_auth.py

@@ -43,6 +43,7 @@ def auth_flow(self, request: Request) -> Generator[Request, Response, None]:
 
             # set the headers of the request with the new key
             request.headers[self.header_name] = key_string
+            #self.logger.info(f"List of Keys: {self.list_of_keys}")
 
         else:
             self.logger.error(f"There are no valid keys to make a request with: {self.list_of_keys}")

augur/tasks/util/redis_list.py

@@ -168,8 +168,10 @@ def pop(self, index: int = None):
         """
 
         if index is None:
-
-            redis.rpop(self.redis_list_key)
+            # This will get a random index from the list and remove it, 
+            # decreasing the likelihood of everyone using the same key all the time
+            #redis.rpop(self.redis_list_key)
+            redis.spop(self.redis_list_key)
 
         else:
             # calls __delitem__