Skip to content

Commit

Permalink
Merge pull request #2530 from chaoss/dev
Browse files Browse the repository at this point in the history 
Release PR
  • Loading branch information
@sgoggins
sgoggins authored Sep 25, 2023
2 parents bf738b3 + 5dc8b9b commit 6e57e89
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 31 deletions.
2 changes: 1 addition & 1 deletion augur/application/db/models/augur_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def retrieve_owner_repos(session, owner: str) -> List[str]:

# collect repo urls for the given owner
repos = []
for page_data in GithubPaginator(url, session.oauths, logger).iter_pages():
for page_data, _ in GithubPaginator(url, session.oauths, logger).iter_pages():

if page_data is None:
break
Expand Down
20 changes: 10 additions & 10 deletions augur/tasks/db/refresh_materialized_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def refresh_materialized_views():
from augur.tasks.init.celery_app import engine

logger = logging.getLogger(refresh_materialized_views.__name__)
self.logger = logging.getLogger(refresh_materialized_views.__name__)
#self.logger = logging.getLogger(refresh_materialized_views.__name__)

mv1_refresh = s.sql.text("""
REFRESH MATERIALIZED VIEW concurrently augur_data.api_get_all_repo_prs with data;
Expand Down Expand Up @@ -65,64 +65,64 @@ def refresh_materialized_views():
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv1_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass


try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv1_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv2_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv3_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv4_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv5_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv6_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv7_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass

try:
with DatabaseSession(logger, engine) as session:
session.execute_sql(mv8_refresh)
except Exception as e:
self.logger.info(f"error is {e}")
logger.info(f"error is {e}")
pass


Expand Down
42 changes: 22 additions & 20 deletions augur/tasks/github/pull_requests/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ def collect_pull_request_reviews(repo_git: str) -> None:

pr_count = len(prs)

all_raw_pr_reviews = []
all_pr_reviews = {}
for index, pr in enumerate(prs):

pr_number = pr.pr_src_number
Expand All @@ -343,40 +343,46 @@ def collect_pull_request_reviews(repo_git: str) -> None:

pr_review_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/reviews"

pr_reviews = GithubPaginator(pr_review_url, manifest.key_auth, logger)

for page_data, page in pr_reviews.iter_pages():
pr_reviews = []
pr_reviews_generator = GithubPaginator(pr_review_url, manifest.key_auth, logger)
for page_data, page in pr_reviews_generator.iter_pages():

if page_data is None:
break

if len(page_data) == 0:
break

all_raw_pr_reviews.extend(page_data)
pr_reviews.extend(page_data)

if pr_reviews:
all_pr_reviews[pull_request_id] = pr_reviews

if not all_raw_pr_reviews:
if not list(all_pr_reviews.keys()):
logger.info(f"{owner}/{repo} No pr reviews for repo")
return

contributors = []
for raw_pr_review in all_raw_pr_reviews:
contributor = process_pull_request_review_contributor(raw_pr_review, tool_source, tool_version, data_source)
if contributor:
contributors.append(contributor)
for pull_request_id in all_pr_reviews.keys():

reviews = all_pr_reviews[pull_request_id]
for review in reviews:
contributor = process_pull_request_review_contributor(review, tool_source, tool_version, data_source)
if contributor:
contributors.append(contributor)

logger.info(f"{owner}/{repo} Pr reviews: Inserting {len(contributors)} contributors")
augur_db.insert_data(contributors, Contributor, ["cntrb_id"])


pr_reviews = []
for raw_pr_review in all_raw_pr_reviews:

logger.info(f"Pr review type: {type(raw_pr_review)}")
logger.info(raw_pr_review)
for pull_request_id in all_pr_reviews.keys():

if "cntrb_id" in raw_pr_review:
pr_reviews.append(extract_needed_pr_review_data(raw_pr_review, pull_request_id, repo_id, platform_id, tool_source, tool_version))
reviews = all_pr_reviews[pull_request_id]
for review in reviews:

if "cntrb_id" in review:
pr_reviews.append(extract_needed_pr_review_data(review, pull_request_id, repo_id, platform_id, tool_source, tool_version))

logger.info(f"{owner}/{repo}: Inserting pr reviews of length: {len(pr_reviews)}")
pr_review_natural_keys = ["pr_review_src_id",]
Expand All @@ -395,7 +401,3 @@ def collect_pull_request_reviews(repo_git: str) -> None:







0 comments on commit 6e57e89

Please sign in to comment.