diff --git a/app/admin.py b/app/admin.py index c1bb3d6..8f7f364 100644 --- a/app/admin.py +++ b/app/admin.py @@ -4,72 +4,72 @@ # admin.site.register(models.GithubAppAuth) -admin.site.register(models.GithubUser) +# admin.site.register(models.GithubUser) -class GithubRepoMapAdmin(admin.TabularInline): - model = models.GithubRepoMap +# class GithubRepoMapAdmin(admin.TabularInline): +# model = models.GithubRepoMap - def formfield_for_foreignkey(self, db_field, request, **kwargs): - print(db_field.name) - # print(request) - # print(kwargs) - if db_field.name in ["code_repo", "documentation_repo"]: - github_app_installation_id = request.resolver_match.kwargs.get("object_id") - kwargs["queryset"] = models.GithubRepository.objects.filter( - owner_id=github_app_installation_id - ) - return super(GithubRepoMapAdmin, self).formfield_for_foreignkey( - db_field, request, **kwargs - ) +# def formfield_for_foreignkey(self, db_field, request, **kwargs): +# print(db_field.name) +# # print(request) +# # print(kwargs) +# if db_field.name in ["code_repo", "documentation_repo"]: +# github_app_installation_id = request.resolver_match.kwargs.get("object_id") +# kwargs["queryset"] = models.GithubRepository.objects.filter( +# owner_id=github_app_installation_id +# ) +# return super(GithubRepoMapAdmin, self).formfield_for_foreignkey( +# db_field, request, **kwargs +# ) - # def get_queryset(self, request): - # qs = super(GithubRepoMapAdmin, self).get_queryset(request) - # return qs.filter(=request.user) +# # def get_queryset(self, request): +# # qs = super(GithubRepoMapAdmin, self).get_queryset(request) +# # return qs.filter(=request.user) -class GithubAppInstallationForm(forms.ModelForm): - class Meta: - model = models.GithubAppInstallation - exclude = ["account_id"] +# class GithubAppInstallationForm(forms.ModelForm): +# class Meta: +# model = models.GithubAppInstallation +# exclude = ["account_id"] -class GithubAppInstallationAdmin(admin.ModelAdmin): - list_display = ( - "account_name", - "created_at", - ) - readonly_fields = ( - "account_name", - "installation_id", - "account_type", - "state", - "creator", - ) +# class GithubAppInstallationAdmin(admin.ModelAdmin): +# list_display = ( +# "account_name", +# "created_at", +# ) +# readonly_fields = ( +# "account_name", +# "installation_id", +# "account_type", +# "state", +# "creator", +# ) - form = GithubAppInstallationForm - inlines = [ - GithubRepoMapAdmin, - ] +# form = GithubAppInstallationForm +# inlines = [ +# GithubRepoMapAdmin, +# ] - def get_queryset(self, request): - qs = super(GithubAppInstallationAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - else: - return qs.filter(creator=request.user.github_user) +# def get_queryset(self, request): +# qs = super(GithubAppInstallationAdmin, self).get_queryset(request) +# if request.user.is_superuser: +# return qs +# else: +# return qs.filter(creator=request.user.github_user) - def get_form(self, request, obj=None, **kwargs): - form = super(GithubAppInstallationAdmin, self).get_form(request, obj, **kwargs) - # form.base_fields['theme'].queryset = Theme.objects.filter(name__iexact='company') - return form +# def get_form(self, request, obj=None, **kwargs): +# form = super(GithubAppInstallationAdmin, self).get_form(request, obj, **kwargs) +# # form.base_fields['theme'].queryset = Theme.objects.filter(name__iexact='company') +# return form -admin.site.register(models.GithubAppInstallation, GithubAppInstallationAdmin) -admin.site.register(models.GithubUserAccessToken) -admin.site.register(models.GithubUserRefreshToken) -admin.site.register(models.GithubInstallationToken) -admin.site.register(models.GithubPullRequest) -admin.site.register(models.MonitoredPullRequest) -admin.site.register(models.GithubCheckRun) -# admin.site.register(models.GithubRepoMap) +# admin.site.register(models.GithubAppInstallation, GithubAppInstallationAdmin) +# admin.site.register(models.GithubUserAccessToken) +# admin.site.register(models.GithubUserRefreshToken) +# admin.site.register(models.GithubInstallationToken) +# admin.site.register(models.GithubPullRequest) +# admin.site.register(models.MonitoredPullRequest) +# admin.site.register(models.GithubCheckRun) +# # admin.site.register(models.GithubRepoMap) diff --git a/app/apps.py b/app/apps.py index d118638..a40177c 100644 --- a/app/apps.py +++ b/app/apps.py @@ -5,5 +5,4 @@ class AppConfig(AppConfig): name = "app" def ready(self): - # import app.jobs # noqa - pass + import app.signals # noqa diff --git a/app/github_jobs.py b/app/github_jobs.py new file mode 100644 index 0000000..9d8bb9e --- /dev/null +++ b/app/github_jobs.py @@ -0,0 +1,50 @@ +import logging +import django + +import django_rq +import github +from django.conf import settings +from django_rq import job + +from app import models as app_models +from allauth.socialaccount import models as social_models + +from app import lib as app_lib + +logger = logging.getLogger(__name__) + + +@job +def sync_repositories_for_installation(social_account: social_models.SocialAccount): + github_instance = github.Github(social_account.socialtoken_set.last()) + existing_mapping = list( + app_models.UserRepoAccess.objects.filter( + social_account=social_account + ).values_list("id", flat=True) + ) + # for project in gitlab_instance.projects.list(all=True, membership=True): + # (repo_instance, _) = app_models.Repository.objects.update_or_create( + # repo_id=project.id, + # app=social_models.SocialApp.objects.get(provider="gitlab"), + # defaults={ + # "repo_name": project.name.replace(" ", ""), + # "repo_full_name": project.path_with_namespace, + # }, + # ) + # project_perms = project._attrs["permissions"] + # user_access_level = max( + # (project_perms.get("project_access") or {}).get("access_level", 0), + # (project_perms.get("group_access") or {}).get("access_level", 0), + # ) + + # (instance, is_new) = app_models.UserRepoAccess.objects.update_or_create( + # repository=repo_instance, + # social_account=social_account, + # defaults={"access": user_access_level}, + # ) + if True: + if is_new is False: + existing_mapping.pop(existing_mapping.index(instance.id)) + app_models.UserRepoAccess.objects.filter(id__in=existing_mapping).update( + access=app_models.UserRepoAccess.AccessLevel.NO_ACCESS + ) diff --git a/app/github_views.py b/app/github_views.py new file mode 100644 index 0000000..2fc82ed --- /dev/null +++ b/app/github_views.py @@ -0,0 +1,363 @@ +import datetime +import json +import logging +import uuid +from distutils.command.clean import clean +from typing import Any, Dict +from urllib.parse import parse_qs + +import django_rq +import github +import requests +from analytics import events as analytics_events +from analytics.decorators import log_http_event +from django.conf import settings +from django.contrib.auth import get_user_model, login +from django.contrib.auth import models as auth_models +from django.contrib.auth import views as auth_views +from django.contrib.auth.decorators import login_required +from django.db import transaction +from django.db.models import Q +from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse +from django.shortcuts import get_object_or_404, render +from django.urls import reverse +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views import View +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import FormView, TemplateView +from requests.models import PreparedRequest +from allauth.socialaccount import models as allauth_social_models +import lib + +from . import jobs as app_jobs +from . import models as app_models + +logger = logging.getLogger(__name__) + + +@method_decorator(csrf_exempt, name="dispatch") +class MarketplaceCallbackView(View): + def post(self, request, *args, **kwargs): + body = request.body + is_verified = lib.verify_signature( + request.headers["X-Hub-Signature-256"], + body, + settings.GITHUB_CREDS["marketplace_signature_secret"], + ) + if not is_verified: + return HttpResponse("Invalid signature", status=403) + payload = json.loads(body) + app_models.GithubMarketplaceEvent.objects.create(payload=payload) + return JsonResponse({"status": True}) + + +@method_decorator(csrf_exempt, name="dispatch") +class AuthCallback(View): + def get_okind_ekind(view, request, *args, **kwargs): + if "installation_id" in request.GET: + if request.GET.get("setup_action") == "update": + return ( + analytics_events.OKind.INSTALLATION, + analytics_events.EKind.UPDATE, + ) + else: + return ( + analytics_events.OKind.INSTALLATION, + analytics_events.EKind.CREATE, + ) + return (analytics_events.OKind.USER, analytics_events.EKind.LOGIN) + + @log_http_event(oid=1, okind_ekind=get_okind_ekind) + def get(self, request, *args, **kwargs): + """ + This method is invoked when the user installs the application and + generates the auth token for the user to be used in the future. + Expected inputs as urlparams: + - code: str + - installation_id: int + - setup_action: str + """ + code: str = request.GET["code"] + + # logger.info(request.GET) + # github_app_auth_user = app_models.GithubAppAuth.objects.create( + # code=code, + # installation_id=installation_id, + # setup_action=setup_action, + # ) + # Get the App installation account for managing the various scopes for a user + # Example, User A can be a part of 2 installations. So, the user will see them as two different projects + # in the UI. + # The user's account using the `code` recieved in the step above + # logger.info(request.get_host()) + social_app = allauth_social_models.SocialApp.objects.get(provider="github") + payload = { + "client_id": social_app.client_id, + "client_secret": social_app.secret, + "code": code, + "redirect_uri": f"https://{request.get_host()}/app/callback/", + "state": uuid.uuid4().__str__(), + } + resp = requests.post( + "https://github.com/login/oauth/access_token", json=payload + ) + logger.info(resp.text) + if resp.ok: + response = parse_qs(resp.text) + if "error" in response: + logger.error(response) + else: + now = timezone.now() + access_token = response["access_token"][0] + logger.info(response) + access_token_expires_at = now + datetime.timedelta( + seconds=int(response["expires_in"][0]) + ) + refresh_token = response["refresh_token"][0] + refresh_token_expires_at = now + datetime.timedelta( + seconds=int(response["refresh_token_expires_in"][0]) + ) + logger.info(access_token) + github_instance = github.Github(access_token) + user_instance = github_instance.get_user() + with transaction.atomic(): + # access_group = auth_models.Group.objects.get(name="github_user") + # TODO: Ideally a new user should be created/verified by email. Can lead to issues + # assert False, user_instance.login + (auth_user_instance, _) = get_user_model().objects.get_or_create( + username=user_instance.login, + defaults={ + "is_active": True, + "is_staff": False, + }, + ) + ( + social_account, + _, + ) = allauth_social_models.SocialAccount.objects.update_or_create( + user=auth_user_instance, + provider="github", + uid=user_instance.id, + defaults={ + "extra_data": user_instance.raw_data, + }, + ) + allauth_social_models.SocialToken.objects.update_or_create( + account=social_account, + app=social_app, + defaults={ + "token": access_token, + "token_secret": refresh_token, + "expires_at": access_token_expires_at, + }, + ) + login( + request, + auth_user_instance, + backend="allauth.account.auth_backends.AuthenticationBackend", + ) + if "installation_id" in request.GET: + installation_id: int = request.GET["installation_id"] + setup_action: str = request.GET["setup_action"] + + github_installation_manager = lib.GithubInstallationManager( + installation_id=installation_id, user_token=access_token + ) + installation_details = ( + github_installation_manager.get_installation_details() + ) + account_details = installation_details["account"] + account_name = account_details["login"] + account_id = account_details["id"] + account_type = account_details["type"] + avatar_url = account_details["avatar_url"] + + with transaction.atomic(): + + ( + installation_instance, + is_new, + ) = app_models.AppInstallation.objects.update_or_create( + installation_id=installation_id, + account_id=account_id, + social_app=social_app, + defaults={ + "account_name": account_name, + "state": app_models.AppInstallation.InstallationState.INSTALLED, + "account_type": account_type, + "avatar_url": avatar_url, + "creator": auth_user_instance, + }, + ) + # ( + installation_instance.save() + # installation_instance.update_token() + # if is_new: + django_rq.enqueue( + app_jobs.sync_repositories_for_installation, + installation_instance, + social_account, + ) + app_models.AppUser.objects.update_or_create( + user=auth_user_instance, + installation=installation_instance, + ) + for installation in user_instance.get_installations(): + if installation.app_id == settings.GITHUB_CREDS["app_id"]: + installation_instance = app_models.AppInstallation.objects.get( + account_id=installation.target_id, + installation_id=installation.id, + ) + app_models.AppUser.objects.update_or_create( + user=auth_user_instance, + installation=installation_instance, + ) + else: + logger.error(f"Unable to get token from the code: {resp.text}") + return HttpResponseRedirect(reverse("initiate_github_login")) + redirect_url = None + + if "state" in request.GET: + next_url = app_models.GithubLoginState.objects.get( + state=request.GET["state"] + ).redirect_url + if next_url is not None and next_url != "": + redirect_url = next_url + return HttpResponseRedirect(redirect_url or "/") + + +@method_decorator(csrf_exempt, name="dispatch") +class WebhookCallback(View): + def post(self, request, *args, **kwargs): + headers = request.headers + body = request.body + is_verified = lib.verify_signature( + headers["X-Hub-Signature-256"], + body, + settings.GITHUB_CREDS["app_signature_secret"], + ) + payload = json.loads(body) + if not is_verified: + return HttpResponse("Invalid signature", status=403) + logger.info( + "Recieved Github webhook", + extra={"data": {"headers": headers, "payload": payload}}, + ) + EVENT_TYPE = headers.get("X-Github-Event", None) + # header is "installation_repositories" -> Updated the repositories installed for the installation + interesting_events = [ + "pull_request", + "installation_repositories", + "check_suite", + "installation", + ] + if EVENT_TYPE in interesting_events: + get_installation_instance = ( + lambda data: app_models.AppInstallation.objects.get( + installation_id=data["installation"]["id"], + social_app__provider="github", + ) + ) + installation_instance = get_installation_instance(payload) + if EVENT_TYPE == "installation": + should_save = False + if payload["action"] == "deleted": + installation_instance.state = ( + app_models.AppInstallation.InstallationState.UNINSTALLED + ) + should_save = True + elif payload["action"] == "suspend": + installation_instance.state = ( + app_models.AppInstallation.InstallationState.SUSPENDED + ) + should_save = True + elif payload["action"] == "unsuspend": + installation_instance.state = ( + app_models.AppInstallation.InstallationState.INSTALLED + ) + should_save = True + if should_save: + installation_instance.save() + elif EVENT_TYPE == "pull_request": + pull_request_data = payload["pull_request"] + (github_repo, _) = app_models.Repository.objects.update_or_create( + repo_id=payload["repository"]["id"], + app=allauth_social_models.SocialApp.objects.get(provider="github"), + defaults={ + "repo_full_name": payload["repository"]["full_name"], + "repo_name": payload["repository"]["name"], + }, + ) + (pr_instance, _,) = app_models.PullRequest.objects.update_or_create( + pr_id=pull_request_data["id"], + pr_number=pull_request_data["number"], + repository=github_repo, + defaults={ + "pr_head_commit_sha": pull_request_data["head"]["sha"], + "pr_title": pull_request_data["title"], + "pr_body": pull_request_data["body"], + "pr_state": pull_request_data["state"], + "pr_created_at": pull_request_data["created_at"], + "pr_updated_at": pull_request_data["updated_at"], + "pr_merged_at": pull_request_data["merged_at"], + "pr_closed_at": pull_request_data["closed_at"], + "pr_merged": pull_request_data["merged"], + "pr_owner_username": pull_request_data["user"]["login"], + }, + ) + django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) + elif EVENT_TYPE == "installation_repositories": + # Repositories changed. Sync again. + django_rq.enqueue( + app_jobs.sync_repositories_for_installation, + installation_instance, + ) + elif EVENT_TYPE == "check_suite": + if payload.get("action") == "requested": + + repository_data = payload.get("repository", {}) + (github_repo, _,) = app_models.Repository.objects.update_or_create( + repo_id=repository_data["id"], + # owner=installation_instance, + defaults={ + "repo_name": repository_data["name"], + "repo_full_name": repository_data["full_name"], + }, + ) + prs = payload.get("check_suite", {}).get("pull_requests", []) + + head_commit_details = payload["check_suite"]["head_commit"] + head_commit_data = { + "pr_head_commit_sha": head_commit_details["id"], + # "pr_head_tree_sha": head_commit_details["tree_id"], + "pr_head_commit_message": head_commit_details["message"], + "pr_head_modified_on": datetime.datetime.strptime( + head_commit_details["timestamp"], "%Y-%m-%dT%H:%M:%S%z" + ), + } + + logger.info( + f"Found PRs for commit ID: {head_commit_details['id']}", + extra={"data": {"pull_requests": prs}}, + ) + # is_documentation_repo = + if ( + github_repo.code_repos.exists() + or github_repo.documentation_repos.exists() + ): + for pr in prs: + pr_id = pr.get("id") + pr_number = pr.get("number") + with transaction.atomic(): + ( + pr_instance, + is_new, + ) = app_models.PullRequest.objects.update_or_create( + pr_id=pr_id, + pr_number=pr_number, + repository=github_repo, + defaults={**head_commit_data}, + ) + django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) + return JsonResponse({"status": True}) diff --git a/app/gitlab_jobs.py b/app/gitlab_jobs.py new file mode 100644 index 0000000..4cad20b --- /dev/null +++ b/app/gitlab_jobs.py @@ -0,0 +1,149 @@ +from typing import Optional +from re import L +from django_rq import job +import gitlab +from app import models as app_models +from allauth.socialaccount import models as social_models +from app import lib as app_lib +from django.conf import settings + + +@job +def sync_repositories_for_installation(social_account: social_models.SocialAccount): + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(social_account).token + ) + existing_mapping = list( + app_models.UserRepoAccess.objects.filter( + social_account=social_account + ).values_list("id", flat=True) + ) + for project in gitlab_instance.projects.list(all=True, membership=True): + (repo_instance, _) = app_models.Repository.objects.update_or_create( + repo_id=project.id, + app=social_models.SocialApp.objects.get(provider="gitlab"), + defaults={ + "repo_name": project.name.replace(" ", ""), + "repo_full_name": project.path_with_namespace, + }, + ) + project_perms = project._attrs["permissions"] + user_access_level = max( + (project_perms.get("project_access") or {}).get("access_level", 0), + (project_perms.get("group_access") or {}).get("access_level", 0), + ) + + (instance, is_new) = app_models.UserRepoAccess.objects.update_or_create( + repository=repo_instance, + social_account=social_account, + defaults={"access": user_access_level}, + ) + if is_new is False: + existing_mapping.pop(existing_mapping.index(instance.id)) + app_models.UserRepoAccess.objects.filter(id__in=existing_mapping).update( + access=app_models.UserRepoAccess.AccessLevel.NO_ACCESS + ) + + +@job +def sync_prs_for_repository( + project_id: int, +): + repository_instance = app_models.Repository.objects.get( + repo_id=project_id, + app__provider="gitlab", + ) + highest_access_user = ( + repository_instance.userrepoaccess_set.all().order_by("-access").first() + ) + if highest_access_user is None: + return + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(highest_access_user.social_account).token + ) + project_instance = gitlab_instance.projects.get(repository_instance.repo_id) + for merge_request in project_instance.mergerequests.list(): + try: + head_commit = merge_request.commits().next() + except StopIteration: + head_commit = object() + (pr_instance, _) = app_models.PullRequest.objects.update_or_create( + pr_id=merge_request.id, + pr_number=merge_request.iid, + repository=repository_instance, + defaults={ + "pr_title": merge_request.title, + "pr_owner_username": merge_request.author["username"], + "pr_head_commit_sha": getattr(head_commit, "id", None) or "", + "pr_head_modified_on": getattr( + head_commit, "authored_date", merge_request.created_at + ), + "pr_head_commit_message": getattr( + head_commit, "title", merge_request.title + ) + or "", + "pr_body": merge_request.description, + "pr_state": merge_request.state, + "pr_created_at": merge_request.created_at, + "pr_updated_at": merge_request.updated_at, + "pr_merged_at": merge_request.merged_at, + "pr_closed_at": merge_request.closed_at, + "pr_merged": merge_request.state == "opened", + }, + ) + if repository_instance.code_repos.exists(): + app_models.MonitoredPullRequest.objects.update_or_create( + code_pull_request=pr_instance, + ) + # Setup a webhook for this project\ + webhook_url = ( + f"https://{settings.DEPLOYMENT_HOST_NAME}/app/gitlab/webhook-callback/" + ) + existing_hooks = project_instance.hooks.list() + for hook in existing_hooks: + if hook.url == webhook_url: + hook.merge_requests_events = True + hook.save() + if len(existing_hooks) == 0: + project_instance.hooks.create( + { + "url": webhook_url, + "merge_requests_events": True, + "push_events": False, + } + ) + + +@job +def monitored_pr_post_save( + monitored_pr_instance: app_models.MonitoredPullRequest, + user_instance: app_models.UserRepoAccess, + old_status: str, +): + if monitored_pr_instance.pull_request_status == old_status: + pass + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(user_instance.social_account).token + ) + project = gitlab_instance.projects.get( + monitored_pr_instance.code_pull_request.repository.repo_id + ) + mr = project.mergerequests.get(monitored_pr_instance.code_pull_request.pr_number) + + +@job +def merge_request_comment( + monitored_pr_instance: app_models.MonitoredPullRequest, + user_instance: app_models.UserRepoAccess, + comment_message: Optional[str], +): + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(user_instance.social_account).token + ) + project = gitlab_instance.projects.get( + monitored_pr_instance.code_pull_request.repository.repo_id + ) + mr = project.mergerequests.get(monitored_pr_instance.code_pull_request.pr_number) + if comment_message is not None: + mr.discussions.create({"body": comment_message}) + mr.pipelines.create() diff --git a/app/gitlab_views.py b/app/gitlab_views.py new file mode 100644 index 0000000..1150b17 --- /dev/null +++ b/app/gitlab_views.py @@ -0,0 +1,128 @@ +import datetime +import gitlab +from django.utils import timezone +from app import lib as app_lib +from django.http import HttpResponseRedirect, HttpResponse, JsonResponse +from django.urls import reverse +from django.views import View +from app import models as app_models +from django.conf import settings +from requests.models import PreparedRequest +import json +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator +import time +from allauth.socialaccount import models as social_models +from app import gitlab_jobs +import django_rq +from app import jobs as app_jobs + + +@method_decorator(csrf_exempt, name="dispatch") +class WebhookCallback(View): + def post(self, request, *args, **kwargs): + headers = request.headers + event_header = headers["X-Gitlab-Event"] + request_body = json.loads(request.body.decode("utf-8")) + # gitlab_instance = gitlab.Gitlab(oauth_token=app_lib.get_active_token()) + print(json.dumps(request_body)) + if ( + event_header == "Merge Request Hook" + and request_body["object_kind"] == "merge_request" + ): + object_attrs = request_body["object_attributes"] + project_name = object_attrs["target"]["path_with_namespace"] + repo = app_models.Repository.objects.get( + repo_full_name=project_name, app__provider="gitlab" + ) + merge_request_id = request_body["object_attributes"]["iid"] + old_instance = app_models.PullRequest.objects.filter( + pr_number=merge_request_id, + pr_id=request_body["object_attributes"]["id"], + repository=repo, + ).first() + + (pr_instance, _) = app_models.PullRequest.objects.update_or_create( + pr_number=merge_request_id, + pr_id=request_body["object_attributes"]["id"], + repository=repo, + defaults={ + "pr_head_commit_sha": object_attrs["last_commit"]["id"], + "pr_head_modified_on": object_attrs["last_commit"]["timestamp"], + "pr_head_commit_message": object_attrs["last_commit"]["title"], + "pr_title": object_attrs["title"], + "pr_body": object_attrs["description"], + "pr_state": object_attrs["state"], + "pr_created_at": timezone.datetime.strptime( + object_attrs["created_at"], "%Y-%m-%d %H:%M:%S %Z" + ), + "pr_updated_at": timezone.datetime.strptime( + object_attrs["updated_at"], "%Y-%m-%d %H:%M:%S %Z" + ), + }, + ) + if repo.code_repos.exists(): + app_models.MonitoredPullRequest.objects.update_or_create( + code_pull_request=pr_instance + ) + monitored_prs = [] + try: + monitored_pr_instance = pr_instance.monitored_code + if old_instance.pr_head_commit_sha != pr_instance.pr_head_commit_sha: + monitored_pr_instance.pull_request_status = ( + app_models.MonitoredPullRequest.PullRequestStatus.STALE_APPROVAL + ) + monitored_pr_instance.save() + monitored_prs.append(monitored_pr_instance) + except: + pass + for doc_pr_instance in pr_instance.monitored_documentation.all(): + # doc_pr_instance = pr_instance.monitored_code + if old_instance.pr_head_commit_sha != pr_instance.pr_head_commit_sha: + doc_pr_instance.pull_request_status = ( + app_models.MonitoredPullRequest.PullRequestStatus.STALE_CODE + ) + doc_pr_instance.save() + monitored_prs.append(doc_pr_instance) + for pr in monitored_prs: + repo_users = ( + pr.code_pull_request.repository.userrepoaccess_set.order_by( + "-access" + ) + ) + if repo_users.exists(): + highest_access = repo_users.first() + provider_comment_action_map = { + "gitlab": gitlab_jobs.merge_request_comment, + "github": app_jobs.merge_request_comment, + } + django_rq.enqueue( + provider_comment_action_map[ + pr.code_pull_request.repository.app.provider + ], + args=(pr, highest_access, None), + ) + return JsonResponse({}) + + +@method_decorator(csrf_exempt, name="dispatch") +class GitlabPipelineRequest(View): + def post(self, request, *args, **kwargs): + body = json.loads(request.body.decode("utf-8")) + gitlab_instance = gitlab.Gitlab(job_token=body["job_token"]) + # Get the job data to ensure no dirty access + job_data = gitlab_instance.http_get("/job/") + if job_data["pipeline"]["source"] == "merge_request_event": + repo_instance = app_models.Repository.objects.get( + repo_id=body["project_id"] + ) + # This assumes the webhook has been updated prior to the pipline being invoked. + # Potential race condition + pr_instance = app_models.PullRequest.objects.get( + pr_id=body["merge_request_id"], + pr_number=body["merge_request_iid"], + repository=repo_instance, + pr_head_commit_sha=body["source_sha"], + ) + return HttpResponse(0 if pr_instance.monitored_code.is_approved else 1) + return HttpResponse(0) diff --git a/app/jobs.py b/app/jobs.py index bb447bf..a7d1c96 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -15,33 +15,52 @@ @job def sync_repositories_for_installation( - installation_instance: app_models.GithubAppInstallation, + installation_instance: app_models.AppInstallation, + social_account, ): logger.info(f"GithubAppInstallation signal received for {installation_instance}") + creator = installation_instance.creator + creator_social_account = creator.socialaccount_set.get(provider="github") github_manager = app_lib.GithubDataManager( installation_instance.installation_id, - installation_instance.creator.get_active_access_token(), + app_lib.get_active_token(creator_social_account).token, ) - github_manager.sync_repositories() + new_repos = github_manager.sync_repositories() + for repo in new_repos: + # access_evaluation for github is pending + app_models.UserRepoAccess.objects.update_or_create( + repository=repo, + social_account=social_account, + access=50, + ) @job def sync_prs_for_repository(repository_id: int): logger.info(f"Sync PR job received for repository_id: {repository_id}") - instance = app_models.GithubRepository.objects.get(id=repository_id) - installation_instance = instance.owner + repository_instance = app_models.Repository.objects.get( + repo_id=repository_id, + app__provider="github", + ) + highest_access_user = ( + repository_instance.userrepoaccess_set.all().order_by("-access").first() + ) + if highest_access_user is None: + return + # installation_instance = instance.owner github_data_manager = app_lib.GithubDataManager( - installation_id=installation_instance.installation_id, - user_token=installation_instance.creator.get_active_access_token(), + # Installation_id not required for syncing the PRs + installation_id=None, + user_token=app_lib.get_active_token(highest_access_user.social_account).token, ) - all_prs = github_data_manager.sync_open_prs(instance) + all_prs = github_data_manager.sync_open_prs(repository_instance) for pr_id in all_prs: django_rq.enqueue(on_pr_update, pr_id) @job def on_pr_update(pull_request_id: int): - instance = app_models.GithubPullRequest.objects.get(id=pull_request_id) + instance = app_models.PullRequest.objects.get(id=pull_request_id) for documentation_pr in app_models.MonitoredPullRequest.objects.filter( documentation_pull_request=instance ).iterator(): @@ -60,12 +79,12 @@ def on_pr_update(pull_request_id: int): code_pr.save() if instance.repository.code_repos.exists(): - installation_instance = instance.repository.code_repos.last().integration + # installation_instance = instance.repository.code_repos.last().integration ( monitored_pr_instance, is_new, ) = app_models.MonitoredPullRequest.objects.get_or_create( - code_pull_request=instance, integration=installation_instance + code_pull_request=instance ) if is_new is False: monitored_pr_instance.save() @@ -75,13 +94,13 @@ def on_pr_update(pull_request_id: int): @job def monitored_pr_post_save(instance_id: int): monitored_pr_instance = app_models.MonitoredPullRequest.objects.get(id=instance_id) - (instance, is_new) = app_models.GithubCheckRun.objects.get_or_create( + (instance, is_new) = app_models.PullRequestCheck.objects.get_or_create( ref_pull_request=monitored_pr_instance, run_sha=monitored_pr_instance.code_pull_request.pr_head_commit_sha, ) if is_new is False: # Here, we can close the previous checks if any - # app_models.GithubCheckRun.objects.filter(ref_pull_request=instance).exclude( + # app_models.PullRequestCheck.objects.filter(ref_pull_request=instance).exclude( # run_sha=instance.code_pull_request.pr_head_commit_sha, # ) instance.save() @@ -89,17 +108,22 @@ def monitored_pr_post_save(instance_id: int): # @job # def update_github_check(github_check_run_id: int): - # instance = app_models.GithubCheckRun.objects.get(id=github_check_run_id) - token = ( - instance.ref_pull_request.code_pull_request.repository.owner.creator.access_token + # instance = app_models.PullRequestCheck.objects.get(id=github_check_run_id) + highest_access_account = ( + instance.ref_pull_request.code_pull_request.repository.userrepoaccess_set.order_by( + "-access" + ) + .first() + .social_account ) + token = app_lib.get_active_token(highest_access_account).token github_app_instance = github.Github(token) github_repo = github_app_instance.get_repo( instance.ref_pull_request.code_pull_request.repository.repo_id ) - current_pr = github_repo.get_pull( - instance.ref_pull_request.code_pull_request.pr_number - ) + # current_pr = github_repo.get_pull( + # instance.ref_pull_request.code_pull_request.pr_number + # ) check_run = github_repo.get_check_run(instance.run_id) @@ -162,5 +186,17 @@ def monitored_pr_post_save(instance_id: int): @job -def test_job(a, b): - assert False, a + b +def merge_request_comment( + monitored_pr_instance: app_models.MonitoredPullRequest, + user_instance: app_models.UserRepoAccess, + comment_message: str, +): + github_instance = github.Github( + app_lib.get_active_token(user_instance.social_account).token + ) + repo = github_instance.get_repo( + monitored_pr_instance.code_pull_request.repository.repo_id + ) + pr = repo.get_pull(monitored_pr_instance.code_pull_request.pr_number) + pr.create_issue_comment(comment_message) + django_rq.enqueue(monitored_pr_post_save, monitored_pr_instance.id) diff --git a/app/lib.py b/app/lib.py index a58b4e1..a74821a 100644 --- a/app/lib.py +++ b/app/lib.py @@ -1,35 +1,44 @@ from . import models as app_models import lib import github +from django.utils import timezone +from requests_oauthlib import OAuth2Session +from allauth.socialaccount import models as social_models +import datetime class GithubDataManager: def __init__(self, installation_id: int, user_token: str): self.installation_id = installation_id self.github_token = user_token - self.installation_instance = app_models.GithubAppInstallation.objects.get( - installation_id=self.installation_id - ) + if installation_id: + self.installation_instance = app_models.AppInstallation.objects.get( + installation_id=self.installation_id + ) + self.github_manager_instance = lib.GithubInstallationManager( + self.installation_id, self.github_token + ) self.github_instance = github.Github( self.github_token, ) - self.github_manager_instance = lib.GithubInstallationManager( - self.installation_id, self.github_token - ) + self.social_app = social_models.SocialApp.objects.get(provider="github") def sync_repositories(self): repo_generator = self.github_manager_instance.get_repositories() + repos = [] for repo in repo_generator: - app_models.GithubRepository.objects.get_or_create( + (repo_instance, _) = app_models.Repository.objects.get_or_create( repo_id=repo["id"], repo_name=repo["name"], repo_full_name=repo["full_name"], - owner=self.installation_instance, + app=self.social_app, ) + repos.append(repo_instance) + return repos # a function called sync_open_prs which takes input the GithubRepository and syncs its open pull requests - def sync_open_prs(self, repo: app_models.GithubRepository): + def sync_open_prs(self, repo: app_models.Repository): all_repo_ids = [] for pr in self.github_instance.get_repo(repo.repo_full_name).get_pulls("open"): extra_data = { @@ -46,7 +55,7 @@ def sync_open_prs(self, repo: app_models.GithubRepository): "pr_merged": pr.merged, "pr_owner_username": pr.user.login, } - (instance, is_new) = app_models.GithubPullRequest.objects.update_or_create( + (instance, is_new) = app_models.PullRequest.objects.update_or_create( pr_id=pr.id, pr_number=pr.number, repository=repo, @@ -54,3 +63,42 @@ def sync_open_prs(self, repo: app_models.GithubRepository): ) all_repo_ids.append(instance.id) return all_repo_ids + + +def get_active_token(social_account): + last_token = social_account.socialtoken_set.last() + if last_token.expires_at > timezone.now(): + return last_token + social_app_instance = social_models.SocialApp.objects.get( + provider=social_account.provider + ) + extra = { + "client_id": social_app_instance.client_id, + "client_secret": social_app_instance.secret, + } + client = OAuth2Session( + social_app_instance.client_id, + token={ + "access_token": last_token.token, + "refresh_token": last_token.token_secret, + "token_type": "Bearer", + "expires_in": "-30", # initially 3600, need to be updated by you + }, + ) + refresh_url = ( + "https://gitlab.com/oauth/token" + if social_account.provider == "gitlab" + else "https://github.com/login/oauth/access_token" + ) + response = client.refresh_token(refresh_url, **extra) + (token, _) = social_models.SocialToken.objects.update_or_create( + app=last_token.app, + account=last_token.account, + defaults={ + "token": response["access_token"], + "token_secret": response["refresh_token"], + "expires_at": timezone.now() + + datetime.timedelta(seconds=response["expires_in"]), + }, + ) + return token diff --git a/app/migrations/0020_appinstallation_appuser_and_more.py b/app/migrations/0020_appinstallation_appuser_and_more.py new file mode 100644 index 0000000..9bc2a96 --- /dev/null +++ b/app/migrations/0020_appinstallation_appuser_and_more.py @@ -0,0 +1,127 @@ +# Generated by Django 4.0.4 on 2022-05-25 06:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0019_githubmarketplaceevent'), + ] + + operations = [ + migrations.CreateModel( + name='AppInstallation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('installation_id', models.BigIntegerField(db_index=True, unique=True)), + ('state', models.CharField(choices=[('INSTALLED', 'Installed'), ('UNINSTALLED', 'Uninstalled'), ('SUSPENDED', 'Suspended')], max_length=20)), + ('account_name', models.CharField(max_length=200)), + ('account_id', models.BigIntegerField()), + ('account_type', models.CharField(max_length=20)), + ('avatar_url', models.CharField(max_length=200)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('social_app', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='socialaccount.socialapp')), + ], + options={ + 'unique_together': {('social_app', 'installation_id')}, + }, + ), + migrations.CreateModel( + name='AppUser', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('installation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.appinstallation')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('installation', 'user')}, + }, + ), + migrations.RenameModel( + old_name='GithubPullRequest', + new_name='PullRequest', + ), + migrations.RenameModel( + old_name='GithubCheckRun', + new_name='PullRequestCheck', + ), + migrations.RemoveField( + model_name='githubappinstallation', + name='creator', + ), + migrations.AlterUniqueTogether( + name='githubappuser', + unique_together=None, + ), + migrations.RemoveField( + model_name='githubappuser', + name='github_user', + ), + migrations.RemoveField( + model_name='githubappuser', + name='installation', + ), + migrations.RemoveField( + model_name='githubinstallationtoken', + name='github_app_installation', + ), + migrations.DeleteModel( + name='GithubLoginState', + ), + migrations.RemoveField( + model_name='githubuser', + name='user', + ), + migrations.RemoveField( + model_name='githubuseraccesstoken', + name='github_user', + ), + migrations.RemoveField( + model_name='githubuserrefreshtoken', + name='github_user', + ), + migrations.RenameModel( + old_name='GithubRepository', + new_name='Repository', + ), + migrations.DeleteModel( + name='GithubAppUser', + ), + migrations.DeleteModel( + name='GithubInstallationToken', + ), + migrations.DeleteModel( + name='GithubUser', + ), + migrations.DeleteModel( + name='GithubUserAccessToken', + ), + migrations.DeleteModel( + name='GithubUserRefreshToken', + ), + migrations.AlterField( + model_name='githubrepomap', + name='integration', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='app.appinstallation'), + ), + migrations.AlterField( + model_name='monitoredpullrequest', + name='integration', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.appinstallation'), + ), + migrations.AlterField( + model_name='repository', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='app.appinstallation'), + ), + migrations.DeleteModel( + name='GithubAppInstallation', + ), + ] diff --git a/app/migrations/0021_userrepoaccess.py b/app/migrations/0021_userrepoaccess.py new file mode 100644 index 0000000..a85ca4e --- /dev/null +++ b/app/migrations/0021_userrepoaccess.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0.4 on 2022-05-28 06:35 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0020_appinstallation_appuser_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='UserRepoAccess', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.repository')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('repository', 'user')}, + }, + ), + ] diff --git a/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py b/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py new file mode 100644 index 0000000..08532ae --- /dev/null +++ b/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.0.4 on 2022-05-28 06:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('app', '0021_userrepoaccess'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='repository', + unique_together=set(), + ), + migrations.AddField( + model_name='repository', + name='app', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.PROTECT, to='socialaccount.socialapp'), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='repository', + unique_together={('repo_id', 'app')}, + ), + migrations.RemoveField( + model_name='repository', + name='owner', + ), + ] diff --git a/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py b/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py new file mode 100644 index 0000000..0efdb46 --- /dev/null +++ b/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.0.4 on 2022-05-28 10:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0022_alter_repository_unique_together_repository_app_and_more'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='githubrepomap', + unique_together={('code_repo', 'documentation_repo')}, + ), + migrations.RemoveField( + model_name='githubrepomap', + name='integration', + ), + ] diff --git a/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py b/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py new file mode 100644 index 0000000..35ea893 --- /dev/null +++ b/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.0.4 on 2022-05-28 13:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('app', '0023_alter_githubrepomap_unique_together_and_more'), + ] + + operations = [ + migrations.RenameModel( + old_name='GithubRepoMap', + new_name='MonitoredRepositoryMap', + ), + migrations.AlterUniqueTogether( + name='userrepoaccess', + unique_together=set(), + ), + migrations.AddField( + model_name='userrepoaccess', + name='access', + field=models.CharField(choices=[('OWNER', 'Owner'), ('MEMBER', 'Member')], default='MEMBER', max_length=20), + preserve_default=False, + ), + migrations.AddField( + model_name='userrepoaccess', + name='social_account', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='socialaccount.socialaccount'), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='userrepoaccess', + unique_together={('repository', 'social_account')}, + ), + migrations.RemoveField( + model_name='userrepoaccess', + name='user', + ), + ] diff --git a/app/migrations/0025_alter_userrepoaccess_access.py b/app/migrations/0025_alter_userrepoaccess_access.py new file mode 100644 index 0000000..71431c4 --- /dev/null +++ b/app/migrations/0025_alter_userrepoaccess_access.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.4 on 2022-05-28 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0024_rename_githubrepomap_monitoredrepositorymap_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='userrepoaccess', + name='access', + field=models.IntegerField(choices=[('50', 'Owner'), ('40', 'Maintainer'), ('30', 'Developer'), ('20', 'Reporter'), ('10', 'Guest'), ('5', 'Minimal'), ('0', 'No Access')], max_length=20), + ), + ] diff --git a/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py b/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py new file mode 100644 index 0000000..48c7415 --- /dev/null +++ b/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.4 on 2022-05-29 05:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0025_alter_userrepoaccess_access'), + ] + + operations = [ + migrations.RemoveField( + model_name='monitoredpullrequest', + name='integration', + ), + migrations.AlterField( + model_name='userrepoaccess', + name='access', + field=models.IntegerField(choices=[('50', 'Owner'), ('40', 'Maintainer'), ('30', 'Developer'), ('20', 'Reporter'), ('10', 'Guest'), ('5', 'Minimal'), ('0', 'No Access')]), + ), + ] diff --git a/app/migrations/0027_githubapptoken.py b/app/migrations/0027_githubapptoken.py new file mode 100644 index 0000000..a7b8e8e --- /dev/null +++ b/app/migrations/0027_githubapptoken.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.4 on 2022-06-03 04:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0026_remove_monitoredpullrequest_integration_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='GithubAppToken', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(max_length=200)), + ('expires_at', models.DateTimeField()), + ('secret', models.CharField(max_length=200)), + ('app', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='app_token', to='app.appinstallation')), + ], + ), + ] diff --git a/app/models.py b/app/models.py index 3def479..a106791 100644 --- a/app/models.py +++ b/app/models.py @@ -2,9 +2,9 @@ from email.policy import default import enum import logging +from typing import Optional import uuid from urllib.parse import parse_qs - import github import lib import requests @@ -12,107 +12,12 @@ from django.contrib.auth import get_user_model from django.db import models from django.utils import timezone +from allauth.socialaccount import models as social_models logger = logging.getLogger(__name__) -class Token(models.Model): - """ - This model takes care of the tokens generated by the Github applications. - """ - - token = models.CharField(max_length=255) - created_at = models.DateTimeField(auto_now_add=True) - expires_at = models.DateTimeField() - - class Meta: - abstract = True - - -class GithubUser(models.Model): - user = models.OneToOneField( - get_user_model(), on_delete=models.CASCADE, related_name="github_user" - ) - account_name = models.CharField(max_length=200) - account_id = models.BigIntegerField() - account_type = models.CharField(max_length=20) - avatar_url = models.CharField(max_length=200) - created_at = models.DateTimeField(auto_now_add=True) - - @property - def access_token(self): - try: - return ( - self.access_tokens.exclude(expires_at__lt=timezone.now()) - .latest("expires_at") - .token - ) - except: - return None - - def get_active_access_token(self): - access_token = self.access_token - if access_token is None: - return self.get_new_tokens()[0] - return access_token - - @property - def refresh_token(self): - try: - return ( - self.refresh_tokens.exclude(expires_at__lt=timezone.now()) - .latest("expires_at") - .token - ) - except: - return None - - def process_token_response(self, token_response): - now = timezone.now() - logger.info(token_response) - data = parse_qs(token_response) - access_token = GithubUserAccessToken.objects.create( - token=data["access_token"][0], - github_user=self, - expires_at=now + datetime.timedelta(seconds=int(data["expires_in"][0])), - ) - refresh_token = GithubUserRefreshToken.objects.create( - token=data["refresh_token"][0], - github_user=self, - expires_at=now - + datetime.timedelta(seconds=int(data["refresh_token_expires_in"][0])), - ) - return (access_token.token, refresh_token.token) - - def get_new_tokens(self): - refresh_token = self.refresh_tokens.exclude( - expires_at__lt=timezone.now() - ).latest("expires_at") - response = requests.post( - "https://github.com/login/oauth/access_token", - data={ - "refresh_token": refresh_token.token, - "grant_type": "refresh_token", - "client_id": settings.GITHUB_CREDS["client_id"], - "client_secret": settings.GITHUB_CREDS["client_secret"], - }, - ) - return self.process_token_response(response.content.decode()) - - -class GithubUserAccessToken(Token): - github_user = models.ForeignKey( - GithubUser, on_delete=models.CASCADE, related_name="access_tokens" - ) - - -class GithubUserRefreshToken(Token): - github_user = models.ForeignKey( - GithubUser, on_delete=models.CASCADE, related_name="refresh_tokens" - ) - - -class GithubAppInstallation(models.Model): +class AppInstallation(models.Model): """ This model takes care of the Github installations. """ @@ -122,8 +27,9 @@ class InstallationState(models.TextChoices): UNINSTALLED = "UNINSTALLED", "Uninstalled" SUSPENDED = "SUSPENDED", "Suspended" + social_app = models.ForeignKey(social_models.SocialApp, on_delete=models.PROTECT) installation_id = models.BigIntegerField(unique=True, db_index=True) - creator = models.ForeignKey(GithubUser, on_delete=models.PROTECT) + creator = models.ForeignKey(get_user_model(), on_delete=models.PROTECT) state = models.CharField(max_length=20, choices=InstallationState.choices) account_name = models.CharField(max_length=200) account_id = models.BigIntegerField() @@ -131,75 +37,78 @@ class InstallationState(models.TextChoices): avatar_url = models.CharField(max_length=200) created_at = models.DateTimeField(auto_now_add=True) + class Meta: + unique_together = ("social_app", "installation_id") + def __str__(self) -> str: - return f"{self.account_name}[{self.installation_id}] Owner: {self.creator.account_name}" + return f"{self.account_name}[{self.installation_id}] Owner: {self.creator.username}" - @property - def access_token(self): - try: - return ( - self.tokens.exclude(expires_at__lte=timezone.now()) - .latest("created_at") - .token - ) - except GithubInstallationToken.DoesNotExist: - self.update_token() - return self.get_latest_active_token() + def get_app_token(self): + if self.social_app.provider == "github": + return self.app_token.token + return None - def update_token(self): - """ - Fetches the latest access token for the installation. Returns true/false depending on whether - the requested token has been updated - """ - # User token not required for this step - github_installation_manager = lib.GithubInstallationManager( - installation_id=self.installation_id, user_token="" - ) - token, expires_at = github_installation_manager.get_installation_access_token() - return GithubInstallationToken.objects.get_or_create( - token=token, - expires_at=expires_at, - github_app_installation=self, - )[1] + +class GithubAppToken(models.Model): + app = models.OneToOneField( + AppInstallation, on_delete=models.CASCADE, related_name="app_token" + ) + token = models.CharField(max_length=200) + expires_at = models.DateTimeField() + secret = models.CharField(max_length=200) -class GithubAppUser(models.Model): - installation = models.ForeignKey(GithubAppInstallation, on_delete=models.CASCADE) - github_user = models.ForeignKey(GithubUser, on_delete=models.CASCADE) +class AppUser(models.Model): + installation = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) + user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("installation", "github_user") - - def __str__(self) -> str: - return f"{self.github_user.account_name}[{self.installation.installation_id}]" - return super().__str__() + unique_together = ("installation", "user") -class GithubInstallationToken(Token): - github_app_installation = models.ForeignKey( - GithubAppInstallation, on_delete=models.CASCADE, related_name="tokens" - ) - - -class GithubRepository(models.Model): +class Repository(models.Model): repo_id = models.BigIntegerField() repo_name = models.CharField(max_length=150) repo_full_name = models.CharField(max_length=200) - owner = models.ForeignKey(GithubAppInstallation, on_delete=models.PROTECT) + app = models.ForeignKey(social_models.SocialApp, on_delete=models.PROTECT) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("repo_id", "owner") + unique_together = ("repo_id", "app") def __str__(self) -> str: return self.repo_full_name + def pulls_page_url(self): + return f"/{self.app.provider}/{self.repo_full_name}/pulls/" + def get_url(self): - return f"https://github.com/{self.repo_full_name}/pulls/" + return f"https://{self.app.provider}.com/{self.repo_full_name}/" + + +class UserRepoAccess(models.Model): + class AccessLevel(models.TextChoices): + OWNER = 50, "Owner" + MAINTAINER = 40, "Maintainer" + DEVELOPER = 30, "Developer" + REPORTER = 20, "Reporter" + GUEST = 10, "Guest" + MINIMAL = 5, "Minimal" + NO_ACCESS = 0, "No Access" + + repository = models.ForeignKey(Repository, on_delete=models.CASCADE) + social_account = models.ForeignKey( + social_models.SocialAccount, on_delete=models.CASCADE + ) + access = models.IntegerField(choices=AccessLevel.choices) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = ("repository", "social_account") -class GithubRepoMap(models.Model): +class MonitoredRepositoryMap(models.Model): class IntegrationType(models.TextChoices): """ FULL integration restricts the PRs to process only after the related PR is merged @@ -209,21 +118,21 @@ class IntegrationType(models.TextChoices): FULL = "FULL", "Full" PARTIAL = "PARTIAL", "Partial" - integration = models.ForeignKey(GithubAppInstallation, on_delete=models.PROTECT) + # integration = models.ForeignKey(AppInstallation, on_delete=models.PROTECT) code_repo = models.ForeignKey( - GithubRepository, on_delete=models.PROTECT, related_name="code_repos" + Repository, on_delete=models.PROTECT, related_name="code_repos" ) documentation_repo = models.ForeignKey( - GithubRepository, on_delete=models.PROTECT, related_name="documentation_repos" + Repository, on_delete=models.PROTECT, related_name="documentation_repos" ) integration_type = models.CharField(max_length=20, choices=IntegrationType.choices) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("integration", "code_repo", "documentation_repo") + unique_together = ("code_repo", "documentation_repo") -class GithubPullRequest(models.Model): +class PullRequest(models.Model): pr_id = models.BigIntegerField() pr_number = models.BigIntegerField() pr_head_commit_sha = models.CharField(max_length=40) @@ -239,15 +148,16 @@ class GithubPullRequest(models.Model): pr_merged = models.BooleanField(default=False) pr_owner_username = models.CharField(max_length=100) updated_on = models.DateTimeField(auto_now=True) - repository = models.ForeignKey(GithubRepository, on_delete=models.PROTECT) + repository = models.ForeignKey(Repository, on_delete=models.PROTECT) def __str__(self) -> str: return f"{self.repository.repo_full_name}/{self.pr_number} @ {self.pr_head_commit_sha[:7]}" def get_url(self): - return ( - f"https://github.com/{self.repository.repo_full_name}/pull/{self.pr_number}" - ) + if self.repository.app.provider == "gitlab": + return f"https://gitlab.com/{self.repository.repo_full_name}/-/merge_requests/{self.pr_number}" + else: + return f"https://github.com/{self.repository.repo_full_name}/pull/{self.pr_number}" class MonitoredPullRequest(models.Model): @@ -268,12 +178,12 @@ class PullRequestStatus(models.TextChoices): MANUAL_APPROVAL = "MANUALLY_APPROVED", "Manual Approval" code_pull_request = models.OneToOneField( - GithubPullRequest, + PullRequest, on_delete=models.CASCADE, related_name="monitored_code", ) documentation_pull_request = models.ForeignKey( - GithubPullRequest, + PullRequest, on_delete=models.CASCADE, blank=True, null=True, @@ -284,7 +194,7 @@ class PullRequestStatus(models.TextChoices): choices=PullRequestStatus.choices, default=PullRequestStatus.NOT_CONNECTED, ) - integration = models.ForeignKey(GithubAppInstallation, on_delete=models.CASCADE) + # integration = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) class Meta: unique_together = ("code_pull_request", "documentation_pull_request") @@ -294,14 +204,6 @@ def __str__(self) -> str: def save(self, *args, **kwargs): if ( - self.documentation_pull_request is not None - and self.pull_request_status - == MonitoredPullRequest.PullRequestStatus.NOT_CONNECTED - ): - self.pull_request_status = ( - MonitoredPullRequest.PullRequestStatus.APPROVAL_PENDING - ) - elif ( self.documentation_pull_request is None and self.pull_request_status != MonitoredPullRequest.PullRequestStatus.MANUAL_APPROVAL @@ -310,6 +212,14 @@ def save(self, *args, **kwargs): self.pull_request_status = ( MonitoredPullRequest.PullRequestStatus.NOT_CONNECTED ) + if ( + self.documentation_pull_request is not None + and self.pull_request_status + == MonitoredPullRequest.PullRequestStatus.NOT_CONNECTED + ): + self.pull_request_status = ( + MonitoredPullRequest.PullRequestStatus.APPROVAL_PENDING + ) return super().save(*args, **kwargs) def get_display_name(self): @@ -341,7 +251,7 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -class GithubCheckRun(models.Model): +class PullRequestCheck(models.Model): run_id = models.BigIntegerField(blank=True, null=True) unique_id = models.UUIDField(default=uuid.uuid4) # Head of the request when it was run @@ -367,9 +277,19 @@ def save(self, *args, **kwargs): if not self.pk: # PK doesn't exist, new instance being saved to the DB. Create a new check - token = ( - self.ref_pull_request.code_pull_request.repository.owner.creator.access_token + from app import lib as app_lib # noqa + + highest_access_account = ( + self.ref_pull_request.code_pull_request.repository.userrepoaccess_set.order_by( + "-access" + ) + .first() + .social_account ) + token = app_lib.get_active_token(highest_access_account).token + # token = ( + # self.ref_pull_request.code_pull_request.repository.owner.creator.access_token + # ) github_app_instance = github.Github(token) github_repo = github_app_instance.get_repo( self.ref_pull_request.code_pull_request.repository.repo_id @@ -388,20 +308,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -class GithubLoginState(models.Model): - """ - This model is used to store the state of the login process. Along with it, - it also stores the redirect URL for the state. - """ - - state = models.UUIDField(default=uuid.uuid4) - redirect_url = models.URLField() - created_at = models.DateTimeField(auto_now_add=True) - - def __str__(self) -> str: - return self.state.__str__() - - class GithubMarketplaceEvent(models.Model): payload = models.JSONField(default=dict) created_at = models.DateTimeField(auto_now_add=True) diff --git a/app/signals.py b/app/signals.py index 5206147..911a08c 100644 --- a/app/signals.py +++ b/app/signals.py @@ -1,165 +1,82 @@ import logging +import django_rq import github + +from app import gitlab_jobs + +# from app import github_jobs from . import lib as app_lib from django.conf import settings from django.db.models.signals import post_save from django.dispatch import receiver from . import models as app_models +from allauth.account.signals import user_signed_up +from allauth.socialaccount.signals import social_account_added +from allauth.socialaccount import models as social_models logger = logging.getLogger(__name__) -# GithubAppInstallation -# @receiver( -# post_save, -# sender=app_models.GithubAppInstallation, -# dispatch_uid="update_github_repositories_for_installation", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# logger.info(f"GithubAppInstallation signal received for {instance}") -# github_manager = app_lib.GithubDataManager( -# instance.installation_id, instance.creator.get_active_access_token() -# ) -# github_manager.sync_repositories() - - -# The job is for syncing PRs, invoke separately for both code and documentation repo -# @receiver( -# post_save, -# sender=app_models.GithubRepoMap, -# dispatch_uid="invoke_actions_on_repo_map_save", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# logger.info(f"GithubRepoMap signal received for {instance}") -# installation_instance = instance.integration -# github_data_manager = app_lib.GithubDataManager( -# installation_id=installation_instance.installation_id, -# user_token=installation_instance.creator.get_active_access_token(), -# ) -# github_data_manager.sync_open_prs(instance.code_repo) -# github_data_manager.sync_open_prs(instance.documentation_repo) - -# @receiver( -# post_save, -# sender=app_models.GithubPullRequest, -# dispatch_uid="invoke_actions_on_pr_update", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# for documentation_pr in app_models.MonitoredPullRequest.objects.filter( -# documentation_pull_request=instance -# ).iterator(): -# if documentation_pr.is_approved: -# documentation_pr.pull_request_status = ( -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_CODE -# ) -# documentation_pr.save() -# for code_pr in app_models.MonitoredPullRequest.objects.filter( -# code_pull_request=instance -# ).iterator(): -# if code_pr.is_approved: -# code_pr.pull_request_status = ( -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_APPROVAL -# ) -# code_pr.save() +def new_installation(social_account): + extra_data = social_account.extra_data + if social_account.provider == "gitlab": + (app_installation, _) = app_models.AppInstallation.objects.update_or_create( + social_app=social_models.SocialApp.objects.get( + provider=social_account.provider + ), + installation_id=extra_data["id"], + creator=social_account.user, + state=app_models.AppInstallation.InstallationState.INSTALLED, + account_name=extra_data["username"], + account_id=extra_data["id"], + account_type="User", + avatar_url=social_account.get_avatar_url(), + ) + app_models.AppUser.objects.update_or_create( + installation=app_installation, user=social_account.user + ) + django_rq.enqueue( + gitlab_jobs.sync_repositories_for_installation, social_account + ) + elif social_account.provider == "github": + github_instance = github.Github(app_lib.get_active_token(social_account).token) + for installation in github_instance.get_user().get_installations(): + if installation.app_id == settings.GITHUB_CREDS["app_id"]: + raw_data = installation.raw_data + try: + app_installation = app_models.AppInstallation.objects.get( + social_app=social_models.SocialApp.objects.get( + provider=social_account.provider + ), + installation_id=installation.id, + ) + app_models.AppUser.objects.update_or_create( + installation=app_installation, user=social_account.user + ) + except app_models.AppInstallation.DoesNotExist: + # App installation not done. Skip mapping + # Error: The App installation not found. Configuration error + pass + # TODO: Schedule github repo sync + # django_rq.enqueue(github_jobs., app_installation) -# if instance.repository.code_repos.exists(): -# installation_instance = instance.repository.code_repos.last().integration -# ( -# monitored_pr_instance, -# is_new, -# ) = app_models.MonitoredPullRequest.objects.get_or_create( -# code_pull_request=instance, integration=installation_instance -# ) -# if is_new is False: -# monitored_pr_instance.save() +# def sync_repos(social_account): +# # assert False, social_account +# if social_account.provider == "gitlab": +# django_rq.enqueue(gitlab_jobs.sync_repositories_for_installation) -# @receiver( -# post_save, -# sender=app_models.MonitoredPullRequest, -# dispatch_uid="invoke_github_check_for_pr", -# ) -# def update_check_run(sender, instance, **kwargs): -# (instance, is_new) = app_models.GithubCheckRun.objects.get_or_create( -# ref_pull_request=instance, run_sha=instance.code_pull_request.pr_head_commit_sha -# ) -# if is_new: -# # Here, we can close the previous checks if any -# # app_models.GithubCheckRun.objects.filter(ref_pull_request=instance).exclude( -# # run_sha=instance.code_pull_request.pr_head_commit_sha, -# # ) -# pass -# else: -# instance.save() +@receiver(user_signed_up) +def _user_signed_up(request, user, *args, **kwargs): + # User signed up. Create + social_accounts = user.socialaccount_set.all() + for social_account in social_accounts: + new_installation(social_account) + # sync_repos(social_account) -# @receiver( -# post_save, sender=app_models.GithubCheckRun, dispatch_uid="update_github_check" -# ) -# def synchronize_github_check(sender, instance, **kwargs): -# token = ( -# instance.ref_pull_request.code_pull_request.repository.owner.creator.access_token -# ) -# github_app_instance = github.Github(token) -# github_repo = github_app_instance.get_repo( -# instance.ref_pull_request.code_pull_request.repository.repo_id -# ) -# check_run = github_repo.get_check_run(instance.run_id) -# data = { -# "name": settings.GITHUB_CREDS["app_name"], -# "head_sha": instance.run_sha, -# "external_id": instance.unique_id.__str__(), -# "status": "completed", -# "conclusion": "action_required", -# "details_url": f"{settings.WEBSITE_HOST}/{github_repo.full_name}/pulls/", -# } -# if ( -# instance.ref_pull_request.pull_request_status -# == app_models.MonitoredPullRequest.PullRequestStatus.NOT_CONNECTED -# ): -# # Update the action with the PR connection action -# data.update( -# { -# "conclusion": "action_required", -# "output": { -# "title": "Documentation PR is not connected", -# "summary": "Please connect the documentation PR", -# "text": "You can connect the documentation PR by clicking on the button below.", -# }, -# } -# ) -# elif instance.ref_pull_request.pull_request_status in [ -# app_models.MonitoredPullRequest.PullRequestStatus.APPROVED, -# app_models.MonitoredPullRequest.PullRequestStatus.MANUAL_APPROVAL, -# ]: -# data.update( -# { -# "conclusion": "success", -# "output": { -# "title": "", -# "summary": "", -# "text": "", -# }, -# } -# ) -# elif instance.ref_pull_request.pull_request_status in [ -# app_models.MonitoredPullRequest.PullRequestStatus.APPROVAL_PENDING, -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_APPROVAL, -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_CODE, -# ]: -# # Update the action with the PR approval action -# data.update( -# { -# "conclusion": "action_required", -# "output": { -# "title": "Approve the PR on CDOC", -# "summary": "Please approve the PR on CDOC", -# "text": "You can connect the documentation PR by clicking on the button below.", -# }, -# } -# ) -# logger.info("Updating github check with data", extra={"payload": data}) -# check_run_instance = check_run.edit(**data) -# logger.info("Updated the check run", extra={"response": check_run_instance}) +@receiver(social_account_added) +def _social_account_added(request, sociallogin, *args, **kwargs): + new_installation(sociallogin.account) diff --git a/app/templates/all_pr_for_org.html b/app/templates/all_pr_for_org.html index 45bf136..362cba9 100644 --- a/app/templates/all_pr_for_org.html +++ b/app/templates/all_pr_for_org.html @@ -10,7 +10,8 @@ data-icon="octicon:repo-16">{{repo_mapping.code_repo.owner.account_name}}{% endcomment %}
- {{repo_mapping.code_repo.repo_full_name}} + {{repo_mapping.code_repo.repo_full_name}} {{repo_mapping.documentation_repo.repo_full_name}}
@@ -20,548 +21,591 @@ {% if open_prs %}
- +
{% endif %} {% if open_prs %}
- {% else %} -
- {% endif %} - - {% for monitored_pr in open_prs %} - - {% with code_pr_number=monitored_pr.code_pull_request.pr_number|stringformat:"s" documentation_pr_number=monitored_pr.documentation_pull_request.pr_number|stringformat:"s" %} - {% with code_pr_name="#"|add:code_pr_number|add:" "|add:monitored_pr.code_pull_request.pr_title %} -
- {% if monitored_pr.pull_request_status == "NOT_CONNECTED" %} -
-
- {{ code_pr_name }} - {{monitored_pr.get_pull_request_status_display}} + {% else %} +
+ {% endif %} + + {% for monitored_pr in open_prs %} + + {% with code_pr_number=monitored_pr.code_pull_request.pr_number|stringformat:"s" documentation_pr_number=monitored_pr.documentation_pull_request.pr_number|stringformat:"s" %} + {% with code_pr_name="#"|add:code_pr_number|add:" "|add:monitored_pr.code_pull_request.pr_title %} + {{code_pr_number}} +
+ {% if monitored_pr.pull_request_status == "NOT_CONNECTED" %} +
+
+ {{ code_pr_name }} + {{monitored_pr.get_pull_request_status_display}} +
-
-
- Connect -
- {% elif monitored_pr.pull_request_status == "MANUALLY_APPROVED" %} -
-
- {{ code_pr_name }} - {{monitored_pr.get_pull_request_status_display}} +
+ Connect
-
-
- Documentation Not Required -
- by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} {{monitored_pr.get_latest_approval.created_on|timesince}} ago + {% elif monitored_pr.pull_request_status == "MANUALLY_APPROVED" %} +
+
+ {{ code_pr_name }} + {{monitored_pr.get_pull_request_status_display}} +
-
- {% else %} - - {% with documentation_pr_name="#"|add:documentation_pr_number|add:" "|add:monitored_pr.documentation_pull_request.pr_title %} - {% if monitored_pr.pull_request_status == "APPROVED" %} -
-
- - {{code_pr_name}} - {{documentation_pr_name}} +
+ Documentation Not Required +
+ by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} + {{monitored_pr.get_latest_approval.created_on|timesince}} ago +
-
-
- Approved -
- by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} {{monitored_pr.get_latest_approval.created_on|timesince}} ago + {% else %} + + {% with documentation_pr_name="#"|add:documentation_pr_number|add:" "|add:monitored_pr.documentation_pull_request.pr_title %} + {% if monitored_pr.pull_request_status == "APPROVED" %} +
+
+ + {{code_pr_name}} + {{documentation_pr_name}} +
-
- {% else %} - {% comment %}Expected states `APPROVAL_PENDING`, `STALE_CODE` or `STALE_APPROVAL`{% endcomment %} -
-
- - {{code_pr_name}} - {{documentation_pr_name}} +
+ Approved +
+ by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} + {{monitored_pr.get_latest_approval.created_on|timesince}} ago +
+ {% else %} + {% comment %}Expected states `APPROVAL_PENDING`, `STALE_CODE` or `STALE_APPROVAL`{% endcomment %} +
+
+ + {{code_pr_name}} + {{documentation_pr_name}} +
+
+
+ Approve +
+ {% endif %} + {% endwith %} + {% endif %}
-
- Approve -
- {% endif %} {% endwith %} - {% endif %} -
- {% endwith %} - {% endwith %} - {% empty %} + {% endwith %} + {% empty %}

No recent entries found

- {% endfor %} - -
- - -