From d370281e92ed3537981721434347307927a641bd Mon Sep 17 00:00:00 2001 From: Shobhit Date: Thu, 19 May 2022 15:09:03 +0530 Subject: [PATCH 01/15] foundation --- app/gitlab_views.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ app/models.py | 1 - app/urls.py | 12 ++++++++- cdoc/settings.py | 23 ++++++++++++++++ cdoc/urls.py | 1 + requirements.in | 2 ++ 6 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 app/gitlab_views.py diff --git a/app/gitlab_views.py b/app/gitlab_views.py new file mode 100644 index 0000000..1de6c9d --- /dev/null +++ b/app/gitlab_views.py @@ -0,0 +1,66 @@ +from django.http import HttpResponseRedirect +from django.urls import reverse +from django.views import View +from app import models as app_models +from django.conf import settings +from requests.models import PreparedRequest +import base64 +import hashlib +import requests + + +class InitiateLoginView(View): + def get(self, request, *args, **kwargs): + if self.request.user.is_authenticated: + if request.GET.get("next"): + return HttpResponseRedirect(request.GET.get("next")) + return HttpResponseRedirect("/") + else: + login_state_instance = app_models.GithubLoginState() + if self.request.GET.get("next"): + login_state_instance.redirect_url = self.request.GET.get("next") + login_state_instance.save() + url = "https://gitlab.com/oauth/authorize" + # https://gitlab.example.com/oauth/authorize?redirect_uri=REDIRECT_URI&scope=REQUESTED_SCOPES&code_challenge=CODE_CHALLENGE&code_challenge_method=S256 + params = { + "client_id": settings.GITLAB_CREDS["application_id"], + "response_type": "code", + "allow_signup": False, + "state": login_state_instance.state.__str__(), + "code_challenge": ( + base64.urlsafe_b64encode( + hashlib.sha256( + (login_state_instance.state.__str__() * 2).encode("ascii") + ).digest() + ) + .decode("ascii") + .rstrip("=") + ), + "code_challenge_method": "S256", + "scope": settings.GITLAB_CREDS["scope"], + "redirect_uri": request.build_absolute_uri( + reverse("gitlab_auth_callback") + ), + } + req = PreparedRequest() + req.prepare_url(url, params) + return HttpResponseRedirect(req.url) + + +class AuthCallback(View): + def get(self, request, *args, **kwargs): + code = request.GET["code"] + state = request.GET["state"] + # assert False, (code, state) + # parameters = 'client_id=APP_ID&code=RETURNED_CODE&grant_type=authorization_code&redirect_uri=REDIRECT_URI&code_verifier=CODE_VERIFIER' + payload = { + "client_id": settings.GITLAB_CREDS["application_id"], + "code": code, + "grant_type": "authorization_code", + "redirect_uri": request.build_absolute_uri(reverse("gitlab_auth_callback")), + "code_verifier": state * 2, + } + response = requests.post("https://gitlab.com/oauth/token", json=payload) + assert False, response.text + + pass diff --git a/app/models.py b/app/models.py index 3def479..6278bac 100644 --- a/app/models.py +++ b/app/models.py @@ -173,7 +173,6 @@ class Meta: def __str__(self) -> str: return f"{self.github_user.account_name}[{self.installation.installation_id}]" - return super().__str__() class GithubInstallationToken(Token): diff --git a/app/urls.py b/app/urls.py index b3970ca..1a01b00 100644 --- a/app/urls.py +++ b/app/urls.py @@ -1,10 +1,20 @@ -from django.urls import path +from django.urls import path, include from . import views +from . import gitlab_views +gitlab_urlpatterns = [ + path( + "auth-callback/", + gitlab_views.AuthCallback.as_view(), + name="gitlab_auth_callback", + ), + path("initiate-login/", gitlab_views.InitiateLoginView.as_view()), +] urlpatterns = [ path("callback/", views.AuthCallback.as_view()), path("webhook-callback/", views.WebhookCallback.as_view()), path("oauth-callback/", views.OauthCallback.as_view()), path("marketplace-callback/", views.MarketplaceCallbackView.as_view()), + path("gitlab/", include(gitlab_urlpatterns)), ] diff --git a/cdoc/settings.py b/cdoc/settings.py index f42ebdc..05b6023 100644 --- a/cdoc/settings.py +++ b/cdoc/settings.py @@ -53,6 +53,11 @@ "whitenoise.runserver_nostatic", "django.contrib.staticfiles", "django_extensions", + "allauth", + "allauth.account", + "allauth.socialaccount", + "allauth.socialaccount.providers.github", + "allauth.socialaccount.providers.gitlab", "app.apps.AppConfig", "analytics", "django_rq", @@ -71,6 +76,12 @@ ROOT_URLCONF = "cdoc.urls" + +AUTHENTICATION_BACKENDS = [ + "django.contrib.auth.backends.ModelBackend", + "allauth.account.auth_backends.AuthenticationBackend", +] + TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", @@ -247,6 +258,12 @@ "NAME": BASE_DIR / "db.sqlite3", } +USE_X_FORWARDED_HOST = True +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +SITE_ID = 1 + + try: from .local_settings import * # noqa except ModuleNotFoundError: @@ -264,6 +281,12 @@ "app_signature_secret": os.environ["CDOC_GITHUB_APP_SIGNATURE_SECRET"], } + GITLAB_CREDS = { + "application_id": os.environ["CDOC_GITLAB_APPLICATION_ID"], + "secret": os.environ["CDOC_GITLAB_SECRET"], + "SCOPE": ["api"], + } + if "CDOC_SENTRY_DSN" in os.environ: import sentry_sdk # noqa from sentry_sdk.integrations.django import DjangoIntegration # noqa diff --git a/cdoc/urls.py b/cdoc/urls.py index 14d211d..5b3f9b8 100644 --- a/cdoc/urls.py +++ b/cdoc/urls.py @@ -31,6 +31,7 @@ "accounts/logout/", auth_views.LogoutView.as_view(next_page="/"), ), + path("accounts/", include("allauth.urls")), # path( # "/repos/", # app_views.ListInstallationRepos.as_view(), diff --git a/requirements.in b/requirements.in index 58d386a..5225f91 100644 --- a/requirements.in +++ b/requirements.in @@ -14,3 +14,5 @@ sentry-sdk rq django-rq ftd-django==0.1.1 +python-gitlab +django-allauth \ No newline at end of file From 9039544c6b7572dfa96d29ea0233b2bf58dc17b7 Mon Sep 17 00:00:00 2001 From: Shobhit Date: Sat, 28 May 2022 15:17:17 +0530 Subject: [PATCH 02/15] Major overhaul. --- app/admin.py | 112 ++--- app/apps.py | 3 +- app/github_views.py | 365 ++++++++++++++++ app/gitlab_jobs.py | 25 ++ app/gitlab_views.py | 55 --- app/jobs.py | 7 +- app/lib.py | 45 +- .../0020_appinstallation_appuser_and_more.py | 127 ++++++ app/migrations/0021_userrepoaccess.py | 28 ++ ...unique_together_repository_app_and_more.py | 33 ++ app/models.py | 201 ++------- app/signals.py | 207 +++------ app/templates/index.html | 127 +++--- app/urls.py | 20 +- app/views.py | 404 ++---------------- cdoc/settings.py | 12 + cdoc/urls.py | 9 +- 17 files changed, 912 insertions(+), 868 deletions(-) create mode 100644 app/github_views.py create mode 100644 app/gitlab_jobs.py create mode 100644 app/migrations/0020_appinstallation_appuser_and_more.py create mode 100644 app/migrations/0021_userrepoaccess.py create mode 100644 app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py diff --git a/app/admin.py b/app/admin.py index c1bb3d6..8f7f364 100644 --- a/app/admin.py +++ b/app/admin.py @@ -4,72 +4,72 @@ # admin.site.register(models.GithubAppAuth) -admin.site.register(models.GithubUser) +# admin.site.register(models.GithubUser) -class GithubRepoMapAdmin(admin.TabularInline): - model = models.GithubRepoMap +# class GithubRepoMapAdmin(admin.TabularInline): +# model = models.GithubRepoMap - def formfield_for_foreignkey(self, db_field, request, **kwargs): - print(db_field.name) - # print(request) - # print(kwargs) - if db_field.name in ["code_repo", "documentation_repo"]: - github_app_installation_id = request.resolver_match.kwargs.get("object_id") - kwargs["queryset"] = models.GithubRepository.objects.filter( - owner_id=github_app_installation_id - ) - return super(GithubRepoMapAdmin, self).formfield_for_foreignkey( - db_field, request, **kwargs - ) +# def formfield_for_foreignkey(self, db_field, request, **kwargs): +# print(db_field.name) +# # print(request) +# # print(kwargs) +# if db_field.name in ["code_repo", "documentation_repo"]: +# github_app_installation_id = request.resolver_match.kwargs.get("object_id") +# kwargs["queryset"] = models.GithubRepository.objects.filter( +# owner_id=github_app_installation_id +# ) +# return super(GithubRepoMapAdmin, self).formfield_for_foreignkey( +# db_field, request, **kwargs +# ) - # def get_queryset(self, request): - # qs = super(GithubRepoMapAdmin, self).get_queryset(request) - # return qs.filter(=request.user) +# # def get_queryset(self, request): +# # qs = super(GithubRepoMapAdmin, self).get_queryset(request) +# # return qs.filter(=request.user) -class GithubAppInstallationForm(forms.ModelForm): - class Meta: - model = models.GithubAppInstallation - exclude = ["account_id"] +# class GithubAppInstallationForm(forms.ModelForm): +# class Meta: +# model = models.GithubAppInstallation +# exclude = ["account_id"] -class GithubAppInstallationAdmin(admin.ModelAdmin): - list_display = ( - "account_name", - "created_at", - ) - readonly_fields = ( - "account_name", - "installation_id", - "account_type", - "state", - "creator", - ) +# class GithubAppInstallationAdmin(admin.ModelAdmin): +# list_display = ( +# "account_name", +# "created_at", +# ) +# readonly_fields = ( +# "account_name", +# "installation_id", +# "account_type", +# "state", +# "creator", +# ) - form = GithubAppInstallationForm - inlines = [ - GithubRepoMapAdmin, - ] +# form = GithubAppInstallationForm +# inlines = [ +# GithubRepoMapAdmin, +# ] - def get_queryset(self, request): - qs = super(GithubAppInstallationAdmin, self).get_queryset(request) - if request.user.is_superuser: - return qs - else: - return qs.filter(creator=request.user.github_user) +# def get_queryset(self, request): +# qs = super(GithubAppInstallationAdmin, self).get_queryset(request) +# if request.user.is_superuser: +# return qs +# else: +# return qs.filter(creator=request.user.github_user) - def get_form(self, request, obj=None, **kwargs): - form = super(GithubAppInstallationAdmin, self).get_form(request, obj, **kwargs) - # form.base_fields['theme'].queryset = Theme.objects.filter(name__iexact='company') - return form +# def get_form(self, request, obj=None, **kwargs): +# form = super(GithubAppInstallationAdmin, self).get_form(request, obj, **kwargs) +# # form.base_fields['theme'].queryset = Theme.objects.filter(name__iexact='company') +# return form -admin.site.register(models.GithubAppInstallation, GithubAppInstallationAdmin) -admin.site.register(models.GithubUserAccessToken) -admin.site.register(models.GithubUserRefreshToken) -admin.site.register(models.GithubInstallationToken) -admin.site.register(models.GithubPullRequest) -admin.site.register(models.MonitoredPullRequest) -admin.site.register(models.GithubCheckRun) -# admin.site.register(models.GithubRepoMap) +# admin.site.register(models.GithubAppInstallation, GithubAppInstallationAdmin) +# admin.site.register(models.GithubUserAccessToken) +# admin.site.register(models.GithubUserRefreshToken) +# admin.site.register(models.GithubInstallationToken) +# admin.site.register(models.GithubPullRequest) +# admin.site.register(models.MonitoredPullRequest) +# admin.site.register(models.GithubCheckRun) +# # admin.site.register(models.GithubRepoMap) diff --git a/app/apps.py b/app/apps.py index d118638..a40177c 100644 --- a/app/apps.py +++ b/app/apps.py @@ -5,5 +5,4 @@ class AppConfig(AppConfig): name = "app" def ready(self): - # import app.jobs # noqa - pass + import app.signals # noqa diff --git a/app/github_views.py b/app/github_views.py new file mode 100644 index 0000000..cec994a --- /dev/null +++ b/app/github_views.py @@ -0,0 +1,365 @@ +import datetime +import json +import logging +import uuid +from distutils.command.clean import clean +from typing import Any, Dict +from urllib.parse import parse_qs + +import django_rq +import github +import requests +from analytics import events as analytics_events +from analytics.decorators import log_http_event +from django.conf import settings +from django.contrib.auth import get_user_model, login +from django.contrib.auth import models as auth_models +from django.contrib.auth import views as auth_views +from django.contrib.auth.decorators import login_required +from django.db import transaction +from django.db.models import Q +from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse +from django.shortcuts import get_object_or_404, render +from django.urls import reverse +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views import View +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import FormView, TemplateView +from requests.models import PreparedRequest +from allauth.socialaccount import models as allauth_social_models +import lib + +from . import jobs as app_jobs +from . import models as app_models + +logger = logging.getLogger(__name__) + + +@method_decorator(csrf_exempt, name="dispatch") +class MarketplaceCallbackView(View): + def post(self, request, *args, **kwargs): + body = request.body + is_verified = lib.verify_signature( + request.headers["X-Hub-Signature-256"], + body, + settings.GITHUB_CREDS["marketplace_signature_secret"], + ) + if not is_verified: + return HttpResponse("Invalid signature", status=403) + payload = json.loads(body) + app_models.GithubMarketplaceEvent.objects.create(payload=payload) + return JsonResponse({"status": True}) + + +@method_decorator(csrf_exempt, name="dispatch") +class AuthCallback(View): + def get_okind_ekind(view, request, *args, **kwargs): + if "installation_id" in request.GET: + if request.GET.get("setup_action") == "update": + return ( + analytics_events.OKind.INSTALLATION, + analytics_events.EKind.UPDATE, + ) + else: + return ( + analytics_events.OKind.INSTALLATION, + analytics_events.EKind.CREATE, + ) + return (analytics_events.OKind.USER, analytics_events.EKind.LOGIN) + + @log_http_event(oid=1, okind_ekind=get_okind_ekind) + def get(self, request, *args, **kwargs): + """ + This method is invoked when the user installs the application and + generates the auth token for the user to be used in the future. + Expected inputs as urlparams: + - code: str + - installation_id: int + - setup_action: str + """ + code: str = request.GET["code"] + + # logger.info(request.GET) + # github_app_auth_user = app_models.GithubAppAuth.objects.create( + # code=code, + # installation_id=installation_id, + # setup_action=setup_action, + # ) + # Get the App installation account for managing the various scopes for a user + # Example, User A can be a part of 2 installations. So, the user will see them as two different projects + # in the UI. + # The user's account using the `code` recieved in the step above + # logger.info(request.get_host()) + payload = { + "client_id": settings.GITHUB_CREDS["client_id"], + "client_secret": settings.GITHUB_CREDS["client_secret"], + "code": code, + "redirect_uri": f"https://{request.get_host()}/app/callback/", + "state": uuid.uuid4().__str__(), + } + resp = requests.post( + "https://github.com/login/oauth/access_token", json=payload + ) + logger.info(resp.text) + if resp.ok: + response = parse_qs(resp.text) + if "error" in response: + logger.error(response) + else: + now = timezone.now() + access_token = response["access_token"][0] + logger.info(response) + access_token_expires_at = now + datetime.timedelta( + seconds=int(response["expires_in"][0]) + ) + refresh_token = response["refresh_token"][0] + refresh_token_expires_at = now + datetime.timedelta( + seconds=int(response["refresh_token_expires_in"][0]) + ) + logger.info(access_token) + github_instance = github.Github(access_token) + user_instance = github_instance.get_user() + with transaction.atomic(): + # access_group = auth_models.Group.objects.get(name="github_user") + (auth_user_instance, _) = get_user_model().objects.get_or_create( + username=user_instance.login, + defaults={ + "is_active": True, + "is_staff": False, + }, + ) + ( + social_account, + _, + ) = allauth_social_models.SocialAccount.objects.update_or_create( + user=auth_user_instance, + provider="github", + uid=user_instance.id, + extra_data={}, + ) + allauth_social_models.SocialToken.objects.create( + token=access_token, + token_secret=refresh_token, + account=social_account, + expires_at=access_token_expires_at, + app=allauth_social_models.SocialApp.objects.get( + provider="github" + ), + ) + login(request, auth_user_instance) + if "installation_id" in request.GET: + installation_id: int = request.GET["installation_id"] + setup_action: str = request.GET["setup_action"] + + github_installation_manager = lib.GithubInstallationManager( + installation_id=installation_id, user_token=access_token + ) + installation_details = ( + github_installation_manager.get_installation_details() + ) + account_details = installation_details["account"] + account_name = account_details["login"] + account_id = account_details["id"] + account_type = account_details["type"] + avatar_url = account_details["avatar_url"] + + with transaction.atomic(): + + ( + installation_instance, + is_new, + ) = app_models.GithubAppInstallation.objects.update_or_create( + installation_id=installation_id, + account_id=account_id, + defaults={ + "account_name": account_name, + "state": app_models.GithubAppInstallation.InstallationState.INSTALLED, + "account_type": account_type, + "avatar_url": avatar_url, + "creator": github_user, + }, + ) + # installation_instance.save() + installation_instance.update_token() + # if is_new: + django_rq.enqueue( + app_jobs.sync_repositories_for_installation, + installation_instance, + ) + app_models.GithubAppUser.objects.update_or_create( + github_user=github_user, + installation=installation_instance, + ) + # logger.info([x for x in user_instance.get_installations()]) + for installation in user_instance.get_installations(): + if installation.app_id == settings.GITHUB_CREDS["app_id"]: + installation_instance = ( + app_models.GithubAppInstallation.objects.get( + account_id=installation.target_id, + installation_id=installation.id, + ) + ) + app_models.GithubAppUser.objects.update_or_create( + github_user=github_user, + installation=installation_instance, + ) + else: + logger.error(f"Unable to get token from the code: {resp.text}") + return HttpResponseRedirect(reverse("initiate_github_login")) + redirect_url = None + + if "state" in request.GET: + next_url = app_models.GithubLoginState.objects.get( + state=request.GET["state"] + ).redirect_url + if next_url is not None and next_url != "": + redirect_url = next_url + return HttpResponseRedirect(redirect_url or "/") + + +@method_decorator(csrf_exempt, name="dispatch") +class WebhookCallback(View): + def post(self, request, *args, **kwargs): + headers = request.headers + body = request.body + is_verified = lib.verify_signature( + headers["X-Hub-Signature-256"], + body, + settings.GITHUB_CREDS["app_signature_secret"], + ) + payload = json.loads(body) + if not is_verified: + return HttpResponse("Invalid signature", status=403) + logger.info( + "Recieved Github webhook", + extra={"data": {"headers": headers, "payload": payload}}, + ) + EVENT_TYPE = headers.get("X-Github-Event", None) + # header is "installation_repositories" -> Updated the repositories installed for the installation + interesting_events = [ + "pull_request", + "installation_repositories", + "check_suite", + "installation", + ] + if EVENT_TYPE in interesting_events: + get_installation_instance = ( + lambda data: app_models.GithubAppInstallation.objects.get( + installation_id=data["installation"]["id"] + ) + ) + installation_instance = get_installation_instance(payload) + if EVENT_TYPE == "installation": + should_save = False + if payload["action"] == "deleted": + installation_instance.state = ( + app_models.GithubAppInstallation.InstallationState.UNINSTALLED + ) + should_save = True + elif payload["action"] == "suspend": + installation_instance.state = ( + app_models.GithubAppInstallation.InstallationState.SUSPENDED + ) + should_save = True + elif payload["action"] == "unsuspend": + installation_instance.state = ( + app_models.GithubAppInstallation.InstallationState.INSTALLED + ) + should_save = True + if should_save: + installation_instance.save() + elif EVENT_TYPE == "pull_request": + pull_request_data = payload["pull_request"] + (github_repo, _) = app_models.GithubRepository.objects.update_or_create( + repo_id=payload["repository"]["id"], + owner=installation_instance, + defaults={ + "repo_full_name": payload["repository"]["full_name"], + "repo_name": payload["repository"]["name"], + }, + ) + ( + pr_instance, + _, + ) = app_models.GithubPullRequest.objects.update_or_create( + pr_id=pull_request_data["id"], + pr_number=pull_request_data["number"], + repository=github_repo, + defaults={ + "pr_head_commit_sha": pull_request_data["head"]["sha"], + "pr_title": pull_request_data["title"], + "pr_body": pull_request_data["body"], + "pr_state": pull_request_data["state"], + "pr_created_at": pull_request_data["created_at"], + "pr_updated_at": pull_request_data["updated_at"], + "pr_merged_at": pull_request_data["merged_at"], + "pr_closed_at": pull_request_data["closed_at"], + "pr_merged": pull_request_data["merged"], + "pr_owner_username": pull_request_data["user"]["login"], + }, + ) + django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) + elif EVENT_TYPE == "installation_repositories": + # Repositories changed. Sync again. + django_rq.enqueue( + app_jobs.sync_repositories_for_installation, + installation_instance, + ) + elif EVENT_TYPE == "check_suite": + if payload.get("action") == "requested": + + repository_data = payload.get("repository", {}) + ( + github_repo, + _, + ) = app_models.GithubRepository.objects.update_or_create( + repo_id=repository_data["id"], + owner=installation_instance, + defaults={ + "repo_name": repository_data["name"], + "repo_full_name": repository_data["full_name"], + }, + ) + prs = payload.get("check_suite", {}).get("pull_requests", []) + + head_commit_details = payload["check_suite"]["head_commit"] + head_commit_data = { + "pr_head_commit_sha": head_commit_details["id"], + # "pr_head_tree_sha": head_commit_details["tree_id"], + "pr_head_commit_message": head_commit_details["message"], + "pr_head_modified_on": datetime.datetime.strptime( + head_commit_details["timestamp"], "%Y-%m-%dT%H:%M:%S%z" + ), + } + + logger.info( + f"Found PRs for commit ID: {head_commit_details['id']}", + extra={"data": {"pull_requests": prs}}, + ) + # is_documentation_repo = + if ( + github_repo.code_repos.exists() + or github_repo.documentation_repos.exists() + ): + for pr in prs: + pr_id = pr.get("id") + pr_number = pr.get("number") + with transaction.atomic(): + ( + pr_instance, + is_new, + ) = app_models.GithubPullRequest.objects.update_or_create( + pr_id=pr_id, + pr_number=pr_number, + repository=github_repo, + defaults={**head_commit_data}, + ) + django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) + return JsonResponse({"status": True}) + + +@method_decorator(csrf_exempt, name="dispatch") +class OauthCallback(View): + def get(self, request): + assert False, request diff --git a/app/gitlab_jobs.py b/app/gitlab_jobs.py new file mode 100644 index 0000000..f12521a --- /dev/null +++ b/app/gitlab_jobs.py @@ -0,0 +1,25 @@ +from django_rq import job +import gitlab +from app import models as app_models +from allauth.socialaccount import models as social_models +from app import lib as app_lib + + +@job +def sync_repositories_for_installation(social_account: social_models.SocialAccount): + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(social_account).token + ) + for project in gitlab_instance.projects.list(all=True, membership=True): + (repo_instance, _) = app_models.Repository.objects.update_or_create( + repo_id=project.id, + app=social_models.SocialApp.objects.get(provider="gitlab"), + defaults={ + "repo_name": project.name, + "repo_full_name": project.name_with_namespace, + }, + ) + app_models.UserRepoAccess.objects.update_or_create( + repository=repo_instance, + user=social_account.user, + ) diff --git a/app/gitlab_views.py b/app/gitlab_views.py index 1de6c9d..e340e10 100644 --- a/app/gitlab_views.py +++ b/app/gitlab_views.py @@ -9,58 +9,3 @@ import requests -class InitiateLoginView(View): - def get(self, request, *args, **kwargs): - if self.request.user.is_authenticated: - if request.GET.get("next"): - return HttpResponseRedirect(request.GET.get("next")) - return HttpResponseRedirect("/") - else: - login_state_instance = app_models.GithubLoginState() - if self.request.GET.get("next"): - login_state_instance.redirect_url = self.request.GET.get("next") - login_state_instance.save() - url = "https://gitlab.com/oauth/authorize" - # https://gitlab.example.com/oauth/authorize?redirect_uri=REDIRECT_URI&scope=REQUESTED_SCOPES&code_challenge=CODE_CHALLENGE&code_challenge_method=S256 - params = { - "client_id": settings.GITLAB_CREDS["application_id"], - "response_type": "code", - "allow_signup": False, - "state": login_state_instance.state.__str__(), - "code_challenge": ( - base64.urlsafe_b64encode( - hashlib.sha256( - (login_state_instance.state.__str__() * 2).encode("ascii") - ).digest() - ) - .decode("ascii") - .rstrip("=") - ), - "code_challenge_method": "S256", - "scope": settings.GITLAB_CREDS["scope"], - "redirect_uri": request.build_absolute_uri( - reverse("gitlab_auth_callback") - ), - } - req = PreparedRequest() - req.prepare_url(url, params) - return HttpResponseRedirect(req.url) - - -class AuthCallback(View): - def get(self, request, *args, **kwargs): - code = request.GET["code"] - state = request.GET["state"] - # assert False, (code, state) - # parameters = 'client_id=APP_ID&code=RETURNED_CODE&grant_type=authorization_code&redirect_uri=REDIRECT_URI&code_verifier=CODE_VERIFIER' - payload = { - "client_id": settings.GITLAB_CREDS["application_id"], - "code": code, - "grant_type": "authorization_code", - "redirect_uri": request.build_absolute_uri(reverse("gitlab_auth_callback")), - "code_verifier": state * 2, - } - response = requests.post("https://gitlab.com/oauth/token", json=payload) - assert False, response.text - - pass diff --git a/app/jobs.py b/app/jobs.py index bb447bf..4aa2cf3 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -15,7 +15,7 @@ @job def sync_repositories_for_installation( - installation_instance: app_models.GithubAppInstallation, + installation_instance: app_models.AppInstallation, ): logger.info(f"GithubAppInstallation signal received for {installation_instance}") github_manager = app_lib.GithubDataManager( @@ -159,8 +159,3 @@ def monitored_pr_post_save(instance_id: int): logger.info("Updating github check with data", extra={"payload": data}) check_run_instance = check_run.edit(**data) logger.info("Updated the check run", extra={"response": check_run_instance}) - - -@job -def test_job(a, b): - assert False, a + b diff --git a/app/lib.py b/app/lib.py index a58b4e1..3e24a29 100644 --- a/app/lib.py +++ b/app/lib.py @@ -1,6 +1,10 @@ from . import models as app_models import lib import github +from django.utils import timezone +from requests_oauthlib import OAuth2Session +from allauth.socialaccount import models as social_models +import datetime class GithubDataManager: @@ -29,7 +33,7 @@ def sync_repositories(self): ) # a function called sync_open_prs which takes input the GithubRepository and syncs its open pull requests - def sync_open_prs(self, repo: app_models.GithubRepository): + def sync_open_prs(self, repo: app_models.Repository): all_repo_ids = [] for pr in self.github_instance.get_repo(repo.repo_full_name).get_pulls("open"): extra_data = { @@ -54,3 +58,42 @@ def sync_open_prs(self, repo: app_models.GithubRepository): ) all_repo_ids.append(instance.id) return all_repo_ids + + +def get_active_token(social_account): + last_token = social_account.socialtoken_set.last() + if last_token.expires_at > timezone.now(): + return last_token + social_app_instance = social_models.SocialApp.objects.get( + provider=social_account.provider + ) + extra = { + "client_id": social_app_instance.client_id, + "client_secret": social_app_instance.secret, + } + client = OAuth2Session( + social_app_instance.client_id, + token={ + "access_token": last_token.token, + "refresh_token": last_token.token_secret, + "token_type": "Bearer", + "expires_in": "-30", # initially 3600, need to be updated by you + }, + ) + refresh_url = ( + "https://gitlab.com/oauth/token" + if social_account.provider == "gitlab" + else "https://github.com/login/oauth/access_token" + ) + response = client.refresh_token(refresh_url, **extra) + (token, _) = social_models.SocialToken.objects.update_or_create( + app=last_token.app, + account=last_token.account, + defaults={ + "token": response["access_token"], + "token_secret": response["refresh_token"], + "expires_at": timezone.now() + + datetime.timedelta(seconds=response["expires_in"]), + }, + ) + return token diff --git a/app/migrations/0020_appinstallation_appuser_and_more.py b/app/migrations/0020_appinstallation_appuser_and_more.py new file mode 100644 index 0000000..9bc2a96 --- /dev/null +++ b/app/migrations/0020_appinstallation_appuser_and_more.py @@ -0,0 +1,127 @@ +# Generated by Django 4.0.4 on 2022-05-25 06:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0019_githubmarketplaceevent'), + ] + + operations = [ + migrations.CreateModel( + name='AppInstallation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('installation_id', models.BigIntegerField(db_index=True, unique=True)), + ('state', models.CharField(choices=[('INSTALLED', 'Installed'), ('UNINSTALLED', 'Uninstalled'), ('SUSPENDED', 'Suspended')], max_length=20)), + ('account_name', models.CharField(max_length=200)), + ('account_id', models.BigIntegerField()), + ('account_type', models.CharField(max_length=20)), + ('avatar_url', models.CharField(max_length=200)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ('social_app', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='socialaccount.socialapp')), + ], + options={ + 'unique_together': {('social_app', 'installation_id')}, + }, + ), + migrations.CreateModel( + name='AppUser', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('installation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.appinstallation')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('installation', 'user')}, + }, + ), + migrations.RenameModel( + old_name='GithubPullRequest', + new_name='PullRequest', + ), + migrations.RenameModel( + old_name='GithubCheckRun', + new_name='PullRequestCheck', + ), + migrations.RemoveField( + model_name='githubappinstallation', + name='creator', + ), + migrations.AlterUniqueTogether( + name='githubappuser', + unique_together=None, + ), + migrations.RemoveField( + model_name='githubappuser', + name='github_user', + ), + migrations.RemoveField( + model_name='githubappuser', + name='installation', + ), + migrations.RemoveField( + model_name='githubinstallationtoken', + name='github_app_installation', + ), + migrations.DeleteModel( + name='GithubLoginState', + ), + migrations.RemoveField( + model_name='githubuser', + name='user', + ), + migrations.RemoveField( + model_name='githubuseraccesstoken', + name='github_user', + ), + migrations.RemoveField( + model_name='githubuserrefreshtoken', + name='github_user', + ), + migrations.RenameModel( + old_name='GithubRepository', + new_name='Repository', + ), + migrations.DeleteModel( + name='GithubAppUser', + ), + migrations.DeleteModel( + name='GithubInstallationToken', + ), + migrations.DeleteModel( + name='GithubUser', + ), + migrations.DeleteModel( + name='GithubUserAccessToken', + ), + migrations.DeleteModel( + name='GithubUserRefreshToken', + ), + migrations.AlterField( + model_name='githubrepomap', + name='integration', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='app.appinstallation'), + ), + migrations.AlterField( + model_name='monitoredpullrequest', + name='integration', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.appinstallation'), + ), + migrations.AlterField( + model_name='repository', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='app.appinstallation'), + ), + migrations.DeleteModel( + name='GithubAppInstallation', + ), + ] diff --git a/app/migrations/0021_userrepoaccess.py b/app/migrations/0021_userrepoaccess.py new file mode 100644 index 0000000..a85ca4e --- /dev/null +++ b/app/migrations/0021_userrepoaccess.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0.4 on 2022-05-28 06:35 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0020_appinstallation_appuser_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='UserRepoAccess', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='app.repository')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('repository', 'user')}, + }, + ), + ] diff --git a/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py b/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py new file mode 100644 index 0000000..08532ae --- /dev/null +++ b/app/migrations/0022_alter_repository_unique_together_repository_app_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.0.4 on 2022-05-28 06:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('app', '0021_userrepoaccess'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='repository', + unique_together=set(), + ), + migrations.AddField( + model_name='repository', + name='app', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.PROTECT, to='socialaccount.socialapp'), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='repository', + unique_together={('repo_id', 'app')}, + ), + migrations.RemoveField( + model_name='repository', + name='owner', + ), + ] diff --git a/app/models.py b/app/models.py index 6278bac..4568114 100644 --- a/app/models.py +++ b/app/models.py @@ -4,7 +4,6 @@ import logging import uuid from urllib.parse import parse_qs - import github import lib import requests @@ -12,107 +11,12 @@ from django.contrib.auth import get_user_model from django.db import models from django.utils import timezone +from allauth.socialaccount import models as social_models logger = logging.getLogger(__name__) -class Token(models.Model): - """ - This model takes care of the tokens generated by the Github applications. - """ - - token = models.CharField(max_length=255) - created_at = models.DateTimeField(auto_now_add=True) - expires_at = models.DateTimeField() - - class Meta: - abstract = True - - -class GithubUser(models.Model): - user = models.OneToOneField( - get_user_model(), on_delete=models.CASCADE, related_name="github_user" - ) - account_name = models.CharField(max_length=200) - account_id = models.BigIntegerField() - account_type = models.CharField(max_length=20) - avatar_url = models.CharField(max_length=200) - created_at = models.DateTimeField(auto_now_add=True) - - @property - def access_token(self): - try: - return ( - self.access_tokens.exclude(expires_at__lt=timezone.now()) - .latest("expires_at") - .token - ) - except: - return None - - def get_active_access_token(self): - access_token = self.access_token - if access_token is None: - return self.get_new_tokens()[0] - return access_token - - @property - def refresh_token(self): - try: - return ( - self.refresh_tokens.exclude(expires_at__lt=timezone.now()) - .latest("expires_at") - .token - ) - except: - return None - - def process_token_response(self, token_response): - now = timezone.now() - logger.info(token_response) - data = parse_qs(token_response) - access_token = GithubUserAccessToken.objects.create( - token=data["access_token"][0], - github_user=self, - expires_at=now + datetime.timedelta(seconds=int(data["expires_in"][0])), - ) - refresh_token = GithubUserRefreshToken.objects.create( - token=data["refresh_token"][0], - github_user=self, - expires_at=now - + datetime.timedelta(seconds=int(data["refresh_token_expires_in"][0])), - ) - return (access_token.token, refresh_token.token) - - def get_new_tokens(self): - refresh_token = self.refresh_tokens.exclude( - expires_at__lt=timezone.now() - ).latest("expires_at") - response = requests.post( - "https://github.com/login/oauth/access_token", - data={ - "refresh_token": refresh_token.token, - "grant_type": "refresh_token", - "client_id": settings.GITHUB_CREDS["client_id"], - "client_secret": settings.GITHUB_CREDS["client_secret"], - }, - ) - return self.process_token_response(response.content.decode()) - - -class GithubUserAccessToken(Token): - github_user = models.ForeignKey( - GithubUser, on_delete=models.CASCADE, related_name="access_tokens" - ) - - -class GithubUserRefreshToken(Token): - github_user = models.ForeignKey( - GithubUser, on_delete=models.CASCADE, related_name="refresh_tokens" - ) - - -class GithubAppInstallation(models.Model): +class AppInstallation(models.Model): """ This model takes care of the Github installations. """ @@ -122,8 +26,9 @@ class InstallationState(models.TextChoices): UNINSTALLED = "UNINSTALLED", "Uninstalled" SUSPENDED = "SUSPENDED", "Suspended" + social_app = models.ForeignKey(social_models.SocialApp, on_delete=models.PROTECT) installation_id = models.BigIntegerField(unique=True, db_index=True) - creator = models.ForeignKey(GithubUser, on_delete=models.PROTECT) + creator = models.ForeignKey(get_user_model(), on_delete=models.PROTECT) state = models.CharField(max_length=20, choices=InstallationState.choices) account_name = models.CharField(max_length=200) account_id = models.BigIntegerField() @@ -131,71 +36,43 @@ class InstallationState(models.TextChoices): avatar_url = models.CharField(max_length=200) created_at = models.DateTimeField(auto_now_add=True) - def __str__(self) -> str: - return f"{self.account_name}[{self.installation_id}] Owner: {self.creator.account_name}" - - @property - def access_token(self): - try: - return ( - self.tokens.exclude(expires_at__lte=timezone.now()) - .latest("created_at") - .token - ) - except GithubInstallationToken.DoesNotExist: - self.update_token() - return self.get_latest_active_token() + class Meta: + unique_together = ("social_app", "installation_id") - def update_token(self): - """ - Fetches the latest access token for the installation. Returns true/false depending on whether - the requested token has been updated - """ - # User token not required for this step - github_installation_manager = lib.GithubInstallationManager( - installation_id=self.installation_id, user_token="" - ) - token, expires_at = github_installation_manager.get_installation_access_token() - return GithubInstallationToken.objects.get_or_create( - token=token, - expires_at=expires_at, - github_app_installation=self, - )[1] + def __str__(self) -> str: + return f"{self.account_name}[{self.installation_id}] Owner: {self.creator.username}" -class GithubAppUser(models.Model): - installation = models.ForeignKey(GithubAppInstallation, on_delete=models.CASCADE) - github_user = models.ForeignKey(GithubUser, on_delete=models.CASCADE) +class AppUser(models.Model): + installation = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) + user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("installation", "github_user") - - def __str__(self) -> str: - return f"{self.github_user.account_name}[{self.installation.installation_id}]" - + unique_together = ("installation", "user") -class GithubInstallationToken(Token): - github_app_installation = models.ForeignKey( - GithubAppInstallation, on_delete=models.CASCADE, related_name="tokens" - ) - -class GithubRepository(models.Model): +class Repository(models.Model): repo_id = models.BigIntegerField() repo_name = models.CharField(max_length=150) repo_full_name = models.CharField(max_length=200) - owner = models.ForeignKey(GithubAppInstallation, on_delete=models.PROTECT) + app = models.ForeignKey(social_models.SocialApp, on_delete=models.PROTECT) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("repo_id", "owner") + unique_together = ("repo_id", "app") def __str__(self) -> str: return self.repo_full_name - def get_url(self): - return f"https://github.com/{self.repo_full_name}/pulls/" + +class UserRepoAccess(models.Model): + repository = models.ForeignKey(Repository, on_delete=models.CASCADE) + user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = ("repository", "user") class GithubRepoMap(models.Model): @@ -208,12 +85,12 @@ class IntegrationType(models.TextChoices): FULL = "FULL", "Full" PARTIAL = "PARTIAL", "Partial" - integration = models.ForeignKey(GithubAppInstallation, on_delete=models.PROTECT) + integration = models.ForeignKey(AppInstallation, on_delete=models.PROTECT) code_repo = models.ForeignKey( - GithubRepository, on_delete=models.PROTECT, related_name="code_repos" + Repository, on_delete=models.PROTECT, related_name="code_repos" ) documentation_repo = models.ForeignKey( - GithubRepository, on_delete=models.PROTECT, related_name="documentation_repos" + Repository, on_delete=models.PROTECT, related_name="documentation_repos" ) integration_type = models.CharField(max_length=20, choices=IntegrationType.choices) created_at = models.DateTimeField(auto_now_add=True) @@ -222,7 +99,7 @@ class Meta: unique_together = ("integration", "code_repo", "documentation_repo") -class GithubPullRequest(models.Model): +class PullRequest(models.Model): pr_id = models.BigIntegerField() pr_number = models.BigIntegerField() pr_head_commit_sha = models.CharField(max_length=40) @@ -238,7 +115,7 @@ class GithubPullRequest(models.Model): pr_merged = models.BooleanField(default=False) pr_owner_username = models.CharField(max_length=100) updated_on = models.DateTimeField(auto_now=True) - repository = models.ForeignKey(GithubRepository, on_delete=models.PROTECT) + repository = models.ForeignKey(Repository, on_delete=models.PROTECT) def __str__(self) -> str: return f"{self.repository.repo_full_name}/{self.pr_number} @ {self.pr_head_commit_sha[:7]}" @@ -267,12 +144,12 @@ class PullRequestStatus(models.TextChoices): MANUAL_APPROVAL = "MANUALLY_APPROVED", "Manual Approval" code_pull_request = models.OneToOneField( - GithubPullRequest, + PullRequest, on_delete=models.CASCADE, related_name="monitored_code", ) documentation_pull_request = models.ForeignKey( - GithubPullRequest, + PullRequest, on_delete=models.CASCADE, blank=True, null=True, @@ -283,7 +160,7 @@ class PullRequestStatus(models.TextChoices): choices=PullRequestStatus.choices, default=PullRequestStatus.NOT_CONNECTED, ) - integration = models.ForeignKey(GithubAppInstallation, on_delete=models.CASCADE) + integration = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) class Meta: unique_together = ("code_pull_request", "documentation_pull_request") @@ -340,7 +217,7 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -class GithubCheckRun(models.Model): +class PullRequestCheck(models.Model): run_id = models.BigIntegerField(blank=True, null=True) unique_id = models.UUIDField(default=uuid.uuid4) # Head of the request when it was run @@ -387,20 +264,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -class GithubLoginState(models.Model): - """ - This model is used to store the state of the login process. Along with it, - it also stores the redirect URL for the state. - """ - - state = models.UUIDField(default=uuid.uuid4) - redirect_url = models.URLField() - created_at = models.DateTimeField(auto_now_add=True) - - def __str__(self) -> str: - return self.state.__str__() - - class GithubMarketplaceEvent(models.Model): payload = models.JSONField(default=dict) created_at = models.DateTimeField(auto_now_add=True) diff --git a/app/signals.py b/app/signals.py index 5206147..fe57059 100644 --- a/app/signals.py +++ b/app/signals.py @@ -1,165 +1,78 @@ import logging +import django_rq import github + +from app import gitlab_jobs + +# from app import github_jobs from . import lib as app_lib from django.conf import settings from django.db.models.signals import post_save from django.dispatch import receiver from . import models as app_models +from allauth.account.signals import user_signed_up +from allauth.socialaccount.signals import social_account_added +from allauth.socialaccount import models as social_models logger = logging.getLogger(__name__) -# GithubAppInstallation -# @receiver( -# post_save, -# sender=app_models.GithubAppInstallation, -# dispatch_uid="update_github_repositories_for_installation", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# logger.info(f"GithubAppInstallation signal received for {instance}") -# github_manager = app_lib.GithubDataManager( -# instance.installation_id, instance.creator.get_active_access_token() -# ) -# github_manager.sync_repositories() - - -# The job is for syncing PRs, invoke separately for both code and documentation repo -# @receiver( -# post_save, -# sender=app_models.GithubRepoMap, -# dispatch_uid="invoke_actions_on_repo_map_save", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# logger.info(f"GithubRepoMap signal received for {instance}") -# installation_instance = instance.integration -# github_data_manager = app_lib.GithubDataManager( -# installation_id=installation_instance.installation_id, -# user_token=installation_instance.creator.get_active_access_token(), -# ) -# github_data_manager.sync_open_prs(instance.code_repo) -# github_data_manager.sync_open_prs(instance.documentation_repo) - -# @receiver( -# post_save, -# sender=app_models.GithubPullRequest, -# dispatch_uid="invoke_actions_on_pr_update", -# ) -# def pull_request_post_save(sender, instance, **kwargs): -# for documentation_pr in app_models.MonitoredPullRequest.objects.filter( -# documentation_pull_request=instance -# ).iterator(): -# if documentation_pr.is_approved: -# documentation_pr.pull_request_status = ( -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_CODE -# ) -# documentation_pr.save() -# for code_pr in app_models.MonitoredPullRequest.objects.filter( -# code_pull_request=instance -# ).iterator(): -# if code_pr.is_approved: -# code_pr.pull_request_status = ( -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_APPROVAL -# ) -# code_pr.save() +def new_installation(social_account): + extra_data = social_account.extra_data + if social_account.provider == "gitlab": + (app_installation, _) = app_models.AppInstallation.objects.update_or_create( + social_app=social_models.SocialApp.objects.get( + provider=social_account.provider + ), + installation_id=extra_data["id"], + creator=social_account.user, + state=app_models.AppInstallation.InstallationState.INSTALLED, + account_name=extra_data["username"], + account_id=extra_data["id"], + account_type="User", + avatar_url=social_account.get_avatar_url(), + ) + app_models.AppUser.objects.update_or_create( + installation=app_installation, user=social_account.user + ) + django_rq.enqueue( + gitlab_jobs.sync_repositories_for_installation, social_account + ) + elif social_account.provider == "github": + github_instance = github.Github(app_lib.get_active_token(social_account).token) + for installation in github_instance.get_user().get_installations(): + if installation.app_id == settings.GITHUB_CREDS["app_id"]: + raw_data = installation.raw_data + app_installation = app_models.AppInstallation.objects.get( + social_app=social_models.SocialApp.objects.get( + provider=social_account.provider + ), + installation_id=installation.id, + ) + app_models.AppUser.objects.update_or_create( + installation=app_installation, user=social_account.user + ) + # TODO: Schedule github repo sync + # django_rq.enqueue(github_jobs., app_installation) -# if instance.repository.code_repos.exists(): -# installation_instance = instance.repository.code_repos.last().integration -# ( -# monitored_pr_instance, -# is_new, -# ) = app_models.MonitoredPullRequest.objects.get_or_create( -# code_pull_request=instance, integration=installation_instance -# ) -# if is_new is False: -# monitored_pr_instance.save() +# def sync_repos(social_account): +# # assert False, social_account +# if social_account.provider == "gitlab": +# django_rq.enqueue(gitlab_jobs.sync_repositories_for_installation) -# @receiver( -# post_save, -# sender=app_models.MonitoredPullRequest, -# dispatch_uid="invoke_github_check_for_pr", -# ) -# def update_check_run(sender, instance, **kwargs): -# (instance, is_new) = app_models.GithubCheckRun.objects.get_or_create( -# ref_pull_request=instance, run_sha=instance.code_pull_request.pr_head_commit_sha -# ) -# if is_new: -# # Here, we can close the previous checks if any -# # app_models.GithubCheckRun.objects.filter(ref_pull_request=instance).exclude( -# # run_sha=instance.code_pull_request.pr_head_commit_sha, -# # ) -# pass -# else: -# instance.save() +@receiver(user_signed_up) +def _user_signed_up(request, user, *args, **kwargs): + # User signed up. Create + social_accounts = user.socialaccount_set.all() + for social_account in social_accounts: + new_installation(social_account) + # sync_repos(social_account) -# @receiver( -# post_save, sender=app_models.GithubCheckRun, dispatch_uid="update_github_check" -# ) -# def synchronize_github_check(sender, instance, **kwargs): -# token = ( -# instance.ref_pull_request.code_pull_request.repository.owner.creator.access_token -# ) -# github_app_instance = github.Github(token) -# github_repo = github_app_instance.get_repo( -# instance.ref_pull_request.code_pull_request.repository.repo_id -# ) -# check_run = github_repo.get_check_run(instance.run_id) -# data = { -# "name": settings.GITHUB_CREDS["app_name"], -# "head_sha": instance.run_sha, -# "external_id": instance.unique_id.__str__(), -# "status": "completed", -# "conclusion": "action_required", -# "details_url": f"{settings.WEBSITE_HOST}/{github_repo.full_name}/pulls/", -# } -# if ( -# instance.ref_pull_request.pull_request_status -# == app_models.MonitoredPullRequest.PullRequestStatus.NOT_CONNECTED -# ): -# # Update the action with the PR connection action -# data.update( -# { -# "conclusion": "action_required", -# "output": { -# "title": "Documentation PR is not connected", -# "summary": "Please connect the documentation PR", -# "text": "You can connect the documentation PR by clicking on the button below.", -# }, -# } -# ) -# elif instance.ref_pull_request.pull_request_status in [ -# app_models.MonitoredPullRequest.PullRequestStatus.APPROVED, -# app_models.MonitoredPullRequest.PullRequestStatus.MANUAL_APPROVAL, -# ]: -# data.update( -# { -# "conclusion": "success", -# "output": { -# "title": "", -# "summary": "", -# "text": "", -# }, -# } -# ) -# elif instance.ref_pull_request.pull_request_status in [ -# app_models.MonitoredPullRequest.PullRequestStatus.APPROVAL_PENDING, -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_APPROVAL, -# app_models.MonitoredPullRequest.PullRequestStatus.STALE_CODE, -# ]: -# # Update the action with the PR approval action -# data.update( -# { -# "conclusion": "action_required", -# "output": { -# "title": "Approve the PR on CDOC", -# "summary": "Please approve the PR on CDOC", -# "text": "You can connect the documentation PR by clicking on the button below.", -# }, -# } -# ) -# logger.info("Updating github check with data", extra={"payload": data}) -# check_run_instance = check_run.edit(**data) -# logger.info("Updated the check run", extra={"response": check_run_instance}) +@receiver(social_account_added) +def _social_account_added(request, socialaccount, *args, **kwargs): + new_installation(socialaccount) + # sync_repos(socialaccount) diff --git a/app/templates/index.html b/app/templates/index.html index fd6bc70..731e9e8 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,4 +1,5 @@ {% extends 'base.html' %} +{% load socialaccount %} {% block content %} {% if request.user.is_authenticated %} @@ -15,7 +16,7 @@
- +
- + - + - + @@ -155,17 +159,27 @@
- Login to Github + Login via
Click below to login with your Github account:
-
- -
+
+ {% csrf_token %} + +
+ +
+ {% csrf_token %} + +
+
@@ -184,7 +198,7 @@ function objectifyForm(formArray) { var data = {}; - $(formArray).each(function(index, obj){ + $(formArray).each(function (index, obj) { data[obj.name] = obj.value; }); return data; @@ -192,47 +206,49 @@ // $("#open-new-repo-connection-modal").click(function(){ // $("#attach-documentation-modal").addClass("opened"); // }); - $(".open-connect-pr-modal").click(function() { + $(".open-connect-pr-modal").click(function () { $("#code-or-doc-repo-modal span.repo_title").text($(this).attr("data-repo-full-name")); - $("#code-or-doc-repo-modal #code-repo-btn").attr("data-code-repo-full-name", $(this).attr("data-repo-full-name")); + $("#code-or-doc-repo-modal #code-repo-btn").attr("data-code-repo-full-name", $(this).attr( + "data-repo-full-name")); $("#code-or-doc-repo-modal #code-repo-btn").attr("data-code-repo-id", $(this).attr("data-repo-id")); - $("#code-or-doc-repo-modal #doc-repo-btn").attr("data-doc-repo-full-name", $(this).attr("data-repo-full-name")); + $("#code-or-doc-repo-modal #doc-repo-btn").attr("data-doc-repo-full-name", $(this).attr( + "data-repo-full-name")); $("#code-or-doc-repo-modal #doc-repo-btn").attr("data-doc-repo-id", $(this).attr("data-repo-id")); $("#code-or-doc-repo-modal").addClass("opened"); }) - $("#code-or-doc-repo-modal #code-repo-btn").click(function() { - + $("#code-or-doc-repo-modal #code-repo-btn").click(function () { + // Set code repo val and reset doc repo val $('select[name="code_repo_id"]').val($(this).attr("data-code-repo-id")); - $('select[name="documentation_repo_id"]').prop('selectedIndex',0); + $('select[name="documentation_repo_id"]').prop('selectedIndex', 0); $("#attach-documentation-modal div.modal-title span").first().text("Attach Documentation repository"); - $("#attach-documentation-modal .for-repo-text").text("For "+ $(this).attr("data-code-repo-full-name")); + $("#attach-documentation-modal .for-repo-text").text("For " + $(this).attr("data-code-repo-full-name")); // Show hide appropriate fields $("#attach-documentation-modal .code-repo-field").hide(); $("#attach-documentation-modal .doc-repo-field").show(); - + // Open the modal $("#attach-documentation-modal").addClass("opened"); - + // Close previous modal $("#code-or-doc-repo-modal").removeClass("opened"); }); - $("#code-or-doc-repo-modal #doc-repo-btn").click(function() { + $("#code-or-doc-repo-modal #doc-repo-btn").click(function () { // Set code repo val and reset doc repo val $('select[name="documentation_repo_id"]').val($(this).attr("data-doc-repo-id")); - $('select[name="code_repo_id"]').prop('selectedIndex',0); + $('select[name="code_repo_id"]').prop('selectedIndex', 0); $("#attach-documentation-modal div.modal-title span").first().text("Attach Code repository"); // Show hide appropriate fields $("#attach-documentation-modal .doc-repo-field").hide(); $("#attach-documentation-modal .code-repo-field").show(); - $("#attach-documentation-modal .for-repo-text").text("For "+ $(this).attr("data-doc-repo-full-name")); + $("#attach-documentation-modal .for-repo-text").text("For " + $(this).attr("data-doc-repo-full-name")); // Open the modal $("#attach-documentation-modal").addClass("opened"); - + // Close previous modal $("#code-or-doc-repo-modal").removeClass("opened"); @@ -243,6 +259,7 @@ // $("#attach-documentation-modal .doc-repo-field").hide(); // $("#attach-documentation-modal").addClass("opened"); }); + function getCookie(name) { let cookieValue = null; if (document.cookie && document.cookie !== '') { @@ -267,11 +284,11 @@ "mode": 'same-origin' // Do not send CSRF token to another domain. } }); - - $("#attach-repos-form").submit(function() { + + $("#attach-repos-form").submit(function () { data = objectifyForm($(this).serializeArray()); - if (data["documentation_repo_id"] == data["code_repo_id"]){ + if (data["documentation_repo_id"] == data["code_repo_id"]) { alert("Documentation and Code Repository cannot be same"); return false; } @@ -281,21 +298,21 @@ method: $(this).attr("method"), data: JSON.stringify(data), dataType: 'json', - success: function(data) { + success: function (data) { location.reload(); } }) return false; }); - $(".clickable-row").click(function(){ + $(".clickable-row").click(function () { window.location.href = $(this).find("a.cta").attr("href") }); {% else %} diff --git a/app/urls.py b/app/urls.py index 1a01b00..a3ce56b 100644 --- a/app/urls.py +++ b/app/urls.py @@ -1,20 +1,20 @@ from django.urls import path, include from . import views +from . import github_views from . import gitlab_views gitlab_urlpatterns = [ - path( - "auth-callback/", - gitlab_views.AuthCallback.as_view(), - name="gitlab_auth_callback", - ), - path("initiate-login/", gitlab_views.InitiateLoginView.as_view()), + # path( + # "auth-callback/", + # gitlab_views.AuthCallback.as_view(), + # name="gitlab_auth_callback", + # ), ] urlpatterns = [ - path("callback/", views.AuthCallback.as_view()), - path("webhook-callback/", views.WebhookCallback.as_view()), - path("oauth-callback/", views.OauthCallback.as_view()), - path("marketplace-callback/", views.MarketplaceCallbackView.as_view()), + path("callback/", github_views.AuthCallback.as_view()), + path("webhook-callback/", github_views.WebhookCallback.as_view()), + path("oauth-callback/", github_views.OauthCallback.as_view()), + path("marketplace-callback/", github_views.MarketplaceCallbackView.as_view()), path("gitlab/", include(gitlab_urlpatterns)), ] diff --git a/app/views.py b/app/views.py index 8b4ee3b..5d90ae8 100644 --- a/app/views.py +++ b/app/views.py @@ -36,321 +36,6 @@ logger = logging.getLogger(__name__) -@method_decorator(csrf_exempt, name="dispatch") -class AuthCallback(View): - def get_okind_ekind(view, request, *args, **kwargs): - if "installation_id" in request.GET: - if request.GET.get("setup_action") == "update": - return ( - analytics_events.OKind.INSTALLATION, - analytics_events.EKind.UPDATE, - ) - else: - return ( - analytics_events.OKind.INSTALLATION, - analytics_events.EKind.CREATE, - ) - return (analytics_events.OKind.USER, analytics_events.EKind.LOGIN) - - @log_http_event(oid=1, okind_ekind=get_okind_ekind) - def get(self, request, *args, **kwargs): - """ - This method is invoked when the user installs the application and - generates the auth token for the user to be used in the future. - Expected inputs as urlparams: - - code: str - - installation_id: int - - setup_action: str - """ - code: str = request.GET["code"] - - # logger.info(request.GET) - # github_app_auth_user = app_models.GithubAppAuth.objects.create( - # code=code, - # installation_id=installation_id, - # setup_action=setup_action, - # ) - # Get the App installation account for managing the various scopes for a user - # Example, User A can be a part of 2 installations. So, the user will see them as two different projects - # in the UI. - # The user's account using the `code` recieved in the step above - # logger.info(request.get_host()) - payload = { - "client_id": settings.GITHUB_CREDS["client_id"], - "client_secret": settings.GITHUB_CREDS["client_secret"], - "code": code, - "redirect_uri": f"https://{request.get_host()}/app/callback/", - "state": uuid.uuid4().__str__(), - } - resp = requests.post( - "https://github.com/login/oauth/access_token", json=payload - ) - logger.info(resp.text) - if resp.ok: - response = parse_qs(resp.text) - if "error" in response: - logger.error(response) - else: - now = timezone.now() - access_token = response["access_token"][0] - logger.info(response) - access_token_expires_at = now + datetime.timedelta( - seconds=int(response["expires_in"][0]) - ) - refresh_token = response["refresh_token"][0] - refresh_token_expires_at = now + datetime.timedelta( - seconds=int(response["refresh_token_expires_in"][0]) - ) - logger.info(access_token) - github_instance = github.Github(access_token) - user_instance = github_instance.get_user() - with transaction.atomic(): - access_group = auth_models.Group.objects.get(name="github_user") - (auth_user_instance, _) = get_user_model().objects.get_or_create( - username=user_instance.login, - defaults={ - "is_active": True, - "is_staff": True, - }, - ) - auth_user_instance.groups.add(access_group) - (github_user, _) = app_models.GithubUser.objects.update_or_create( - account_id=user_instance.id, - user=auth_user_instance, - defaults={ - "account_type": user_instance.type, - "account_name": user_instance.login, - "avatar_url": user_instance.avatar_url, - }, - ) - app_models.GithubUserAccessToken.objects.create( - token=access_token, - expires_at=access_token_expires_at, - github_user=github_user, - ) - app_models.GithubUserRefreshToken.objects.create( - token=refresh_token, - expires_at=refresh_token_expires_at, - github_user=github_user, - ) - login(request, auth_user_instance) - if "installation_id" in request.GET: - installation_id: int = request.GET["installation_id"] - setup_action: str = request.GET["setup_action"] - - github_installation_manager = lib.GithubInstallationManager( - installation_id=installation_id, user_token=access_token - ) - installation_details = ( - github_installation_manager.get_installation_details() - ) - account_details = installation_details["account"] - account_name = account_details["login"] - account_id = account_details["id"] - account_type = account_details["type"] - avatar_url = account_details["avatar_url"] - - with transaction.atomic(): - - ( - installation_instance, - is_new, - ) = app_models.GithubAppInstallation.objects.update_or_create( - installation_id=installation_id, - account_id=account_id, - defaults={ - "account_name": account_name, - "state": app_models.GithubAppInstallation.InstallationState.INSTALLED, - "account_type": account_type, - "avatar_url": avatar_url, - "creator": github_user, - }, - ) - # installation_instance.save() - installation_instance.update_token() - # if is_new: - django_rq.enqueue( - app_jobs.sync_repositories_for_installation, - installation_instance, - ) - app_models.GithubAppUser.objects.update_or_create( - github_user=github_user, - installation=installation_instance, - ) - # logger.info([x for x in user_instance.get_installations()]) - for installation in user_instance.get_installations(): - if installation.app_id == settings.GITHUB_CREDS["app_id"]: - installation_instance = ( - app_models.GithubAppInstallation.objects.get( - account_id=installation.target_id, - installation_id=installation.id, - ) - ) - app_models.GithubAppUser.objects.update_or_create( - github_user=github_user, - installation=installation_instance, - ) - else: - logger.error(f"Unable to get token from the code: {resp.text}") - return HttpResponseRedirect(reverse("initiate_github_login")) - redirect_url = None - - if "state" in request.GET: - next_url = app_models.GithubLoginState.objects.get( - state=request.GET["state"] - ).redirect_url - if next_url is not None and next_url != "": - redirect_url = next_url - return HttpResponseRedirect(redirect_url or "/") - - -@method_decorator(csrf_exempt, name="dispatch") -class WebhookCallback(View): - def post(self, request, *args, **kwargs): - headers = request.headers - body = request.body - is_verified = lib.verify_signature( - headers["X-Hub-Signature-256"], - body, - settings.GITHUB_CREDS["app_signature_secret"], - ) - payload = json.loads(body) - if not is_verified: - return HttpResponse("Invalid signature", status=403) - logger.info( - "Recieved Github webhook", - extra={"data": {"headers": headers, "payload": payload}}, - ) - EVENT_TYPE = headers.get("X-Github-Event", None) - # header is "installation_repositories" -> Updated the repositories installed for the installation - interesting_events = [ - "pull_request", - "installation_repositories", - "check_suite", - "installation", - ] - if EVENT_TYPE in interesting_events: - get_installation_instance = ( - lambda data: app_models.GithubAppInstallation.objects.get( - installation_id=data["installation"]["id"] - ) - ) - installation_instance = get_installation_instance(payload) - if EVENT_TYPE == "installation": - should_save = False - if payload["action"] == "deleted": - installation_instance.state = ( - app_models.GithubAppInstallation.InstallationState.UNINSTALLED - ) - should_save = True - elif payload["action"] == "suspend": - installation_instance.state = ( - app_models.GithubAppInstallation.InstallationState.SUSPENDED - ) - should_save = True - elif payload["action"] == "unsuspend": - installation_instance.state = ( - app_models.GithubAppInstallation.InstallationState.INSTALLED - ) - should_save = True - if should_save: - installation_instance.save() - elif EVENT_TYPE == "pull_request": - pull_request_data = payload["pull_request"] - (github_repo, _) = app_models.GithubRepository.objects.update_or_create( - repo_id=payload["repository"]["id"], - owner=installation_instance, - defaults={ - "repo_full_name": payload["repository"]["full_name"], - "repo_name": payload["repository"]["name"], - }, - ) - ( - pr_instance, - _, - ) = app_models.GithubPullRequest.objects.update_or_create( - pr_id=pull_request_data["id"], - pr_number=pull_request_data["number"], - repository=github_repo, - defaults={ - "pr_head_commit_sha": pull_request_data["head"]["sha"], - "pr_title": pull_request_data["title"], - "pr_body": pull_request_data["body"], - "pr_state": pull_request_data["state"], - "pr_created_at": pull_request_data["created_at"], - "pr_updated_at": pull_request_data["updated_at"], - "pr_merged_at": pull_request_data["merged_at"], - "pr_closed_at": pull_request_data["closed_at"], - "pr_merged": pull_request_data["merged"], - "pr_owner_username": pull_request_data["user"]["login"], - }, - ) - django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) - elif EVENT_TYPE == "installation_repositories": - # Repositories changed. Sync again. - django_rq.enqueue( - app_jobs.sync_repositories_for_installation, - installation_instance, - ) - elif EVENT_TYPE == "check_suite": - if payload.get("action") == "requested": - - repository_data = payload.get("repository", {}) - ( - github_repo, - _, - ) = app_models.GithubRepository.objects.update_or_create( - repo_id=repository_data["id"], - owner=installation_instance, - defaults={ - "repo_name": repository_data["name"], - "repo_full_name": repository_data["full_name"], - }, - ) - prs = payload.get("check_suite", {}).get("pull_requests", []) - - head_commit_details = payload["check_suite"]["head_commit"] - head_commit_data = { - "pr_head_commit_sha": head_commit_details["id"], - # "pr_head_tree_sha": head_commit_details["tree_id"], - "pr_head_commit_message": head_commit_details["message"], - "pr_head_modified_on": datetime.datetime.strptime( - head_commit_details["timestamp"], "%Y-%m-%dT%H:%M:%S%z" - ), - } - - logger.info( - f"Found PRs for commit ID: {head_commit_details['id']}", - extra={"data": {"pull_requests": prs}}, - ) - # is_documentation_repo = - if ( - github_repo.code_repos.exists() - or github_repo.documentation_repos.exists() - ): - for pr in prs: - pr_id = pr.get("id") - pr_number = pr.get("number") - with transaction.atomic(): - ( - pr_instance, - is_new, - ) = app_models.GithubPullRequest.objects.update_or_create( - pr_id=pr_id, - pr_number=pr_number, - repository=github_repo, - defaults={**head_commit_data}, - ) - django_rq.enqueue(app_jobs.on_pr_update, pr_instance.id) - return JsonResponse({"status": True}) - - -@method_decorator(csrf_exempt, name="dispatch") -class OauthCallback(View): - def get(self, request): - assert False, request - - @method_decorator(login_required, name="dispatch") class AllPRView(TemplateView): template_name = "all_pr_for_org.html" @@ -533,16 +218,16 @@ def post(self, request, *args, **kwargs): class AppIndexPage(TemplateView): template_name = "index.html" - def get_okind_ekind(view, request, *args, **kwargs): - if request.method == "GET": - if request.user.is_authenticated: - return (analytics_events.OKind.ORGANIZATION, analytics_events.EKind.GET) - else: - return (analytics_events.OKind.VISIT, analytics_events.EKind.CREATE) - else: - return (analytics_events.OKind.REPOSITORY, analytics_events.EKind.CONNECT) + # def get_okind_ekind(view, request, *args, **kwargs): + # if request.method == "GET": + # if request.user.is_authenticated: + # return (analytics_events.OKind.ORGANIZATION, analytics_events.EKind.GET) + # else: + # return (analytics_events.OKind.VISIT, analytics_events.EKind.CREATE) + # else: + # return (analytics_events.OKind.REPOSITORY, analytics_events.EKind.CONNECT) - @log_http_event(oid=1, okind_ekind=get_okind_ekind) + # @log_http_event(oid=1, okind_ekind=get_okind_ekind) def post(self, request, *args, **kwargs): payload = json.loads(request.body) payload["all_installations"] = app_models.GithubAppInstallation.objects.filter( @@ -566,53 +251,55 @@ def post(self, request, *args, **kwargs): ) return JsonResponse({"success": True}) - @log_http_event(oid=1, okind_ekind=get_okind_ekind) + # @log_http_event(oid=1, okind_ekind=get_okind_ekind) def get(self, request, *args, **kwargs) -> HttpResponse: return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: context = super().get_context_data(**kwargs) if self.request.user.is_authenticated: - all_installations = app_models.GithubAppInstallation.objects.filter( - id__in=app_models.GithubAppUser.objects.filter( - github_user=self.request.user.github_user - ).values_list("installation_id", flat=True), - state=app_models.GithubAppInstallation.InstallationState.INSTALLED, - ) - context["all_installations"] = all_installations + # all_installations = app_models.AppInstallation.objects.filter( + # id__in=app_models.AppUser.objects.filter( + # user=self.request.user + # ).values_list("installation_id", flat=True), + # state=app_models.AppInstallation.InstallationState.INSTALLED, + # ) + # context["all_installations"] = all_installations context["all_repo_map"] = app_models.GithubRepoMap.objects.filter( - integration__in=all_installations + code_repo__userrepoaccess__user=self.request.user, + documentation_repo__userrepoaccess__user=self.request.user, ) context[ "available_repos_for_mapping" - ] = app_models.GithubRepository.objects.filter( - owner__in=all_installations, + ] = app_models.Repository.objects.filter( + userrepoaccess__user=self.request.user, code_repos__isnull=True, documentation_repos__isnull=True, ) context["unmapped_repos_display"] = context["available_repos_for_mapping"] search_query = self.request.GET.get("q") if search_query: - context["all_repo_map"] = context["all_repo_map"].filter( - Q(code_repo__repo_full_name__icontains=search_query) - | Q(documentation_repo__repo_full_name__icontains=search_query) - ) + # context["all_repo_map"] = context["all_repo_map"].filter( + # Q(code_repo__repo_full_name__icontains=search_query) + # | Q(documentation_repo__repo_full_name__icontains=search_query) + # ) context["unmapped_repos_display"] = context[ "unmapped_repos_display" ].filter(repo_full_name__icontains=search_query) context["q"] = search_query - context["all_repo_map"] = context["all_repo_map"][:10] + # context["all_repo_map"] = context["all_repo_map"][:10] context["unmapped_repos_display"] = context["unmapped_repos_display"][:10] else: - login_state_instance = app_models.GithubLoginState() - if self.request.GET.get("next"): - login_state_instance.redirect_url = self.request.GET.get("next") - login_state_instance.save() + pass + # login_state_instance = app_models.GithubLoginState() + # if self.request.GET.get("next"): + # login_state_instance.redirect_url = self.request.GET.get("next") + # login_state_instance.save() url = "https://github.com/login/oauth/authorize" params = { "client_id": settings.GITHUB_CREDS["client_id"], "allow_signup": False, - "state": login_state_instance.state.__str__(), + # "state": login_state_instance.state.__str__(), } req = PreparedRequest() req.prepare_url(url, params) @@ -635,18 +322,19 @@ def get(self, request, *args, **kwargs): return HttpResponseRedirect(request.GET.get("next")) return HttpResponseRedirect("/") else: - login_state_instance = app_models.GithubLoginState() - if self.request.GET.get("next"): - login_state_instance.redirect_url = self.request.GET.get("next") - login_state_instance.save() + # login_state_instance = app_models.GithubLoginState() + # if self.request.GET.get("next"): + # login_state_instance.redirect_url = self.request.GET.get("next") + # login_state_instance.save() url = "https://github.com/login/oauth/authorize" params = { "client_id": settings.GITHUB_CREDS["client_id"], "allow_signup": False, - "state": login_state_instance.state.__str__(), + # "state": login_state_instance.state.__str__(), } req = PreparedRequest() req.prepare_url(url, params) + assert False, req.url return HttpResponseRedirect(req.url) @@ -657,19 +345,3 @@ def get_context_data(self, *args, **kwargs): context = super(IndexView, self).get_context_data(*args, **kwargs) context["asd"] = "Message from context" return context - - -@method_decorator(csrf_exempt, name="dispatch") -class MarketplaceCallbackView(View): - def post(self, request, *args, **kwargs): - body = request.body - is_verified = lib.verify_signature( - request.headers["X-Hub-Signature-256"], - body, - settings.GITHUB_CREDS["marketplace_signature_secret"], - ) - if not is_verified: - return HttpResponse("Invalid signature", status=403) - payload = json.loads(body) - app_models.GithubMarketplaceEvent.objects.create(payload=payload) - return JsonResponse({"status": True}) diff --git a/cdoc/settings.py b/cdoc/settings.py index 05b6023..7036e59 100644 --- a/cdoc/settings.py +++ b/cdoc/settings.py @@ -63,6 +63,18 @@ "django_rq", ] +SOCIALACCOUNT_PROVIDERS = { + "gitlab": { + "SCOPE": [ + "api", + ], + }, + # "github": {"SCOPE": []}, +} + +LOGIN_REDIRECT_URL = "/" +ACCOUNT_EMAIL_VERIFICATION = "none" +SOCIALACCOUNT_STORE_TOKENS = True MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", diff --git a/cdoc/urls.py b/cdoc/urls.py index 5b3f9b8..76f144c 100644 --- a/cdoc/urls.py +++ b/cdoc/urls.py @@ -23,7 +23,10 @@ import glob +import gitlab + from app import views as app_views +from app import gitlab_views as gitlab_views urlpatterns = [ path("django-rq/", include("django_rq.urls")), @@ -31,6 +34,10 @@ "accounts/logout/", auth_views.LogoutView.as_view(next_page="/"), ), + # path( + # "accounts/gitlab/login/callback/", + # gitlab_views.AuthCallback.as_view(), + # ), path("accounts/", include("allauth.urls")), # path( # "/repos/", @@ -73,7 +80,7 @@ def s(p, **data): d = json.load(open(p)) p = p.replace("samples/", "").replace(".json", "") - print(p, d) + # print(p, d) return ( path("samples/" + ("" if p == "index" else p + "/"), make_v(d["template"], d)), From e1fd29c3b7de330f828621148676a90450ae3189 Mon Sep 17 00:00:00 2001 From: Shobhit Date: Mon, 30 May 2022 09:37:32 +0530 Subject: [PATCH 03/15] Overhaul 2. Setup gitlab data imports and webhooks --- app/github_views.py | 10 +- app/gitlab_jobs.py | 93 ++++++++++- app/gitlab_views.py | 31 +++- app/jobs.py | 2 +- ..._githubrepomap_unique_together_and_more.py | 21 +++ ...repomap_monitoredrepositorymap_and_more.py | 43 +++++ .../0025_alter_userrepoaccess_access.py | 18 ++ ...nitoredpullrequest_integration_and_more.py | 22 +++ app/models.py | 27 ++- app/signals.py | 28 ++-- app/templates/index.html | 10 +- app/urls.py | 10 +- app/views.py | 155 ++++++------------ cdoc/urls.py | 14 +- 14 files changed, 330 insertions(+), 154 deletions(-) create mode 100644 app/migrations/0023_alter_githubrepomap_unique_together_and_more.py create mode 100644 app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py create mode 100644 app/migrations/0025_alter_userrepoaccess_access.py create mode 100644 app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py diff --git a/app/github_views.py b/app/github_views.py index cec994a..e75104a 100644 --- a/app/github_views.py +++ b/app/github_views.py @@ -138,14 +138,16 @@ def get(self, request, *args, **kwargs): uid=user_instance.id, extra_data={}, ) - allauth_social_models.SocialToken.objects.create( - token=access_token, - token_secret=refresh_token, + allauth_social_models.SocialToken.objects.update_or_create( account=social_account, - expires_at=access_token_expires_at, app=allauth_social_models.SocialApp.objects.get( provider="github" ), + defaults={ + "token": access_token, + "token_secret": refresh_token, + "expires_at": access_token_expires_at, + }, ) login(request, auth_user_instance) if "installation_id" in request.GET: diff --git a/app/gitlab_jobs.py b/app/gitlab_jobs.py index f12521a..a9880a0 100644 --- a/app/gitlab_jobs.py +++ b/app/gitlab_jobs.py @@ -3,6 +3,7 @@ from app import models as app_models from allauth.socialaccount import models as social_models from app import lib as app_lib +from django.conf import settings @job @@ -10,16 +11,100 @@ def sync_repositories_for_installation(social_account: social_models.SocialAccou gitlab_instance = gitlab.Gitlab( oauth_token=app_lib.get_active_token(social_account).token ) + existing_mapping = list( + app_models.UserRepoAccess.objects.filter( + social_account=social_account + ).values_list("id", flat=True) + ) for project in gitlab_instance.projects.list(all=True, membership=True): (repo_instance, _) = app_models.Repository.objects.update_or_create( repo_id=project.id, app=social_models.SocialApp.objects.get(provider="gitlab"), defaults={ - "repo_name": project.name, - "repo_full_name": project.name_with_namespace, + "repo_name": project.name.replace(" ", ""), + "repo_full_name": project.name_with_namespace.replace(" ", ""), }, ) - app_models.UserRepoAccess.objects.update_or_create( + project_perms = project._attrs["permissions"] + user_access_level = max( + (project_perms.get("project_access") or {}).get("access_level", 0), + (project_perms.get("group_access") or {}).get("access_level", 0), + ) + + (instance, is_new) = app_models.UserRepoAccess.objects.update_or_create( repository=repo_instance, - user=social_account.user, + social_account=social_account, + defaults={"access": user_access_level}, + ) + if is_new is False: + existing_mapping.pop(existing_mapping.index(instance.id)) + app_models.UserRepoAccess.objects.filter(id__in=existing_mapping).update( + access=app_models.UserRepoAccess.AccessLevel.NO_ACCESS + ) + + +@job +def sync_prs_for_repository( + project_id: int, +): + repository_instance = app_models.Repository.objects.get( + repo_id=project_id, + app__provider="gitlab", + ) + highest_access_user = ( + repository_instance.userrepoaccess_set.all().order_by("-access").first() + ) + if highest_access_user is None: + return + gitlab_instance = gitlab.Gitlab( + oauth_token=app_lib.get_active_token(highest_access_user.social_account).token + ) + project_instance = gitlab_instance.projects.get(repository_instance.repo_id) + for merge_request in project_instance.mergerequests.list(): + try: + head_commit = merge_request.commits().next() + except StopIteration: + head_commit = object() + (pr_instance, _) = app_models.PullRequest.objects.update_or_create( + pr_id=merge_request.id, + pr_number=merge_request.iid, + repository=repository_instance, + defaults={ + "pr_title": merge_request.title, + "pr_owner_username": merge_request.author["username"], + "pr_head_commit_sha": getattr(head_commit, "id", None) or "", + "pr_head_modified_on": getattr( + head_commit, "authored_date", merge_request.created_at + ), + "pr_head_commit_message": getattr( + head_commit, "title", merge_request.title + ) + or "", + "pr_body": merge_request.description, + "pr_state": merge_request.state, + "pr_created_at": merge_request.created_at, + "pr_updated_at": merge_request.updated_at, + "pr_merged_at": merge_request.merged_at, + "pr_closed_at": merge_request.closed_at, + "pr_merged": merge_request.state == "opened", + }, + ) + if repository_instance.code_repos.exists(): + app_models.MonitoredPullRequest.objects.update_or_create( + code_pull_request=pr_instance, + ) + # Setup a webhook for this project\ + webhook_url = f"https://{settings.DEPLOYMENT_HOST_NAME}/app/gitlab/webhook-callback/" + existing_hooks = project_instance.hooks.list() + for hook in existing_hooks: + if hook.url == webhook_url: + hook.merge_requests_events = True + hook.save() + if len(existing_hooks) == 0: + project_instance.hooks.create( + { + "url": webhook_url, + "merge_requests_events": True, + "push_events": False, + } ) diff --git a/app/gitlab_views.py b/app/gitlab_views.py index e340e10..478f62c 100644 --- a/app/gitlab_views.py +++ b/app/gitlab_views.py @@ -1,11 +1,36 @@ +import gitlab +from app import lib as app_lib from django.http import HttpResponseRedirect from django.urls import reverse from django.views import View from app import models as app_models from django.conf import settings from requests.models import PreparedRequest -import base64 -import hashlib -import requests +import json +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator +@method_decorator(csrf_exempt, name="dispatch") +class WebhookCallback(View): + def post(self, request, *args, **kwargs): + headers = request.headers + event_header = headers["X-Gitlab-Event"] + request_body = json.loads(request.body.decode("utf-8")) + gitlab_instance = gitlab.Gitlab(oauth_token=app_lib.get_active_token()) + if ( + event_header == "Merge Request Hook" + and request_body["object_kind"] == "merge_request" + ): + project_name = request_body["request_body"]["path_with_namespace"] + merge_request_id = request_body["object_attributes"]["iid"] + app_models.PullRequest.objects.update_or_create( + pr_number=merge_request_id, + pr_id=request_body["object_attributes"]["id"], + defaults={ + + } + ) + pass + assert False, (request.body, request.headers) + pass diff --git a/app/jobs.py b/app/jobs.py index 4aa2cf3..4edc9b7 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -28,7 +28,7 @@ def sync_repositories_for_installation( @job def sync_prs_for_repository(repository_id: int): logger.info(f"Sync PR job received for repository_id: {repository_id}") - instance = app_models.GithubRepository.objects.get(id=repository_id) + instance = app_models.Repository.objects.get(id=repository_id) installation_instance = instance.owner github_data_manager = app_lib.GithubDataManager( installation_id=installation_instance.installation_id, diff --git a/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py b/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py new file mode 100644 index 0000000..0efdb46 --- /dev/null +++ b/app/migrations/0023_alter_githubrepomap_unique_together_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.0.4 on 2022-05-28 10:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0022_alter_repository_unique_together_repository_app_and_more'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='githubrepomap', + unique_together={('code_repo', 'documentation_repo')}, + ), + migrations.RemoveField( + model_name='githubrepomap', + name='integration', + ), + ] diff --git a/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py b/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py new file mode 100644 index 0000000..35ea893 --- /dev/null +++ b/app/migrations/0024_rename_githubrepomap_monitoredrepositorymap_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.0.4 on 2022-05-28 13:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('app', '0023_alter_githubrepomap_unique_together_and_more'), + ] + + operations = [ + migrations.RenameModel( + old_name='GithubRepoMap', + new_name='MonitoredRepositoryMap', + ), + migrations.AlterUniqueTogether( + name='userrepoaccess', + unique_together=set(), + ), + migrations.AddField( + model_name='userrepoaccess', + name='access', + field=models.CharField(choices=[('OWNER', 'Owner'), ('MEMBER', 'Member')], default='MEMBER', max_length=20), + preserve_default=False, + ), + migrations.AddField( + model_name='userrepoaccess', + name='social_account', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='socialaccount.socialaccount'), + preserve_default=False, + ), + migrations.AlterUniqueTogether( + name='userrepoaccess', + unique_together={('repository', 'social_account')}, + ), + migrations.RemoveField( + model_name='userrepoaccess', + name='user', + ), + ] diff --git a/app/migrations/0025_alter_userrepoaccess_access.py b/app/migrations/0025_alter_userrepoaccess_access.py new file mode 100644 index 0000000..71431c4 --- /dev/null +++ b/app/migrations/0025_alter_userrepoaccess_access.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.4 on 2022-05-28 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0024_rename_githubrepomap_monitoredrepositorymap_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='userrepoaccess', + name='access', + field=models.IntegerField(choices=[('50', 'Owner'), ('40', 'Maintainer'), ('30', 'Developer'), ('20', 'Reporter'), ('10', 'Guest'), ('5', 'Minimal'), ('0', 'No Access')], max_length=20), + ), + ] diff --git a/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py b/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py new file mode 100644 index 0000000..48c7415 --- /dev/null +++ b/app/migrations/0026_remove_monitoredpullrequest_integration_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.0.4 on 2022-05-29 05:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0025_alter_userrepoaccess_access'), + ] + + operations = [ + migrations.RemoveField( + model_name='monitoredpullrequest', + name='integration', + ), + migrations.AlterField( + model_name='userrepoaccess', + name='access', + field=models.IntegerField(choices=[('50', 'Owner'), ('40', 'Maintainer'), ('30', 'Developer'), ('20', 'Reporter'), ('10', 'Guest'), ('5', 'Minimal'), ('0', 'No Access')]), + ), + ] diff --git a/app/models.py b/app/models.py index 4568114..b40f3fb 100644 --- a/app/models.py +++ b/app/models.py @@ -65,17 +65,32 @@ class Meta: def __str__(self) -> str: return self.repo_full_name + def pulls_page_url(self): + return f"/{self.app.provider}/{self.repo_full_name}/pulls/" + class UserRepoAccess(models.Model): + class AccessLevel(models.TextChoices): + OWNER = 50, "Owner" + MAINTAINER = 40, "Maintainer" + DEVELOPER = 30, "Developer" + REPORTER = 20, "Reporter" + GUEST = 10, "Guest" + MINIMAL = 5, "Minimal" + NO_ACCESS = 0, "No Access" + repository = models.ForeignKey(Repository, on_delete=models.CASCADE) - user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + social_account = models.ForeignKey( + social_models.SocialAccount, on_delete=models.CASCADE + ) + access = models.IntegerField(choices=AccessLevel.choices) created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("repository", "user") + unique_together = ("repository", "social_account") -class GithubRepoMap(models.Model): +class MonitoredRepositoryMap(models.Model): class IntegrationType(models.TextChoices): """ FULL integration restricts the PRs to process only after the related PR is merged @@ -85,7 +100,7 @@ class IntegrationType(models.TextChoices): FULL = "FULL", "Full" PARTIAL = "PARTIAL", "Partial" - integration = models.ForeignKey(AppInstallation, on_delete=models.PROTECT) + # integration = models.ForeignKey(AppInstallation, on_delete=models.PROTECT) code_repo = models.ForeignKey( Repository, on_delete=models.PROTECT, related_name="code_repos" ) @@ -96,7 +111,7 @@ class IntegrationType(models.TextChoices): created_at = models.DateTimeField(auto_now_add=True) class Meta: - unique_together = ("integration", "code_repo", "documentation_repo") + unique_together = ("code_repo", "documentation_repo") class PullRequest(models.Model): @@ -160,7 +175,7 @@ class PullRequestStatus(models.TextChoices): choices=PullRequestStatus.choices, default=PullRequestStatus.NOT_CONNECTED, ) - integration = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) + # integration = models.ForeignKey(AppInstallation, on_delete=models.CASCADE) class Meta: unique_together = ("code_pull_request", "documentation_pull_request") diff --git a/app/signals.py b/app/signals.py index fe57059..911a08c 100644 --- a/app/signals.py +++ b/app/signals.py @@ -44,15 +44,20 @@ def new_installation(social_account): for installation in github_instance.get_user().get_installations(): if installation.app_id == settings.GITHUB_CREDS["app_id"]: raw_data = installation.raw_data - app_installation = app_models.AppInstallation.objects.get( - social_app=social_models.SocialApp.objects.get( - provider=social_account.provider - ), - installation_id=installation.id, - ) - app_models.AppUser.objects.update_or_create( - installation=app_installation, user=social_account.user - ) + try: + app_installation = app_models.AppInstallation.objects.get( + social_app=social_models.SocialApp.objects.get( + provider=social_account.provider + ), + installation_id=installation.id, + ) + app_models.AppUser.objects.update_or_create( + installation=app_installation, user=social_account.user + ) + except app_models.AppInstallation.DoesNotExist: + # App installation not done. Skip mapping + # Error: The App installation not found. Configuration error + pass # TODO: Schedule github repo sync # django_rq.enqueue(github_jobs., app_installation) @@ -73,6 +78,5 @@ def _user_signed_up(request, user, *args, **kwargs): @receiver(social_account_added) -def _social_account_added(request, socialaccount, *args, **kwargs): - new_installation(socialaccount) - # sync_repos(socialaccount) +def _social_account_added(request, sociallogin, *args, **kwargs): + new_installation(sociallogin.account) diff --git a/app/templates/index.html b/app/templates/index.html index 58c3aa0..6416a91 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -40,7 +40,7 @@
- > + >
{% endfor %} @@ -81,7 +81,7 @@
- +
@@ -130,9 +130,9 @@
diff --git a/app/urls.py b/app/urls.py index a3ce56b..d44b386 100644 --- a/app/urls.py +++ b/app/urls.py @@ -5,11 +5,11 @@ from . import gitlab_views gitlab_urlpatterns = [ - # path( - # "auth-callback/", - # gitlab_views.AuthCallback.as_view(), - # name="gitlab_auth_callback", - # ), + path( + "webhook-callback/", + gitlab_views.WebhookCallback.as_view(), + name="gitlab_webhook_callback", + ), ] urlpatterns = [ path("callback/", github_views.AuthCallback.as_view()), diff --git a/app/views.py b/app/views.py index 5d90ae8..e230a3c 100644 --- a/app/views.py +++ b/app/views.py @@ -45,41 +45,41 @@ class AllPRView(TemplateView): okind_ekind=(analytics_events.OKind.REPOSITORY, analytics_events.EKind.GET), ) def get(self, request, *args: Any, **kwargs: Any): - github_user = request.user.github_user + # github_user = request.user.github_user - if github_user is None: - return Http404("User not found") + # if github_user is None: + # return Http404("User not found") context = self.get_context_data(**kwargs) - get_object_or_404( - app_models.GithubAppUser, - github_user=github_user, - installation=context["repo_mapping"].integration, - ) + # get_object_or_404( + # app_models.GithubAppUser, + # github_user=github_user, + # installation=context["repo_mapping"].integration, + # ) return self.render_to_response(context) def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: context = super().get_context_data(**kwargs) matches = self.request.resolver_match.kwargs - context["all_installations"] = app_models.GithubAppInstallation.objects.filter( - id__in=app_models.GithubAppUser.objects.filter( - github_user=self.request.user.github_user, - ).values_list("installation_id", flat=True), - state=app_models.GithubAppInstallation.InstallationState.INSTALLED, - ) - context["repo_mapping"] = app_models.GithubRepoMap.objects.get( - code_repo__repo_full_name__iexact="{}/{}".format( - matches["account_name"], matches["repo_name"] - ), - integration__in=context["all_installations"], - ) - current_installation = context["all_installations"].get( - account_name=matches["account_name"] + # assert False, matches + # context["all_installations"] = app_models.GithubAppInstallation.objects.filter( + # id__in=app_models.GithubAppUser.objects.filter( + # github_user=self.request.user.github_user, + # ).values_list("installation_id", flat=True), + # state=app_models.GithubAppInstallation.InstallationState.INSTALLED, + # ) + context["repo_mapping"] = app_models.MonitoredRepositoryMap.objects.get( + code_repo__repo_full_name__iexact=matches["repo_full_name"], ) + # assert False, context + # current_installation = context["all_installations"].get( + # account_name=matches["account_name"] + # ) - context["current_installation"] = current_installation + # context["current_installation"] = current_installation context["open_prs"] = app_models.MonitoredPullRequest.objects.filter( code_pull_request__repository=context["repo_mapping"].code_repo, - code_pull_request__pr_state="open", + code_pull_request__pr_state__in=["open", "opened"], + # open for Github, Opened for gitlab ) search_query = self.request.GET.get("q") if search_query: @@ -88,12 +88,9 @@ def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: | Q(documentation_pull_request__pr_title__icontains=search_query) ) context["q"] = search_query - # context["all_repo_map"] = context["all_repo_map"].filter( - # Q(code_repo__repo_full_name__icontains=search_query) - # | Q(documentation_repo__repo_full_name__icontains=search_query) - # ) - context["all_documentation_prs"] = app_models.GithubPullRequest.objects.filter( - repository=context["repo_mapping"].documentation_repo, pr_state="open" + context["all_documentation_prs"] = app_models.PullRequest.objects.filter( + repository=context["repo_mapping"].documentation_repo, + pr_state__in=["open", "opened"], ) return context @@ -230,25 +227,19 @@ class AppIndexPage(TemplateView): # @log_http_event(oid=1, okind_ekind=get_okind_ekind) def post(self, request, *args, **kwargs): payload = json.loads(request.body) - payload["all_installations"] = app_models.GithubAppInstallation.objects.filter( - id__in=app_models.GithubAppUser.objects.filter( - github_user=self.request.user.github_user - ).values_list("installation_id", flat=True), - state=app_models.GithubAppInstallation.InstallationState.INSTALLED, - ) - code_repo = app_models.GithubRepository.objects.get(id=payload["code_repo_id"]) - (instance, _) = app_models.GithubRepoMap.objects.update_or_create( - integration=code_repo.owner, - code_repo=code_repo, + (instance, _) = app_models.MonitoredRepositoryMap.objects.update_or_create( + code_repo_id=payload["code_repo_id"], documentation_repo_id=payload["documentation_repo_id"], defaults={ - "integration_type": app_models.GithubRepoMap.IntegrationType.FULL, + "integration_type": app_models.MonitoredRepositoryMap.IntegrationType.FULL, }, ) - django_rq.enqueue(app_jobs.sync_prs_for_repository, payload["code_repo_id"]) - django_rq.enqueue( - app_jobs.sync_prs_for_repository, payload["documentation_repo_id"] - ) + # TODO: Sync Open PRs for the repositories + # TODO: Setup the webhook for the mapped repositories + # django_rq.enqueue(app_jobs.sync_prs_for_repository, payload["code_repo_id"]) + # django_rq.enqueue( + # app_jobs.sync_prs_for_repository, payload["documentation_repo_id"] + # ) return JsonResponse({"success": True}) # @log_http_event(oid=1, okind_ekind=get_okind_ekind) @@ -258,86 +249,36 @@ def get(self, request, *args, **kwargs) -> HttpResponse: def get_context_data(self, **kwargs: Any) -> Dict[str, Any]: context = super().get_context_data(**kwargs) if self.request.user.is_authenticated: - # all_installations = app_models.AppInstallation.objects.filter( - # id__in=app_models.AppUser.objects.filter( - # user=self.request.user - # ).values_list("installation_id", flat=True), - # state=app_models.AppInstallation.InstallationState.INSTALLED, - # ) - # context["all_installations"] = all_installations - context["all_repo_map"] = app_models.GithubRepoMap.objects.filter( - code_repo__userrepoaccess__user=self.request.user, - documentation_repo__userrepoaccess__user=self.request.user, + context["all_repo_map"] = app_models.MonitoredRepositoryMap.objects.filter( + code_repo__userrepoaccess__social_account__user=self.request.user, + code_repo__userrepoaccess__access__gte=app_models.UserRepoAccess.AccessLevel.MINIMAL, + documentation_repo__userrepoaccess__social_account__user=self.request.user, + documentation_repo__userrepoaccess__access__gte=app_models.UserRepoAccess.AccessLevel.MINIMAL, ) context[ "available_repos_for_mapping" ] = app_models.Repository.objects.filter( - userrepoaccess__user=self.request.user, + userrepoaccess__social_account__user=self.request.user, + userrepoaccess__access__gte=app_models.UserRepoAccess.AccessLevel.MINIMAL, code_repos__isnull=True, documentation_repos__isnull=True, ) context["unmapped_repos_display"] = context["available_repos_for_mapping"] search_query = self.request.GET.get("q") if search_query: - # context["all_repo_map"] = context["all_repo_map"].filter( - # Q(code_repo__repo_full_name__icontains=search_query) - # | Q(documentation_repo__repo_full_name__icontains=search_query) - # ) + context["all_repo_map"] = context["all_repo_map"].filter( + Q(code_repo__repo_full_name__icontains=search_query) + | Q(documentation_repo__repo_full_name__icontains=search_query) + ) context["unmapped_repos_display"] = context[ "unmapped_repos_display" ].filter(repo_full_name__icontains=search_query) context["q"] = search_query - # context["all_repo_map"] = context["all_repo_map"][:10] + context["all_repo_map"] = context["all_repo_map"][:10] context["unmapped_repos_display"] = context["unmapped_repos_display"][:10] - else: - pass - # login_state_instance = app_models.GithubLoginState() - # if self.request.GET.get("next"): - # login_state_instance.redirect_url = self.request.GET.get("next") - # login_state_instance.save() - url = "https://github.com/login/oauth/authorize" - params = { - "client_id": settings.GITHUB_CREDS["client_id"], - "allow_signup": False, - # "state": login_state_instance.state.__str__(), - } - req = PreparedRequest() - req.prepare_url(url, params) - context["github_login_url"] = req.url - return context -class InitializeGithubLogin(View): - @log_http_event( - oid=1, - okind_ekind=( - analytics_events.OKind.USER, - analytics_events.EKind.GITHUB_HANDOVER, - ), - ) - def get(self, request, *args, **kwargs): - if self.request.user.is_authenticated: - if request.GET.get("next"): - return HttpResponseRedirect(request.GET.get("next")) - return HttpResponseRedirect("/") - else: - # login_state_instance = app_models.GithubLoginState() - # if self.request.GET.get("next"): - # login_state_instance.redirect_url = self.request.GET.get("next") - # login_state_instance.save() - url = "https://github.com/login/oauth/authorize" - params = { - "client_id": settings.GITHUB_CREDS["client_id"], - "allow_signup": False, - # "state": login_state_instance.state.__str__(), - } - req = PreparedRequest() - req.prepare_url(url, params) - assert False, req.url - return HttpResponseRedirect(req.url) - - class IndexView(TemplateView): template_name = "/" diff --git a/cdoc/urls.py b/cdoc/urls.py index 76f144c..f549265 100644 --- a/cdoc/urls.py +++ b/cdoc/urls.py @@ -44,20 +44,20 @@ # app_views.ListInstallationRepos.as_view(), # ), path( - "//pull//", + "//pull//", app_views.PRView.as_view(), ), path( - "//pulls/", + "//pulls/", app_views.AllPRView.as_view(), ), path("app/", include("app.urls")), path("admin/", admin.site.urls), - path( - "initiate-github-login/", - app_views.InitializeGithubLogin.as_view(), - name="initiate_github_login", - ), + # path( + # "initiate-github-login/", + # app_views.InitializeGithubLogin.as_view(), + # name="initiate_github_login", + # ), path( "", app_views.AppIndexPage.as_view(), From 21c0bfbb38728d1ab83d8f5f4f6bbb0716d3efff Mon Sep 17 00:00:00 2001 From: Shobhit Date: Mon, 30 May 2022 18:25:31 +0530 Subject: [PATCH 04/15] blank commit From b28f9a4ef37f3dc1dd0cd4a8523774041327b9d8 Mon Sep 17 00:00:00 2001 From: Shobhit Date: Mon, 30 May 2022 23:21:19 +0530 Subject: [PATCH 05/15] Gitlab functionality modified. --- app/gitlab_jobs.py | 6 +- app/gitlab_views.py | 57 +- app/templates/all_pr_for_org.html | 942 ++++++++++++++++-------------- app/urls.py | 1 + app/views.py | 112 ++-- cdoc/urls.py | 36 +- 6 files changed, 617 insertions(+), 537 deletions(-) diff --git a/app/gitlab_jobs.py b/app/gitlab_jobs.py index a9880a0..743b32e 100644 --- a/app/gitlab_jobs.py +++ b/app/gitlab_jobs.py @@ -22,7 +22,7 @@ def sync_repositories_for_installation(social_account: social_models.SocialAccou app=social_models.SocialApp.objects.get(provider="gitlab"), defaults={ "repo_name": project.name.replace(" ", ""), - "repo_full_name": project.name_with_namespace.replace(" ", ""), + "repo_full_name": project.path_with_namespace, }, ) project_perms = project._attrs["permissions"] @@ -94,7 +94,9 @@ def sync_prs_for_repository( code_pull_request=pr_instance, ) # Setup a webhook for this project\ - webhook_url = f"https://{settings.DEPLOYMENT_HOST_NAME}/app/gitlab/webhook-callback/" + webhook_url = ( + f"https://{settings.DEPLOYMENT_HOST_NAME}/app/gitlab/webhook-callback/" + ) existing_hooks = project_instance.hooks.list() for hook in existing_hooks: if hook.url == webhook_url: diff --git a/app/gitlab_views.py b/app/gitlab_views.py index 478f62c..dc4af82 100644 --- a/app/gitlab_views.py +++ b/app/gitlab_views.py @@ -1,6 +1,7 @@ +import datetime import gitlab from app import lib as app_lib -from django.http import HttpResponseRedirect +from django.http import HttpResponseRedirect, HttpResponse, JsonResponse from django.urls import reverse from django.views import View from app import models as app_models @@ -9,6 +10,7 @@ import json from django.views.decorators.csrf import csrf_exempt from django.utils.decorators import method_decorator +import time @method_decorator(csrf_exempt, name="dispatch") @@ -17,20 +19,59 @@ def post(self, request, *args, **kwargs): headers = request.headers event_header = headers["X-Gitlab-Event"] request_body = json.loads(request.body.decode("utf-8")) - gitlab_instance = gitlab.Gitlab(oauth_token=app_lib.get_active_token()) + # gitlab_instance = gitlab.Gitlab(oauth_token=app_lib.get_active_token()) if ( event_header == "Merge Request Hook" and request_body["object_kind"] == "merge_request" ): - project_name = request_body["request_body"]["path_with_namespace"] + object_attrs = request_body["object_attributes"] + project_name = object_attrs["target"]["path_with_namespace"] + repo = app_models.Repository.objects.get( + repo_full_name=project_name, app__provider="gitlab" + ) merge_request_id = request_body["object_attributes"]["iid"] app_models.PullRequest.objects.update_or_create( pr_number=merge_request_id, pr_id=request_body["object_attributes"]["id"], + repository=repo, defaults={ - - } + "pr_head_commit_sha": object_attrs["last_commit"]["id"], + "pr_head_modified_on": object_attrs["last_commit"]["timestamp"], + "pr_head_commit_message": object_attrs["last_commit"]["title"], + "pr_title": object_attrs["title"], + "pr_body": object_attrs["description"], + "pr_state": object_attrs["state"], + "pr_created_at": datetime.datetime.strptime( + object_attrs["created_at"], "%Y-%m-%d %H:%M:%S %Z" + ), + "pr_updated_at": datetime.datetime.strptime( + object_attrs["updated_at"], "%Y-%m-%d %H:%M:%S %Z" + ), + }, + ) + # print(request_body) + # print(headers) + return JsonResponse({}) + + +@method_decorator(csrf_exempt, name="dispatch") +class GitlabPipelineRequest(View): + def post(self, request, *args, **kwargs): + body = json.loads(request.body.decode("utf-8")) + gitlab_instance = gitlab.Gitlab(job_token=body["job_token"]) + # Get the job data to ensure no dirty access + job_data = gitlab_instance.http_get("/job/") + if job_data["pipeline"]["source"] == "merge_request_event": + repo_instance = app_models.Repository.objects.get( + repo_id=body["project_id"] + ) + # This assumes the webhook has been updated prior to the pipline being invoked. + # Potential race condition + pr_instance = app_models.PullRequest.objects.get( + pr_id=body["merge_request_id"], + pr_number=body["merge_request_iid"], + repository=repo_instance, + pr_head_commit_sha=body["source_sha"], ) - pass - assert False, (request.body, request.headers) - pass + return HttpResponse(0 if pr_instance.monitored_code.is_approved else 1) + return HttpResponse(0) diff --git a/app/templates/all_pr_for_org.html b/app/templates/all_pr_for_org.html index 45bf136..6d48b50 100644 --- a/app/templates/all_pr_for_org.html +++ b/app/templates/all_pr_for_org.html @@ -10,7 +10,8 @@ data-icon="octicon:repo-16">{{repo_mapping.code_repo.owner.account_name}}{% endcomment %}
- {{repo_mapping.code_repo.repo_full_name}} + {{repo_mapping.code_repo.repo_full_name}} {{repo_mapping.documentation_repo.repo_full_name}}
@@ -20,548 +21,585 @@ {% if open_prs %}
- +
{% endif %} {% if open_prs %}
- {% else %} -
- {% endif %} - - {% for monitored_pr in open_prs %} - - {% with code_pr_number=monitored_pr.code_pull_request.pr_number|stringformat:"s" documentation_pr_number=monitored_pr.documentation_pull_request.pr_number|stringformat:"s" %} - {% with code_pr_name="#"|add:code_pr_number|add:" "|add:monitored_pr.code_pull_request.pr_title %} -
- {% if monitored_pr.pull_request_status == "NOT_CONNECTED" %} -
-
- {{ code_pr_name }} - {{monitored_pr.get_pull_request_status_display}} + {% else %} +
+ {% endif %} + + {% for monitored_pr in open_prs %} + + {% with code_pr_number=monitored_pr.code_pull_request.pr_number|stringformat:"s" documentation_pr_number=monitored_pr.documentation_pull_request.pr_number|stringformat:"s" %} + {% with code_pr_name="#"|add:code_pr_number|add:" "|add:monitored_pr.code_pull_request.pr_title %} + {{code_pr_number}} +
+ {% if monitored_pr.pull_request_status == "NOT_CONNECTED" %} +
+
+ {{ code_pr_name }} + {{monitored_pr.get_pull_request_status_display}} +
-
-
- Connect -
- {% elif monitored_pr.pull_request_status == "MANUALLY_APPROVED" %} -
-
- {{ code_pr_name }} - {{monitored_pr.get_pull_request_status_display}} +
+ Connect
-
-
- Documentation Not Required -
- by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} {{monitored_pr.get_latest_approval.created_on|timesince}} ago + {% elif monitored_pr.pull_request_status == "MANUALLY_APPROVED" %} +
+
+ {{ code_pr_name }} + {{monitored_pr.get_pull_request_status_display}} +
-
- {% else %} - - {% with documentation_pr_name="#"|add:documentation_pr_number|add:" "|add:monitored_pr.documentation_pull_request.pr_title %} - {% if monitored_pr.pull_request_status == "APPROVED" %} -
-
- - {{code_pr_name}} - {{documentation_pr_name}} +
+ Documentation Not Required +
+ by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} + {{monitored_pr.get_latest_approval.created_on|timesince}} ago +
-
-
- Approved -
- by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} {{monitored_pr.get_latest_approval.created_on|timesince}} ago + {% else %} + + {% with documentation_pr_name="#"|add:documentation_pr_number|add:" "|add:monitored_pr.documentation_pull_request.pr_title %} + {% if monitored_pr.pull_request_status == "APPROVED" %} +
+
+ + {{code_pr_name}} + {{documentation_pr_name}} +
-
- {% else %} - {% comment %}Expected states `APPROVAL_PENDING`, `STALE_CODE` or `STALE_APPROVAL`{% endcomment %} -
-
- - {{code_pr_name}} - {{documentation_pr_name}} +
+ Approved +
+ by {{monitored_pr.get_latest_approval.approver.github_user.account_name}} + {{monitored_pr.get_latest_approval.created_on|timesince}} ago +
+ {% else %} + {% comment %}Expected states `APPROVAL_PENDING`, `STALE_CODE` or `STALE_APPROVAL`{% endcomment %} +
+
+ + {{code_pr_name}} + {{documentation_pr_name}} +
+
+
+ Approve +
+ {% endif %} + {% endwith %} + {% endif %}
-
- Approve -
- {% endif %} {% endwith %} - {% endif %} -
- {% endwith %} - {% endwith %} - {% empty %} + {% endwith %} + {% empty %}

No recent entries found

- {% endfor %} - -
- - -