Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added slack notifications to bishwas once a source is marked curated #625

Closed
wants to merge 1 commit into from
Closed

Added slack notifications to bishwas once a source is marked curated #625

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 7 additions & 12 deletions config/settings/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,7 @@
]
# https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
},
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
Expand Down Expand Up @@ -254,12 +252,7 @@
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"verbose": {
"format": "%(levelname)s %(asctime)s %(module)s "
"%(process)d %(thread)d %(message)s"
}
},
"formatters": {"verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}},
"handlers": {
"console": {
"level": "DEBUG",
Expand Down Expand Up @@ -334,9 +327,7 @@
"rest_framework.renderers.BrowsableAPIRenderer",
"rest_framework_datatables.renderers.DatatablesRenderer",
),
"DEFAULT_FILTER_BACKENDS": (
"rest_framework_datatables.filters.DatatablesFilterBackend",
),
"DEFAULT_FILTER_BACKENDS": ("rest_framework_datatables.filters.DatatablesFilterBackend",),
"DEFAULT_PAGINATION_CLASS": "rest_framework_datatables.pagination.DatatablesPageNumberPagination",
"PAGE_SIZE": 50,
"EXCEPTION_HANDLER": "sde_indexing_helper.utils.exceptions.custom_exception_handler",
Expand All @@ -345,3 +336,7 @@
GITHUB_ACCESS_TOKEN = env("GITHUB_ACCESS_TOKEN")
SINEQUA_CONFIGS_GITHUB_REPO = env("SINEQUA_CONFIGS_GITHUB_REPO")
GITHUB_BRANCH_FOR_WEBAPP = env("GITHUB_BRANCH_FOR_WEBAPP")

# Slack webhook URLS
# ------------------------------------------------------------------------------
BISHWAS_SLACK_WEBHOOK_URL = env("BISHWAS_SLACK_WEBHOOK_URL")
96 changes: 42 additions & 54 deletions sde_collections/models/collection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import urllib.parse

import requests
from django.conf import settings
from django.contrib.auth import get_user_model
from django.db import models
from slugify import slugify
Expand All @@ -26,54 +27,28 @@ class Collection(models.Model):
"""Model definition for Collection."""

name = models.CharField("Name", max_length=1024)
config_folder = models.CharField(
"Config Folder", max_length=2048, unique=True, editable=False
)
config_folder = models.CharField("Config Folder", max_length=2048, unique=True, editable=False)
url = models.URLField("URL", max_length=2048, blank=True)
division = models.IntegerField(choices=Divisions.choices)
turned_on = models.BooleanField("Turned On", default=True)
connector = models.IntegerField(
choices=ConnectorChoices.choices, default=ConnectorChoices.CRAWLER2
)
connector = models.IntegerField(choices=ConnectorChoices.choices, default=ConnectorChoices.CRAWLER2)

source = models.IntegerField(
choices=SourceChoices.choices, default=SourceChoices.BOTH
)
update_frequency = models.IntegerField(
choices=UpdateFrequencies.choices, default=UpdateFrequencies.WEEKLY
)
document_type = models.IntegerField(
choices=DocumentTypes.choices, null=True, blank=True
)
tree_root_deprecated = models.CharField(
"Tree Root", max_length=1024, default="", blank=True
)
source = models.IntegerField(choices=SourceChoices.choices, default=SourceChoices.BOTH)
update_frequency = models.IntegerField(choices=UpdateFrequencies.choices, default=UpdateFrequencies.WEEKLY)
document_type = models.IntegerField(choices=DocumentTypes.choices, null=True, blank=True)
tree_root_deprecated = models.CharField("Tree Root", max_length=1024, default="", blank=True)
delete = models.BooleanField(default=False)

# audit columns for production
audit_hierarchy = models.CharField(
"Audit Hierarchy", max_length=2048, default="", blank=True
)
audit_hierarchy = models.CharField("Audit Hierarchy", max_length=2048, default="", blank=True)
audit_url = models.CharField("Audit URL", max_length=2048, default="", blank=True)
audit_mapping = models.CharField(
"Audit Mapping", max_length=2048, default="", blank=True
)
audit_label = models.CharField(
"Audit Label", max_length=2048, default="", blank=True
)
audit_query = models.CharField(
"Audit Query", max_length=2048, default="", blank=True
)
audit_duplicate_results = models.CharField(
"Audit Duplicate Results", max_length=2048, default="", blank=True
)
audit_metrics = models.CharField(
"Audit Metrics", max_length=2048, default="", blank=True
)
audit_mapping = models.CharField("Audit Mapping", max_length=2048, default="", blank=True)
audit_label = models.CharField("Audit Label", max_length=2048, default="", blank=True)
audit_query = models.CharField("Audit Query", max_length=2048, default="", blank=True)
audit_duplicate_results = models.CharField("Audit Duplicate Results", max_length=2048, default="", blank=True)
audit_metrics = models.CharField("Audit Metrics", max_length=2048, default="", blank=True)

cleaning_assigned_to = models.CharField(
"Cleaning Assigned To", max_length=128, default="", blank=True
)
cleaning_assigned_to = models.CharField("Cleaning Assigned To", max_length=128, default="", blank=True)

github_issue_number = models.IntegerField("Issue Number in Github", default=0)
notes = models.TextField("Notes", blank=True, default="")
Expand All @@ -89,9 +64,7 @@ class Collection(models.Model):
choices=WorkflowStatusChoices.choices,
default=WorkflowStatusChoices.RESEARCH_IN_PROGRESS,
)
curated_by = models.ForeignKey(
User, on_delete=models.DO_NOTHING, null=True, blank=True
)
curated_by = models.ForeignKey(User, on_delete=models.DO_NOTHING, null=True, blank=True)
curation_started = models.DateTimeField("Curation Started", null=True, blank=True)

class Meta:
Expand Down Expand Up @@ -156,15 +129,11 @@ def workflow_status_button_color(self) -> str:

def _process_exclude_list(self):
"""Process the exclude list."""
return [
pattern._process_match_pattern() for pattern in self.excludepattern.all()
]
return [pattern._process_match_pattern() for pattern in self.excludepattern.all()]

def _process_include_list(self):
"""Process the include list."""
return [
pattern._process_match_pattern() for pattern in self.includepattern.all()
]
return [pattern._process_match_pattern() for pattern in self.includepattern.all()]

def _process_title_list(self):
"""Process the title list"""
Expand Down Expand Up @@ -194,19 +163,15 @@ def create_config_xml(self):
and xml file on sde-backend/sources/SDE/<config_folder>/default.xml
"""

original_config_string = open(
"config_generation/xmls/indexing_template.xml"
).read()
original_config_string = open("config_generation/xmls/indexing_template.xml").read()
editor = XmlEditor(original_config_string)

# add the URL
editor.update_or_add_element_value("Url", self.url)

editor.update_or_add_element_value("TreeRoot", self.tree_root)
if self.document_type:
editor.add_document_type_mapping(
document_type=self.get_document_type_display(), criteria=None
)
editor.add_document_type_mapping(document_type=self.get_document_type_display(), criteria=None)

updated_config_xml_string = editor.update_config_xml()

Expand Down Expand Up @@ -403,12 +368,35 @@ def apply_all_patterns(self) -> None:

def save(self, *args, **kwargs):
# Call the function to generate the value for the generated_field based on the original_field
is_new = self._state.adding

if not self.config_folder:
self.config_folder = self._compute_config_folder_name()

if not is_new:
current_instance = Collection.objects.get(pk=self.pk)
previous_status = current_instance.workflow_status
else:
previous_status = None

# Call the parent class's save method
super().save(*args, **kwargs)

if previous_status != self.workflow_status and self.workflow_status == WorkflowStatusChoices.CURATED:
self.notify_slack()

def notify_slack(self):
# This function is called every time a new collection is created and a team member is notified
webhook_url = settings.BISHWAS_SLACK_WEBHOOK_URL
message = f"A collection named '{self.name}' has been curated and is now ready to go on Test."
data = {"text": message}

try:
response = requests.post(webhook_url, json=data)
response.raise_for_status()
except requests.exceptions.RequestException as e:
print(f"Failed to send notification to Slack: {e}")


class RequiredUrls(models.Model):
"""
Expand Down