Reduzir consultas e conexões ao banco de dados

article/controller.py

@@ -345,7 +345,7 @@ def bulk_export_articles_to_articlemeta(
             )
             return False
 
-        for article in queryset.iterator():
+        for article in queryset.select_related("journal", "journal__official", "pp_xml").iterator():
             try:
                 if force_update:
                     article.check_availability(user)

bigbang/tasks.py

@@ -8,6 +8,7 @@
 from collection.models import Collection
 from config import celery_app
 from core.models import Gender, Language, License
+from core.utils.utils import _get_user
 from editorialboard.models import RoleModel
 from institution.models import Institution, InstitutionType
 from journal.models import (
@@ -26,21 +27,14 @@
 User = get_user_model()
 
 
-def _get_user(user_id, username):
-    if user_id:
-        return User.objects.get(pk=user_id)
-    if username:
-        return User.objects.get(username=username)
-
-
 @celery_app.task(bind=True)
 def task_start(
     self,
     user_id=None,
     username=None,
 ):
     try:
-        user = _get_user(user_id, username)
+        user = _get_user(request=None, user_id=user_id, username=username)
         Language.load(user)
         Collection.load(user)
         Vocabulary.load(user)
@@ -73,7 +67,7 @@ def task_start(
 @celery_app.task(bind=True)
 def task_create_tasks(self, user_id=None, username=None, tasks_data=None):
     if not tasks_data:
-        user = _get_user(user_id, username)
+        user = _get_user(request=None, user_id=user_id, username=username)
         return schedule_tasks(user.username)
     for task_data in tasks_data:
         # {

collection/tasks.py

@@ -11,6 +11,8 @@
 def task_load_collections(self, user_id=None, username=None):
     if user_id:
         user = User.objects.get(pk=user_id)
-    if username:
+    elif username:
         user = User.objects.get(username=username)
+    else:
+        raise ValueError("user_id or username is required")
     Collection.load(user)

config/settings/base.py

@@ -374,11 +374,21 @@
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_serializer
 CELERY_RESULT_SERIALIZER = "json"
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-time-limit
-# TODO: set to whatever value is adequate in your circumstances
-CELERY_TASK_TIME_LIMIT = 5 * 60
+# Hard time limit: o worker é terminado (SIGKILL) após este tempo.
+# Deve ser MAIOR que o soft time limit.
+CELERY_TASK_TIME_LIMIT = env.int('CELERY_TASK_TIME_LIMIT', default=36000)
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit
-# TODO: set to whatever value is adequate in your circumstances
-CELERY_TASK_SOFT_TIME_LIMIT = 36000
+# Soft time limit: levanta SoftTimeLimitExceeded, permitindo cleanup.
+# Deve ser MENOR que o hard time limit.
+CELERY_TASK_SOFT_TIME_LIMIT = env.int('CELERY_TASK_SOFT_TIME_LIMIT', default=3600)
+
+# Recicla worker após N tarefas para liberar memória e conexões de banco de dados.
+# Cada worker filho é substituído após processar este número de tarefas.
+CELERY_WORKER_MAX_TASKS_PER_CHILD = env.int('CELERY_WORKER_MAX_TASKS_PER_CHILD', default=100)
+
+# Limita a concorrência do Celery worker.
+# Controla quantas tarefas cada worker processa simultaneamente.
+CELERY_WORKER_CONCURRENCY = env.int('CELERY_WORKER_CONCURRENCY', default=4)
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#beat-scheduler
 CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html
@@ -404,8 +414,11 @@
 RUN_ASYNC = env.bool('RUN_ASYNC', default=0)
 # Celery Results
 # ------------------------------------------------------------------------------
-# https: // django-celery-results.readthedocs.io/en/latest/getting_started.html
-CELERY_RESULT_BACKEND = "django-db"
+# https://django-celery-results.readthedocs.io/en/latest/getting_started.html
+# NOTA: Não usar "django-db" como result backend em produção.
+# O result backend já está configurado como Redis (CELERY_BROKER_URL) acima.
+# Manter "django-db" aqui causaria escritas extras no PostgreSQL a cada tarefa concluída.
+# CELERY_RESULT_BACKEND = "django-db"  # REMOVIDO: já usa Redis acima
 CELERY_CACHE_BACKEND = "django-cache"
 CELERY_RESULT_EXTENDED = True
 

config/settings/production.py

@@ -14,19 +14,13 @@
 # ------------------------------------------------------------------------------
 DATABASES["default"] = env.db("DATABASE_URL")  # noqa F405
 DATABASES["default"]["ATOMIC_REQUESTS"] = True  # noqa F405
-DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=0) or env.int("DJANGO_CONN_MAX_AGE", default=60)  # noqa F405
+# Reutilizar conexões por 60s por padrão para reduzir overhead de reconexão.
+# Em produção com gunicorn+gevent ou Celery, isto evita abrir/fechar conexão a cada request.
+DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60)  # noqa F405
 DATABASES["default"]["CONN_HEALTH_CHECKS"] = env.bool('DJANGO_CONN_HEALTH_CHECKS', True)
 DATABASES["default"]["ENGINE"] = 'django_prometheus.db.backends.postgresql'
-# Melhoria: Usando variáveis de ambiente para OPTIONS e POOL_OPTIONS com defaults
 DATABASES["default"]["OPTIONS"] = {
     "connect_timeout": env.int("DB_CONNECT_TIMEOUT", default=10),
-    # Adicione outras opções de conexão aqui se necessário
-}
-DATABASES["default"]["POOL_OPTIONS"] = {
-    'POOL_SIZE': env.int("DB_POOL_SIZE", default=10),
-    'MAX_OVERFLOW': env.int("DB_MAX_OVERFLOW", default=20),
-    'RECYCLE': env.int("DB_RECYCLE", default=300),
-    # Adicione outras opções do pool aqui se necessário
 }
 # CACHES
 # ------------------------------------------------------------------------------
@@ -82,6 +76,11 @@
 # Ex: export DJANGO_SESSION_SAVE_EVERY_REQUEST=False
 SESSION_SAVE_EVERY_REQUEST = env.bool('DJANGO_SESSION_SAVE_EVERY_REQUEST', True)
 
+# Sessões armazenadas no Redis em vez do PostgreSQL.
+# Reduz escritas no banco de dados a cada request quando SESSION_SAVE_EVERY_REQUEST=True.
+SESSION_ENGINE = "django.contrib.sessions.backends.cache"
+SESSION_CACHE_ALIAS = "default"
+
 # STATIC
 # ------------------------
 STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"

core/middleware.py

@@ -15,9 +15,10 @@ def __call__(self, request):
 
         if request.user.is_authenticated:
             set_current_user(request.user)
-            set_current_collections(request.user.collection.all())
+            collections = request.user.collection.all()
+            set_current_collections(collections)
 
-            request.user_collection = request.user.collection.all()
+            request.user_collection = collections
         else:
             set_current_user(None)
             set_current_collections(None)

core/utils/utils.py

@@ -91,12 +91,14 @@ def fetch_data(url, headers=None, json=False, timeout=FETCH_DATA_TIMEOUT, verify
 
 def _get_user(request, username=None, user_id=None):
     try:
-        return User.objects.get(pk=request.user_id)
+        if request.user.is_authenticated:
+            return request.user
     except AttributeError:
-        if user_id:
-            return User.objects.get(pk=user_id)
-        if username:
-            return User.objects.get(username=username)
+        pass
+    if user_id:
+        return User.objects.get(pk=user_id)
+    if username:
+        return User.objects.get(username=username)
 
 
 def formated_date_api_params(query_params):

core/wagtail_hooks.py

@@ -175,3 +175,6 @@ class LicenseViewSet(SnippetViewSet):
     search_fields = ("license_type", "version")
     list_export = ("license_type", "version")
     inspect_view_enabled = True
+
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("creator")

editorialboard/views.py

@@ -71,15 +71,16 @@ def import_file_ebm(request):
     file_path = file_upload.attachment.file.path
 
     try:
+        user = request.user
         with open(file_path, "r") as csvfile:
             data = csv.DictReader(csvfile, delimiter=";")
             for line, row in enumerate(data):
                 given_names = row.get("Nome do membro")
                 last_name = row.get("Sobrenome")
                 journal = Journal.objects.get(title__icontains=row.get("Periódico"))
-                gender = Gender.create_or_update(user=request.user, code=row.get("Gender"), gender="F")
+                gender = Gender.create_or_update(user=user, code=row.get("Gender"), gender="F")
                 location = Location.create_or_update(
-                    user=request.user,
+                    user=user,
                     city_name=row.get("institution_city_name"),
                     state_text=row.get("institution_state_text"),
                     state_acronym=row.get("institution_state_acronym"),
@@ -89,7 +90,7 @@ def import_file_ebm(request):
                     country_name=row.get("institution_country_name"),
                 )
                 researcher = Researcher.create_or_update(
-                    user=request.user,
+                    user=user,
                     given_names=given_names,
                     last_name=last_name,
                     suffix=row.get("Suffix"),
@@ -104,7 +105,7 @@ def import_file_ebm(request):
                     aff_name=row.get("Instituição"),
                 )
                 EditorialBoardMember.create_or_update(
-                    user=request.user,
+                    user=user,
                     researcher=researcher,
                     journal=journal,
                     declared_role=row["Cargo / instância do membro"],

institution/models.py

@@ -201,10 +201,16 @@ def create_or_update(
                 level_3=level_3,
                 location=location,
             )
-            institution.updated_by = user
-            institution.institution_type = institution_type or institution.institution_type
-            institution.url = url or institution.url
-            institution.save()
+            changed = False
+            if institution_type and institution_type != institution.institution_type:
+                institution.institution_type = institution_type
+                changed = True
+            if url and url != institution.url:
+                institution.url = url
+                changed = True
+            if changed:
+                institution.updated_by = user
+                institution.save()
             return institution
         except cls.DoesNotExist:
             return cls._create(
@@ -968,10 +974,16 @@ def create_or_update(
 
         try:
             obj = cls._get(name=name, acronym=acronym)
-            obj.updated_by = user
-            obj.is_official = is_official or obj.is_official
-            obj.official = official or obj.official
-            obj.save()
+            changed = False
+            if is_official is not None and is_official != obj.is_official:
+                obj.is_official = is_official
+                changed = True
+            if official and official != obj.official:
+                obj.official = official
+                changed = True
+            if changed:
+                obj.updated_by = user
+                obj.save()
             return obj
         except cls.DoesNotExist:
             return cls._create(

issue/wagtail_hooks.py

@@ -89,6 +89,8 @@ def get_queryset(self, request):
         # Base queryset com otimizações
         qs = Issue.objects.select_related(
             "journal",
+            "creator",
+            "updated_by",
         )
 
         user = request.user

journal/models.py

@@ -287,26 +287,20 @@ def create_or_update(
     def add_old_title(self, user, title):
         if not title:
             return
-        old_title = None
-        for item in OfficialJournal.objects.filter(title=title).iterator():
-            old_title = item
-            break
+        old_title = OfficialJournal.objects.filter(title=title).first()
         if not old_title:
             old_title = OfficialJournal.objects.create(title=title, creator=user)
         self.old_title.add(old_title)
-        self.save()
+        # M2M .add() grava diretamente na tabela intermediária; não precisa de self.save()
 
     def add_new_title(self, user, title):
         if not title:
             return
-        new_title = None
-        for item in OfficialJournal.objects.filter(title=title).iterator():
-            new_title = item
-            break
+        new_title = OfficialJournal.objects.filter(title=title).first()
         if not new_title:
             new_title = OfficialJournal.objects.create(title=title, creator=user)
         self.new_title = new_title
-        self.save()
+        # Não faz save() aqui; chamador é responsável por salvar
 
     @property
     def parallel_titles(self):
@@ -1026,10 +1020,10 @@ def get_issn_list(cls, collection_acron_list=None, journal_acron_list=None):
 
     @property
     def collection_acrons(self):
-        acrons = []
-        for item in self.scielojournal_set.all():
-            acrons.append(item.collection.acron3)
-        return "|".join(acrons)
+        return "|".join(
+            self.scielojournal_set.select_related("collection")
+            .values_list("collection__acron3", flat=True)
+        )
 
     @classmethod
     def get_ids(cls, collection_acron_list=None, journal_acron_list=None):
@@ -1042,7 +1036,7 @@ def select_collections(self, collection_acron_list=None, is_active=None):
             params["collection__is_active"] = is_active
         if collection_acron_list:
             params["collection__acron3__in"] = collection_acron_list
-        for item in self.scielojournal_set.filter(**params):
+        for item in self.scielojournal_set.select_related("collection").filter(**params):
             yield item.collection
 
     def get_legacy_keys(self, collection_acron_list=None, is_active=None):
@@ -1052,7 +1046,7 @@ def get_legacy_keys(self, collection_acron_list=None, is_active=None):
         if is_active:
             params["collection__is_active"] = bool(is_active)
         data = {}
-        for item in self.scielojournal_set.filter(**params):
+        for item in self.scielojournal_set.select_related("collection").filter(**params):
             data[item.collection.acron3] = item.legacy_keys
         if not data:
             UnexpectedEvent.create(
@@ -2930,39 +2924,25 @@ def am_to_core(
             "suspended-by-committee": "by-committee",
             "suspended-by-editor": "by-editor",
         }
-        try:
-            obj = cls.objects.get(
-                scielo_journal=scielo_journal,
-                year=initial_year,
-                month=initial_month,
-                day=initial_day,
-            )
-        except cls.DoesNotExist:
-            obj = cls()
-            obj.scielo_journal = scielo_journal
-            obj.year = initial_year
-            obj.month = initial_month
-            obj.day = initial_day
-        obj.event_type = "ADMITTED"
-        obj.save()
+        obj, _ = cls.objects.update_or_create(
+            scielo_journal=scielo_journal,
+            year=initial_year,
+            month=initial_month,
+            day=initial_day,
+            defaults={"event_type": "ADMITTED"},
+        )
 
         if final_year and event_type:
-            try:
-                obj = cls.objects.get(
-                    scielo_journal=scielo_journal,
-                    year=final_year,
-                    month=final_month,
-                    day=final_day,
-                )
-            except cls.DoesNotExist:
-                obj = cls()
-                obj.scielo_journal = scielo_journal
-                obj.year = final_year
-                obj.month = final_month
-                obj.day = final_day
-            obj.event_type = "INTERRUPTED"
-            obj.interruption_reason = reasons.get(interruption_reason)
-            obj.save()
+            obj, _ = cls.objects.update_or_create(
+                scielo_journal=scielo_journal,
+                year=final_year,
+                month=final_month,
+                day=final_day,
+                defaults={
+                    "event_type": "INTERRUPTED",
+                    "interruption_reason": reasons.get(interruption_reason),
+                },
+            )
 
 
 class TitleInDatabase(Orderable, CommonControlField):