diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index fa48aa8..0000000 --- a/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -# Use an official Python runtime based on Debian 12 "bookworm" as a parent image. -FROM python:3.12-slim-bookworm - -# Add user that will be used in the container. -RUN useradd wagtail - -# Port used by this container to serve HTTP. -EXPOSE 8000 - -# Set environment variables. -# 1. Force Python stdout and stderr streams to be unbuffered. -# 2. Set PORT variable that is used by Gunicorn. This should match "EXPOSE" -# command. -ENV PYTHONUNBUFFERED=1 \ - PORT=8000 - -# Install system packages required by Wagtail and Django. -RUN apt-get update --yes --quiet && apt-get install --yes --quiet --no-install-recommends \ - build-essential \ - libpq-dev \ - libmariadb-dev \ - libjpeg62-turbo-dev \ - zlib1g-dev \ - libwebp-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install the application server. -RUN pip install "gunicorn==20.0.4" - -# Install the project requirements. -COPY requirements.txt / -RUN pip install -r /requirements.txt - -# Use /app folder as a directory where the source code is stored. -WORKDIR /app - -# Set this directory to be owned by the "wagtail" user. This Wagtail project -# uses SQLite, the folder needs to be owned by the user that -# will be writing to the database file. -RUN chown wagtail:wagtail /app - -# Copy the source code of the project into the container. -COPY --chown=wagtail:wagtail . . - -# Use user "wagtail" to run the build commands below and the server itself. -USER wagtail - -# Collect static files. -RUN python manage.py collectstatic --noinput --clear - -# Runtime command that executes when "docker run" is called, it does the -# following: -# 1. Migrate the database. -# 2. Start the application server. -# WARNING: -# Migrating database at the same time as starting the server IS NOT THE BEST -# PRACTICE. The database should be migrated manually or using the release -# phase facilities of your hosting platform. This is used only so the -# Wagtail instance can be started with a simple "docker run" command. -CMD set -xe; python manage.py migrate --noinput; gunicorn markapi.wsgi:application diff --git a/Makefile b/Makefile index 57f209d..79cd5da 100755 --- a/Makefile +++ b/Makefile @@ -32,71 +32,71 @@ build_date: ## Show build date @echo "Build date: " $(SCMS_BUILD_DATE) ############################################ -## atalhos docker-compose desenvolvimento ## +## atalhos docker compose desenvolvimento ## ############################################ build: ## Build app using $(compose) - @docker-compose -f $(compose) build + @docker compose -f $(compose) build build_no_cache: ## Build app using $(compose) - @docker-compose -f $(compose) build --no-cache + @docker compose -f $(compose) build --no-cache up: ## Start app using $(compose) - @docker-compose -f $(compose) up -d + @docker compose -f $(compose) up -d logs: ## See all app logs using $(compose) - @docker-compose -f $(compose) logs -f + @docker compose -f $(compose) logs -f stop: ## Stop all app using $(compose) - @docker-compose -f $(compose) stop + @docker compose -f $(compose) stop restart: - @docker-compose -f $(compose) restart + @docker compose -f $(compose) restart ps: ## See all containers using $(compose) - @docker-compose -f $(compose) ps + @docker compose -f $(compose) ps rm: ## Remove all containers using $(compose) - @docker-compose -f $(compose) rm -f + @docker compose -f $(compose) rm -f django_shell: ## Open python terminal from django $(compose) - @docker-compose -f $(compose) run --rm django python manage.py shell + @docker compose -f $(compose) run --rm django python manage.py shell wagtail_sync: ## Wagtail sync Page fields (repeat every time you add a new language and to update the wagtailcore_page translations) $(compose) - @docker-compose -f $(compose) run --rm django python manage.py sync_page_translation_fields + @docker compose -f $(compose) run --rm django python manage.py sync_page_translation_fields wagtail_update_translation_field: ## Wagtail update translation fields, user this command first $(compose) - @docker-compose -f $(compose) run --rm django python manage.py update_translation_fields + @docker compose -f $(compose) run --rm django python manage.py update_translation_fields django_createsuperuser: ## Create a super user from django $(compose) - @docker-compose -f $(compose) run --rm django python manage.py createsuperuser + @docker compose -f $(compose) run --rm django python manage.py createsuperuser django_bash: ## Open a bash terminar from django container using $(compose) - @docker-compose -f $(compose) run --rm django bash + @docker compose -f $(compose) run --rm django bash django_test: ## Run tests from django container using $(compose) - @docker-compose -f $(compose) run --rm django python manage.py test + @docker compose -f $(compose) run --rm django python manage.py test django_fast: ## Run tests fast from django container using $(compose) - @docker-compose -f $(compose) run --rm django python manage.py test --failfast + @docker compose -f $(compose) run --rm django python manage.py test --failfast django_makemigrations: ## Run makemigrations from django container using $(compose) - @docker-compose -f $(compose) run --rm django python manage.py makemigrations + @docker compose -f $(compose) run --rm django python manage.py makemigrations django_migrate: ## Run migrate from django container using $(compose) - @docker-compose -f $(compose) run --rm django python manage.py migrate + @docker compose -f $(compose) run --rm django python manage.py migrate django_makemessages: ## Run ./manage.py makemessages $(compose) - @docker-compose -f $(compose) run --rm django python manage.py makemessages --all + @docker compose -f $(compose) run --rm django python manage.py makemessages --all django_compilemessages: ## Run ./manage.py compilemessages $(compose) - @docker-compose -f $(compose) run --rm django python manage.py compilemessages + @docker compose -f $(compose) run --rm django python manage.py compilemessages django_dump_auth: ## Run manage.py dumpdata auth --indent=2 $(compose) - @docker-compose -f $(compose) run --rm django python manage.py dumpdata auth --indent=2 --output=fixtures/auth.json + @docker compose -f $(compose) run --rm django python manage.py dumpdata auth --indent=2 --output=fixtures/auth.json django_load_auth: ## Run manage.py dumpdata auth --indent=2 $(compose) - @docker-compose -f $(compose) run --rm django python manage.py loaddata --database=default fixtures/auth.json + @docker compose -f $(compose) run --rm django python manage.py loaddata --database=default fixtures/auth.json dump_data: ## Dump database into .sql $(compose) docker exec -t scielo_markup_local_postgres pg_dumpall -c -U debug > dump_`date +%d-%m-%Y"_"%H_%M_%S`.sql @@ -121,7 +121,7 @@ clean_project_images: ## Remove all images with "markup" on name @docker rmi -f $$(docker images --filter=reference='*scielo_markup*' -q) volume_down: ## Remove all volume - @docker-compose -f $(compose) down -v + @docker compose -f $(compose) down -v clean_migrations: ## Remove all migrations @echo "Cleaning migrations..." diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6c6aa7c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/config/settings/base.py b/config/settings/base.py index f78af49..365a1d1 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -120,13 +120,8 @@ # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(BASE_DIR, "db.sqlite3"), - } -} +DATABASES = {"default": env.db("DATABASE_URL")} +DATABASES["default"]["ATOMIC_REQUESTS"] = True # Password validation diff --git a/kubernetes/hml/configmap-markapi-hml.yml b/kubernetes/hml/configmap-markapi-hml.yml new file mode 100644 index 0000000..6f9b451 --- /dev/null +++ b/kubernetes/hml/configmap-markapi-hml.yml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: secrets-markapi-hml + namespace: markapi-hml + labels: + k8slens-edit-resource-version: v1 +spec: {} +data: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_FLOWER_PASSWORD: + CELERY_FLOWER_USER: + DATABASE_URL: + DJANGO_ADMIN_URL: django-admin/ + DJANGO_ALLOWED_HOSTS: '*' + DJANGO_DEBUG: 'True' + DJANGO_DEFAULT_FROM_EMAIL: + DJANGO_EMAIL_SUBJECT_PREFIX: '[MARKAPI-HML]' + DJANGO_SECRET_KEY: + DJANGO_SETTINGS_MODULE: config.settings.production + EMAIL_HOST: smtp.gmail.com + EMAIL_HOST_PASSWORD: + EMAIL_HOST_USER: + EMAIL_PORT: '587' + EMAIL_USE_TLS: 'True' + IPYTHONDIR: /app/.ipython + MAINTENANCE_MODE: 'False' + REDIS_URL: redis://redis:6379/0 + SENTRY_DSN: + USE_DOCKER: 'yes' + WAGTAILADMIN_BASE_URL: https://hml-markapi.scielo.org + WAGTAIL_ADMIN_URL: admin/ + HF_TOKEN: diff --git a/kubernetes/hml/deployment-markapi-hml-celerybeat.yml b/kubernetes/hml/deployment-markapi-hml-celerybeat.yml new file mode 100644 index 0000000..1f5a811 --- /dev/null +++ b/kubernetes/hml/deployment-markapi-hml-celerybeat.yml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: markapi-hml-celerybeat + namespace: markapi-hml + generation: 1 + labels: + app: markapi-hml-celerybeat + k8slens-edit-resource-version: v1 + annotations: + deployment.kubernetes.io/revision: '1' + selfLink: /apis/apps/v1/namespaces/markapi-hml/deployments/markapi-hml-celerybeat +spec: + replicas: 1 + selector: + matchLabels: + app: markapi-hml-celerybeat + template: + metadata: + labels: + app: markapi-hml-celerybeat + spec: + volumes: + - name: markapi-hml-django-media + persistentVolumeClaim: + claimName: markapi-hml-django-media-pvclaim + - name: markapi-hml-django-tmp + persistentVolumeClaim: + claimName: markapi-hml-django-tmp-pvclaim + containers: + - name: webapp + image: scieloorg/markapi:v0.1.0 + command: + - /start-celerybeat + envFrom: + - configMapRef: + name: configmap-markapi-hml + resources: {} + volumeMounts: + - name: markapi-hml-django-media + mountPath: /app/core/media + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: Always + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + tolerations: + - key: app + operator: Equal + value: producao + effect: NoSchedule + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/kubernetes/hml/deployment-markapi-hml-celeryworker.yml b/kubernetes/hml/deployment-markapi-hml-celeryworker.yml new file mode 100644 index 0000000..4290be9 --- /dev/null +++ b/kubernetes/hml/deployment-markapi-hml-celeryworker.yml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: markapi-hml-celeryworker + namespace: markapi-hml + generation: 1 + labels: + app: markapi-hml-celeryworker + k8slens-edit-resource-version: v1 + annotations: + deployment.kubernetes.io/revision: '1' + selfLink: /apis/apps/v1/namespaces/markapi-hml/deployments/markapi-hml-celeryworker +spec: + replicas: 1 + selector: + matchLabels: + app: markapi-hml-celeryworker + template: + metadata: + creationTimestamp: null + labels: + app: markapi-hml-celeryworker + spec: + volumes: + - name: markapi-hml-django-media + persistentVolumeClaim: + claimName: markapi-hml-django-media-pvclaim + - name: markapi-hml-django-tmp + persistentVolumeClaim: + claimName: markapi-hml-django-tmp-pvclaim + containers: + - name: webapp + image: scieloorg/markapi:v0.1.0 + command: + - /start-celeryworker + envFrom: + - configMapRef: + name: configmap-markapi-hml + resources: {} + volumeMounts: + - name: markapi-hml-django-media + mountPath: /app/core/media + - name: markapi-hml-django-tmp + mountPath: /tmp + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: Always + restartPolicy: Always + terminationGracePeriodSeconds: 120 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + tolerations: + - key: app + operator: Equal + value: producao + effect: NoSchedule + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/kubernetes/hml/deployment-markapi-hml-django.yml b/kubernetes/hml/deployment-markapi-hml-django.yml new file mode 100644 index 0000000..edd1697 --- /dev/null +++ b/kubernetes/hml/deployment-markapi-hml-django.yml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: markapi-hml-django + namespace: markapi-hml + generation: 1 + labels: + app: markapi-hml-django + k8slens-edit-resource-version: 1 + annotations: + deployment.kubernetes.io/revision: '1' + selfLink: /apis/apps/v1/namespaces/markapi-hml/deployments/markapi-hml-django +spec: + replicas: 1 + selector: + matchLabels: + app: markapi-hml-django + template: + metadata: + creationTimestamp: null + labels: + app: markapi-hml-django + spec: + volumes: + - name: markapi-hml-django-media + persistentVolumeClaim: + claimName: markapi-hml-django-media-pvclaim + - name: markapi-hml-django-tmp + persistentVolumeClaim: + claimName: markapi-hml-django-tmp-pvclaim + containers: + - name: webapp + image: scieloorg/markapi:v0.1.0 + command: + - /start + ports: + - name: webapp + containerPort: 5000 + protocol: TCP + envFrom: + - configMapRef: + name: configmap-markapi-hml + resources: {} + volumeMounts: + - name: markapi-hml-django-media + mountPath: /app/core/media + - name: markapi-hml-django-tmp + mountPath: /tmp + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: Always + securityContext: + privileged: true + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + tolerations: + - key: app + operator: Equal + value: producao + effect: NoSchedule + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/kubernetes/hml/statefulset-markapi-hml-postgresql.yml b/kubernetes/hml/statefulset-markapi-hml-postgresql.yml new file mode 100644 index 0000000..92fb50d --- /dev/null +++ b/kubernetes/hml/statefulset-markapi-hml-postgresql.yml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: markapi-hml-postgres + namespace: markapi-hml + generation: 1 + annotations: + selfLink: /apis/apps/v1/namespaces/markapi-hml/statefulsets/markapi-hml-postgres +spec: + replicas: 1 + selector: + matchLabels: + app: markapi-hml-postgres + template: + metadata: + creationTimestamp: null + labels: + app: markapi-hml-postgres + spec: + volumes: + - name: markapi-hml-postgres-data + persistentVolumeClaim: + claimName: markapi-hml-postgres-pvc + containers: + - name: postgres + image: postgres:15.3 + ports: + - containerPort: 5432 + protocol: TCP + envFrom: + - configMapRef: + name: configmap-markapi-hml + resources: {} + volumeMounts: + - name: markapi-hml-postgres-data + mountPath: /var/lib/postgresql/data + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + serviceAccountName: postgres + serviceAccount: postgres + securityContext: + runAsUser: 999 + fsGroup: 999 + schedulerName: default-scheduler + tolerations: + - key: app + operator: Equal + value: producao + effect: NoSchedule + serviceName: markapi-hml-postgres + podManagementPolicy: OrderedReady + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + revisionHistoryLimit: 10 diff --git a/kubernetes/hml/statefulset-markapi-hml-redis.yml b/kubernetes/hml/statefulset-markapi-hml-redis.yml new file mode 100644 index 0000000..da24716 --- /dev/null +++ b/kubernetes/hml/statefulset-markapi-hml-redis.yml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: markapi-hml + generation: 1 + labels: + k8slens-edit-resource-version: v1 + annotations: + selfLink: /apis/apps/v1/namespaces/markapi-hml/statefulsets/redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + env: markapi-hml + template: + metadata: + creationTimestamp: null + labels: + app: redis + env: markapi-hml + spec: + volumes: + - name: markapi-hml-redis + persistentVolumeClaim: + claimName: markapi-hml-redis-pvclaim + containers: + - name: redis + image: redis:7.2 + resources: {} + volumeMounts: + - name: markapi-hml-redis + mountPath: /data + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + serviceAccountName: redis + serviceAccount: redis + securityContext: + runAsUser: 1001 + fsGroup: 1001 + schedulerName: default-scheduler + tolerations: + - key: app + operator: Equal + value: producao + effect: NoSchedule + serviceName: redis + podManagementPolicy: OrderedReady + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + revisionHistoryLimit: 10 diff --git a/kubernetes/hml/svc-markapi-hml-django.yml b/kubernetes/hml/svc-markapi-hml-django.yml new file mode 100644 index 0000000..9c1e088 --- /dev/null +++ b/kubernetes/hml/svc-markapi-hml-django.yml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: markapi-hml-django-svc + namespace: markapi-hml + labels: + app: markapi-hml-django + selfLink: /api/v1/namespaces/markapi-hml/services/markapi-hml-django-svc +spec: + ports: + - name: webapp + protocol: TCP + port: 5000 + targetPort: 5000 + nodePort: + selector: + app: markapi-hml-django + clusterIP: + clusterIPs: + type: NodePort + sessionAffinity: None + externalTrafficPolicy: Cluster diff --git a/kubernetes/hml/svc-markapi-hml-postgresql.yml b/kubernetes/hml/svc-markapi-hml-postgresql.yml new file mode 100644 index 0000000..b16f736 --- /dev/null +++ b/kubernetes/hml/svc-markapi-hml-postgresql.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: markapi-hml-postgres + namespace: markapi-hml + labels: + app: markapi-hml-postgres + annotations: + selfLink: /api/v1/namespaces/markapi-hml/services/markapi-hml-postgres +spec: + ports: + - name: postgres + protocol: TCP + port: 5432 + targetPort: 5432 + selector: + app: markapi-hml-postgres + clusterIP: + clusterIPs: + type: ClusterIP + sessionAffinity: None diff --git a/kubernetes/hml/svc-markapi-hml-redis.yml b/kubernetes/hml/svc-markapi-hml-redis.yml new file mode 100644 index 0000000..a4cdc97 --- /dev/null +++ b/kubernetes/hml/svc-markapi-hml-redis.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: markapi-hml + labels: + app: redis + annotations: + selfLink: /api/v1/namespaces/markapi-hml/services/redis +spec: + ports: + - name: redis + protocol: TCP + port: 6379 + targetPort: 6379 + selector: + app: redis + clusterIP: + clusterIPs: + type: ClusterIP + sessionAffinity: None diff --git a/llama3/download_model.py b/llama3/download_model.py index f95014e..0b93f8f 100644 --- a/llama3/download_model.py +++ b/llama3/download_model.py @@ -1,6 +1,7 @@ from huggingface_hub import login from huggingface_hub import hf_hub_download + HF_TOKEN = 'INTRODUCE_TOKEN' login(token=HF_TOKEN) diff --git a/local.yml b/local.yml index deb337a..b01cb87 100644 --- a/local.yml +++ b/local.yml @@ -1,5 +1,3 @@ -version: "3" - services: django: &django build: @@ -13,9 +11,6 @@ services: - redis - postgres - mailhog - - solr - links: - - solr volumes: - .:/app:z env_file: @@ -30,17 +25,6 @@ services: container_name: markapi_local_mailhog ports: - "8029:8025" - - solr: - image: solr:9.3 - restart: always - ports: - - "8983:8983" - volumes: - - ./index:/var/solr - environment: - - SOLR_JAVA_MEM=-Xms512m -Xmx512m - - SOLR_HEAP=512m postgres: build: @@ -62,7 +46,6 @@ services: ports: - "6399:6379" - celeryworker: <<: *django image: markapi_local_celeryworker diff --git a/requirements/base.txt b/requirements/base.txt index d17a652..0e5f82f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -26,14 +26,6 @@ wagtail-autocomplete==0.11.0 # ------------------------------------------------------------------------------ lxml==4.9.4 # https://github.com/lxml/lxml -# Solr -# ------------------------------------------------------------------------------ -# django-haystack==3.2.1 -git+https://github.com/django-haystack/django-haystack.git - -# PySolr -# ------------------------------------------------------------------------------ -pysolr==3.9.0 # Kombu # ------------------------------------------------------------------------------