From 669ed1596249b4d17ec6e2ba8a50fc4534293ca4 Mon Sep 17 00:00:00 2001 From: Marina Date: Tue, 20 Aug 2024 11:58:32 +0300 Subject: [PATCH 1/6] base for new task --- app/server.py | 7 +++++-- app/tasks.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/server.py b/app/server.py index b8dbc5b7..25f14fe0 100644 --- a/app/server.py +++ b/app/server.py @@ -27,7 +27,7 @@ init_criterions, BASE_PRES_CRITERION, BASE_REPORT_CRITERION from root_logger import get_logging_stdout_handler, get_root_logger from servants import pre_luncher -from tasks import create_task +from tasks import create_task, convert_to_pdf from utils import checklist_filter, decorator_assertion, get_file_len, format_check from app.main.checks import CRITERIA_INFO from routes.admin import admin @@ -223,7 +223,10 @@ def run_task(): pdf_file.save(filepathpdf) converted_id = db_methods.add_file_to_db(filenamepdf, filepathpdf) else: - converted_id = db_methods.write_pdf(filename, filepath) + logger.info( + f"Запуск конвертации файла '{file.filename}' в pdf") + converted_id = convert_to_pdf(filename, filepath) + check = Check({ '_id': file_id, 'conv_pdf_fs_id': converted_id, diff --git a/app/tasks.py b/app/tasks.py index c7ba47df..549a14a6 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -77,6 +77,14 @@ def create_task(self, check_info): logger.error(f"\tПри обработке произошла ошибка: {e}. Попытка повторного запуска", exc_info=True) self.retry(countdown=TASK_RETRY_COUNTDOWN) # Retry the task, adding it to the back of the queue. +@celery.task(name="convert_to_pdf", bind=True) +def convert_to_pdf(self, filename, filepath): + try: + converted_id = db_methods.write_pdf(filename, filepath) + return converted_id + except Exception as e: + logger.error(f"При конвертации файла произошла ошибка: {e}. Новая попытка", exc_info=True) + raise self.retry(countdown=TASK_RETRY_COUNTDOWN) @celery.task(name="passback-task", queue='passback-grade') def passback_task(): From a173f008261293854cdf3a97fd849ab4335b2570 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 22 Aug 2024 13:13:20 +0300 Subject: [PATCH 2/6] getting id before pdf-convert --- app/db/db_methods.py | 9 ++++++--- app/server.py | 10 +++++++++- app/tasks.py | 9 +++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/app/db/db_methods.py b/app/db/db_methods.py index d80d92db..517c3b8c 100644 --- a/app/db/db_methods.py +++ b/app/db/db_methods.py @@ -145,13 +145,16 @@ def add_check(file_id, check): def update_check(check): return bool(checks_collection.find_one_and_replace({'_id': check._id}, check.pack())) +def get_pdf_id(file_id=None): + if not file_id: file_id = ObjectId() + return file_id -def write_pdf(filename, filepath): +def write_pdf(filename, filepath, file_id): converted_filepath = convert_to(filepath, target_format='pdf') - return add_file_to_db(filename, converted_filepath) + return add_file_to_db(filename, converted_filepath, file_id) -def add_file_to_db(filename, filepath, file_id=None): +def add_file_to_db(filename, filepath, file_id): if not file_id: file_id = ObjectId() fs.upload_from_stream_with_id(file_id, filename, open(filepath, 'rb')) return file_id diff --git a/app/server.py b/app/server.py index 25f14fe0..e8ba0f31 100644 --- a/app/server.py +++ b/app/server.py @@ -225,7 +225,8 @@ def run_task(): else: logger.info( f"Запуск конвертации файла '{file.filename}' в pdf") - converted_id = convert_to_pdf(filename, filepath) + converted_id = str(db_methods.get_pdf_id(file_id=None)) + convert_to_pdf.delay(filename, filepath, converted_id) check = Check({ '_id': file_id, @@ -244,6 +245,13 @@ def run_task(): db_methods.add_check(file_id, check) # add check for parsed_file to db task = create_task.delay(check.pack(to_str=True)) # add check to queue db_methods.add_celery_task(task.id, file_id) # mapping celery_task to check (check_id = file_id) +# task_chain = chain( +# convert_to_pdf.s(filename, filepath, converted_id), +# create_task.s(check.pack(to_str=True)) +# ) +# result = task_chain.apply_async() +# task_id = result.id + return {'task_id': task.id, 'check_id': str(file_id)} diff --git a/app/tasks.py b/app/tasks.py index 549a14a6..b59cebc8 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -77,11 +77,12 @@ def create_task(self, check_info): logger.error(f"\tПри обработке произошла ошибка: {e}. Попытка повторного запуска", exc_info=True) self.retry(countdown=TASK_RETRY_COUNTDOWN) # Retry the task, adding it to the back of the queue. -@celery.task(name="convert_to_pdf", bind=True) -def convert_to_pdf(self, filename, filepath): +@celery.task(name="convert_to_pdf", queue='check-solution', bind=True) +def convert_to_pdf(self, filename, filepath, file_id): try: - converted_id = db_methods.write_pdf(filename, filepath) - return converted_id + conv_id = db_methods.write_pdf(filename, filepath, file_id) + # converted_id = db_methods.write_pdf(filename, filepath) + return conv_id except Exception as e: logger.error(f"При конвертации файла произошла ошибка: {e}. Новая попытка", exc_info=True) raise self.retry(countdown=TASK_RETRY_COUNTDOWN) From b7672d63afbb391e5ac3e383f3c8f0b26e8af288 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 22 Aug 2024 17:33:06 +0300 Subject: [PATCH 3/6] fix objectId --- app/db/db_methods.py | 4 ++-- app/tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/db/db_methods.py b/app/db/db_methods.py index 517c3b8c..8f18e973 100644 --- a/app/db/db_methods.py +++ b/app/db/db_methods.py @@ -155,8 +155,8 @@ def write_pdf(filename, filepath, file_id): def add_file_to_db(filename, filepath, file_id): - if not file_id: file_id = ObjectId() - fs.upload_from_stream_with_id(file_id, filename, open(filepath, 'rb')) + pdf_obj_id = ObjectId(file_id) + fs.upload_from_stream_with_id(pdf_obj_id, filename, open(filepath, 'rb')) return file_id diff --git a/app/tasks.py b/app/tasks.py index b59cebc8..752623f7 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -78,9 +78,9 @@ def create_task(self, check_info): self.retry(countdown=TASK_RETRY_COUNTDOWN) # Retry the task, adding it to the back of the queue. @celery.task(name="convert_to_pdf", queue='check-solution', bind=True) -def convert_to_pdf(self, filename, filepath, file_id): +def convert_to_pdf(self, filename, filepath, pdf_id): try: - conv_id = db_methods.write_pdf(filename, filepath, file_id) + conv_id = db_methods.write_pdf(filename, filepath, pdf_id) # converted_id = db_methods.write_pdf(filename, filepath) return conv_id except Exception as e: From 063f5fa72669218451f9cb711dd442836e39c16c Mon Sep 17 00:00:00 2001 From: Marina Date: Sun, 1 Sep 2024 15:58:01 +0300 Subject: [PATCH 4/6] delete spaces --- app/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server.py b/app/server.py index e8ba0f31..f00bce72 100644 --- a/app/server.py +++ b/app/server.py @@ -224,7 +224,7 @@ def run_task(): converted_id = db_methods.add_file_to_db(filenamepdf, filepathpdf) else: logger.info( - f"Запуск конвертации файла '{file.filename}' в pdf") + f"Запуск конвертации файла '{file.filename}' в pdf") converted_id = str(db_methods.get_pdf_id(file_id=None)) convert_to_pdf.delay(filename, filepath, converted_id) From 81a5cbeb6880d2a7fa63916050a35183849c3d86 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 26 Sep 2024 17:25:39 +0300 Subject: [PATCH 5/6] add chain: start check after convert to pdf --- app/server.py | 22 +++++++++++----------- app/tasks.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/server.py b/app/server.py index 130dfbba..50f87cd6 100644 --- a/app/server.py +++ b/app/server.py @@ -10,6 +10,7 @@ import bson import pandas as pd from bson import ObjectId +from celery import chain from celery.result import AsyncResult from flask import (Flask, Response, abort, jsonify, redirect, render_template, request, url_for) @@ -226,7 +227,7 @@ def run_task(): logger.info( f"Запуск конвертации файла '{file.filename}' в pdf") converted_id = str(db_methods.get_pdf_id(file_id=None)) - convert_to_pdf.delay(filename, filepath, converted_id) + # convert_to_pdf.delay(filename, filepath, converted_id) check = Check({ '_id': file_id, @@ -243,16 +244,15 @@ def run_task(): 'params_for_passback': current_user.params_for_passback }) db_methods.add_check(file_id, check) # add check for parsed_file to db - task = create_task.delay(check.pack(to_str=True)) # add check to queue - db_methods.add_celery_task(task.id, file_id) # mapping celery_task to check (check_id = file_id) -# task_chain = chain( -# convert_to_pdf.s(filename, filepath, converted_id), -# create_task.s(check.pack(to_str=True)) -# ) -# result = task_chain.apply_async() -# task_id = result.id - - return {'task_id': task.id, 'check_id': str(file_id)} + # task = create_task.delay(check.pack(to_str=True)) # add check to queue + # db_methods.add_celery_task(task.id, file_id) # mapping celery_task to check (check_id = file_id) + task_chain = chain( + convert_to_pdf.s(filename, filepath, converted_id), + create_task.s(check.pack(to_str=True)) +) + result = task_chain.apply_async() + task_id = result.id + return {'task_id': task_id, 'check_id': str(file_id)} @app.route("/recheck/", methods=["GET"]) diff --git a/app/tasks.py b/app/tasks.py index 752623f7..08b83e2a 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -45,7 +45,7 @@ def at_start(sender, **k): @celery.task(name="create_task", queue='check-solution', bind=True) -def create_task(self, check_info): +def create_task(self, conv_id, check_info): check_obj = Check(check_info) check_id = str(check_obj._id) # get check files filepath From c3b0ab1534bd963cd2a6b4ca411dccada26865b6 Mon Sep 17 00:00:00 2001 From: Marina Date: Thu, 26 Sep 2024 18:06:25 +0300 Subject: [PATCH 6/6] html: pdf upload after check_is_ended --- app/templates/results.html | 14 +++++++------- assets/scripts/results.js | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/app/templates/results.html b/app/templates/results.html index 8f05f6f4..a728a397 100644 --- a/app/templates/results.html +++ b/app/templates/results.html @@ -96,18 +96,18 @@

{% endfor %} +
+ + + + Слайд из + +
{% endif %} {% if not current_user.is_anonymous and results.conv_pdf_fs_id %}
-
- - - - Слайд из - -
{% endif %} diff --git a/assets/scripts/results.js b/assets/scripts/results.js index 3f43d45b..b8c2ecac 100644 --- a/assets/scripts/results.js +++ b/assets/scripts/results.js @@ -87,7 +87,11 @@ $(function(){ canvas = document.getElementById('the-canvas'); ctx = canvas.getContext('2d'); - + + ctx.font = "20px Times New Roman"; + ctx.textAlign = "center"; + ctx.fillText("Конвертация файла в PDF...", canvas.width / 2, canvas.height / 2); + pdfjsLib .getDocument(href) .promise.then(pdfDoc_ => {