sio2project
diff --git a/‎oioioi/base/utils/execute.py
+3-1 b/‎oioioi/base/utils/execute.py
+3-1
diff --git a/‎oioioi/base/utils/pdf.py
+43-3 b/‎oioioi/base/utils/pdf.py
+43-3
diff --git a/‎oioioi/contests/tests/tests.py
+1-3 b/‎oioioi/contests/tests/tests.py
+1-3
diff --git a/‎oioioi/default_settings.py
+7-7 b/‎oioioi/default_settings.py
+7-7
diff --git a/‎oioioi/disqualification/admin.py
+3-3 b/‎oioioi/disqualification/admin.py
+3-3
diff --git a/‎oioioi/oireports/tests.py
+11-13 b/‎oioioi/oireports/tests.py
+11-13
diff --git a/‎oioioi/pa/score.py
+3 b/‎oioioi/pa/score.py
+3
diff --git a/‎oioioi/pa/tests.py
+1-2 b/‎oioioi/pa/tests.py
+1-2
diff --git a/‎oioioi/printing/pdf.py
+2-1 b/‎oioioi/printing/pdf.py
+2-1
diff --git a/‎oioioi/printing/tests.py
+6-11 b/‎oioioi/printing/tests.py
+6-11
diff --git a/‎oioioi/problems/models.py
+1-1 b/‎oioioi/problems/models.py
+1-1
diff --git a/‎oioioi/problems/tests.py
+20-4 b/‎oioioi/problems/tests.py
+20-4
diff --git a/‎oioioi/problems/views.py
+6-7 b/‎oioioi/problems/views.py
+6-7
@@ -2,6 +2,8 @@
 import string
 import subprocess
 
+import six
+
 # Reliable quoting, taken as-is from `pipes` standard module.
 # There is a slight chance it may work on non-Unix platforms, but
 # I wouldn't count on it. Either way tests will show.
@@ -94,7 +96,7 @@ def set_cwd():
             preexec_fn=set_cwd,
         )
 
-    stdout, _ = p.communicate(stdin)
+    stdout, _ = p.communicate(six.ensure_binary(stdin))
     rc = p.returncode
 
     if split_lines:
 
@@ -1,17 +1,25 @@
-# pylint: disable=dangerous-default-value
 import codecs
+import io
 import os.path
 import shutil
 import tempfile
 
+import pdfminer.layout
+import six
 from django.core.files.base import File
+from pdfminer.converter import TextConverter
+from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager
+from pdfminer.pdfpage import PDFPage
 from six.moves import range
 
 from oioioi.base.utils.execute import execute
 from oioioi.filetracker.utils import stream_file
 
 
-def generate_pdf(tex_code, filename, extra_args=[], num_passes=3):
+def generate_pdf(tex_code, filename, extra_args=None, num_passes=3):
+    if extra_args is None:
+        extra_args = []
+
     # Create temporary file and folder
     tmp_folder = tempfile.mkdtemp()
     try:
@@ -28,7 +36,39 @@ def generate_pdf(tex_code, filename, extra_args=[], num_passes=3):
             execute(command, cwd=tmp_folder)
 
         # Get PDF file contents
-        pdf_file = open(os.path.splitext(tex_path)[0] + '.pdf')
+        pdf_file = io.open(os.path.splitext(tex_path)[0] + '.pdf', "rb")
         return stream_file(File(pdf_file), filename)
     finally:
         shutil.rmtree(tmp_folder)
+
+
+def extract_text_from_pdf(pdf_file):
+    # pdf_file must be a a file-like object
+    # returns a list of strings, each string containing text from one page
+
+    # the char_margin is needed because pdfminer.six has a problem
+    # that causes lines with big spacing between text blocks to be split into
+    # many lines, sometimes out-of-reasonable-orded
+    # the value needs to be high enough so that char_width * char_margin > page_width
+
+    laparams = pdfminer.layout.LAParams(char_margin=2000)
+
+    pages = []
+    output_string = six.BytesIO()
+    rsrcmgr = PDFResourceManager()
+    device = TextConverter(rsrcmgr, output_string, codec='latin-1', laparams=laparams)
+    interpreter = PDFPageInterpreter(rsrcmgr, device)
+
+    for page in PDFPage.get_pages(
+        pdf_file,
+        None,
+        maxpages=0,
+        password='',
+        caching=False,
+        check_extractable=True,
+    ):
+        interpreter.process_page(page)
+        pages.append(output_string.getvalue())
+
+    output_string.close()
+    return pages
@@ -2851,9 +2851,7 @@ def setUp(self):
 
     def test_submission_view_without_contest(self):
         submission = Submission.objects.get(id=1)
-        response = self.client.get(
-            reverse('submission', kwargs={'submission_id': 1})
-        )
+        response = self.client.get(reverse('submission', kwargs={'submission_id': 1}))
         self.assertEqual(response.status_code, 200)
 
         # Submit another button
 
@@ -1,4 +1,3 @@
-# pylint: disable=wildcard-import
 import sys
 
 from oioioi.base.utils.finders import find_executable_path
@@ -497,22 +496,23 @@
 
 # Celery configuration
 
-from sio.celery.default_config import *
+CELERY_QUEUES = {}
+CELERY_RESULT_BACKEND = 'amqp'
+CELERY_ACKS_LATE = True
+CELERY_SEND_EVENTS = True
 
 BROKER_URL = 'sqla+sqlite:///' + os.path.join(tempfile.gettempdir(),
                                              'celerydb.sqlite')
 
-# pylint: disable=undefined-variable
-
-CELERY_IMPORTS += [
+CELERY_IMPORTS = [
     'oioioi.evalmgr.tasks',
     'oioioi.problems.unpackmgr',
 ]
 
-CELERY_ROUTES.update({
+CELERY_ROUTES = {
     'oioioi.evalmgr.tasks.evalmgr_job': dict(queue='evalmgr'),
     'oioioi.problems.unpackmgr.unpackmgr_job': dict(queue='unpackmgr'),
-})
+}
 
 # Number of concurrently evaluated submissions
 EVALMGR_CONCURRENCY = 1
 
@@ -1,13 +1,13 @@
-from django.db import models
-from django.forms import Textarea
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
+from django.db import models
+from django.forms import Textarea
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
 from oioioi.base import admin
 from oioioi.base.utils import make_html_link
-from oioioi.contests.admin import contest_site, ContestAdmin
+from oioioi.contests.admin import ContestAdmin, contest_site
 from oioioi.contests.menu import contest_admin_menu_registry
 from oioioi.contests.models import Submission
 from oioioi.contests.utils import is_contest_admin
 
@@ -1,18 +1,13 @@
 from datetime import datetime  # pylint: disable=E0611
 
-import six
+from six import BytesIO
 from django import VERSION as DJANGO_VERSION
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.utils.timezone import utc
-from six import BytesIO
-
-if six.PY2:
-    import slate
-else:
-    import slate3k as slate
 
 from oioioi.base.tests import TestCase, fake_time
+from oioioi.base.utils.pdf import extract_text_from_pdf
 from oioioi.contests.models import Contest
 from oioioi.filetracker.tests import TestStreamingMixin
 from oioioi.oireports.views import CONTEST_REPORT_KEY
@@ -58,12 +53,15 @@ def test_pdf_report_view(self):
         self.assertTrue(self.client.login(username='test_admin'))
         with fake_time(datetime(2015, 8, 5, tzinfo=utc)):
             response = self.client.post(url, post_vars)
-            pages = slate.PDF(BytesIO(self.streamingContent(response)))
-            self.assertIn("test_user", pages[0])
-            self.assertIn("Wynik:34", pages[0])
-            self.assertIn("ZAD1", pages[0])
-            self.assertIn("1bRuntimeerror0.00s/0.10sprogramexited", pages[0])
-            self.assertNotIn("test_user2", pages.text())
+
+            pages = extract_text_from_pdf(BytesIO(self.streamingContent(response)))
+            self.assertIn(b"test_user", pages[0])
+            self.assertIn(b"Wynik:34", pages[0].replace(b' ', b''))
+            self.assertIn(b"ZAD1", pages[0])
+            self.assertIn(
+                b"1bRuntimeerror0.00s/0.10sprogramexited", pages[0].replace(b' ', b'')
+            )
+            self.assertTrue(all(b"test_user2" not in page for page in pages))
 
     def test_xml_view(self):
         contest = Contest.objects.get()
 
@@ -91,6 +91,9 @@ def __unicode__(self):
     def __repr__(self):
         return "PAScore(%r, %r)" % (self.points, self.distribution)
 
+    def __str__(self):
+        return str(self.points)
+
     @classmethod
     def _from_repr(cls, value):
         points, distribution = value.split(';')
 
@@ -1,5 +1,4 @@
 import re
-import urllib
 from datetime import datetime  # pylint: disable=E0611
 
 import six
@@ -225,7 +224,7 @@ def check_order(response, expected):
             # 28 (10, 8, 6, 4), 28 (9, 9, 7, 3), 10 (10)
             response = self.client.get(self._ranking_url(A_PLUS_B_RANKING_KEY))
             check_order(response, [b'Test User', b'Test User 2', b'Test User 3'])
-            self.assertContains(response, '28</td>')
+            self.assertContains(response, b'28</td>')
 
             # 10 (10), 10 (7, 3), 10 (6, 4)
             response = self.client.get(self._ranking_url(B_RANKING_KEY))
 
@@ -1,5 +1,6 @@
 import os
 
+import six
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 from fpdf import FPDF
@@ -111,4 +112,4 @@ def generator(source, header):
     pdf.multi_cell(
         w=pdf.column_width, h=pdf.cell_height, txt=source, border=0, align='L'
     )
-    return pdf.output(dest='S')
+    return six.ensure_binary(pdf.output(dest='S'), 'latin-1')
@@ -1,15 +1,10 @@
-import six
+from six import BytesIO
 from django.core.files.base import ContentFile
 from django.core.urlresolvers import reverse
 from django.test.utils import override_settings
-from six import StringIO
-
-if six.PY2:
-    import slate
-else:
-    import slate3k as slate
 
 from oioioi.base.tests import TestCase
+from oioioi.base.utils.pdf import extract_text_from_pdf
 from oioioi.contests.controllers import ContestController
 from oioioi.contests.models import Contest
 from oioioi.printing.pdf import generator
@@ -37,11 +32,11 @@ def can_print_files(self, request):
 
 class TestPDFGenerator(TestCase):
     def test_pdf_generation(self):
-        pdf = StringIO(generator(source=SAMPLE_TEXT, header='header'))
-        text = slate.PDF(pdf)
+        pdf = BytesIO(generator(source=SAMPLE_TEXT, header='header'))
+        text = extract_text_from_pdf(pdf)
         self.assertEqual(9, len(text))
-        self.assertIn('Lorem ipsum dolor', text[0])
-        self.assertIn('Sed egestas dui tellus', text[4])
+        self.assertIn(b'Lorem ipsum dolor', text[0])
+        self.assertIn(b'Sed egestas dui tellus', text[4])
 
 
 class TestPrintingView(TestCase):
 
@@ -161,7 +161,7 @@ def __str__(self):
         }
 
     def save(self, *args, **kwargs):
-        self.ascii_name = unidecode(self.name)
+        self.ascii_name = unidecode(six.text_type(self.name))
         super(Problem, self).save(*args, **kwargs)
 
 
 
@@ -2,6 +2,7 @@
 
 import os.path
 from datetime import datetime  # pylint: disable=E0611
+from functools import cmp_to_key
 
 import pytest
 import six.moves.urllib.parse
@@ -1993,14 +1994,29 @@ def _get_table_contents(self, html):
                     pos = pos2 + len('</td>')
         return rows
 
+    @staticmethod
+    def _cmp_str_with_none(key_fn):
+        def _cmp(a, b):
+            key_a = key_fn(a)
+            key_b = key_fn(b)
+            if key_a is None:
+                return True
+            if key_b is None:
+                return False
+            return key_a < key_b
+
+        return _cmp
+
     def _assert_rows_sorted(self, rows, order_by=0, desc=False):
         # Nones should be treated as if they were less than zeroes
         # (i.e. listed last when desc=True and listed first otherwise).
-        denullified_rows = [map(lambda x: -1 if x is None else x, row) for row in rows]
-
         self.assertEqual(
-            denullified_rows,
-            sorted(denullified_rows, key=lambda x: x[order_by], reverse=desc),
+            rows,
+            sorted(
+                rows,
+                key=cmp_to_key(self._cmp_str_with_none(lambda x: x[order_by])),
+                reverse=desc,
+            ),
         )
 
     def test_statistics_problem_list(self):
 
@@ -246,7 +246,7 @@ def filter_problems_by_origin(problems, origintags):
 
 
 def search_problems_in_problemset(datadict):
-    query = unidecode(datadict.get('q', ''))
+    query = unidecode(six.text_type(datadict.get('q', '')))
     tags = datadict.getlist('tag')
     algorithmtags = datadict.getlist('algorithm')
     difficultytags = datadict.getlist('difficulty')
@@ -1137,12 +1137,11 @@ def get_search_hints_view(request, view_type):
         raise PermissionDenied
     query = unidecode(request.GET.get('q', ''))
 
-    result = (
-        list(get_problem_hints(query, view_type, request.user))
-        + get_tag_hints(query)
-        + list(get_origintag_hints(query))
-        + list(get_origininfovalue_hints(query))
-    )
+    result = []
+    result.extend(list(get_problem_hints(query, view_type, request.user)))
+    result.extend(get_tag_hints(query))
+    result.extend(get_origintag_hints(query))
+    result.extend(get_origininfovalue_hints(query))
 
     # Convert category names in results from lazy translation to strings
     # Since jsonify throws error if given lazy translation objects
Original file line number	Diff line number	Diff line change
`@@ -161,7 +161,7 @@ def __str__(self):`
`161`	`161`	`}`
`162`	`162`
`163`	`163`	`def save(self, args, *kwargs):`
`164`		`- self.ascii_name = unidecode(self.name)`
	`164`	`+ self.ascii_name = unidecode(six.text_type(self.name))`
`165`	`165`	`super(Problem, self).save(args, *kwargs)`
`166`	`166`
`167`	`167`