.gitlab-ci.yml

 # This base image can be found in 'Dockerfile'
-image: zesje/base
+image: gitlab.kwant-project.org:5005/zesje/zesje/test:latest
 
 stages:
   - build
@@ -13,11 +13,13 @@ stages:
     paths:
       - .yarn-cache
   before_script:
+    - source activate zesje-dev
     - yarn install --cache-folder .yarn-cache
 
 .python_packages: &python_packages
   before_script:
-    - pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+    - source activate zesje-dev
+    - conda env update
 
 build:
   <<: *node_modules

Dockerfile

-FROM archlinux/base
+FROM continuumio/miniconda3
 
-## Install packages and clear the cache after installation. Yarn is fixed at 1.6.0 untill 1.8.0 is released due to a critical bug.
-RUN pacman -Sy --noconfirm nodejs python-pip git libdmtx libsm libxrender libxext gcc libmagick6 imagemagick ghostscript; \
-    pacman -U --noconfirm https://archive.archlinux.org/packages/y/yarn/yarn-1.6.0-1-any.pkg.tar.xz
+RUN apt-get update -y && apt-get install -y libdmtx0a libmagickwand-dev
 
-WORKDIR ~
-ADD requirements*.txt ./
-#ADD package.json .
-RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt;
-#RUN yarn install; \
-#    yarn cache clean; \
-#    rm package.json
+RUN apt-get update && \
+    apt-get install -y \
+        curl \
+        poppler-utils build-essential libgl1-mesa-glx \
+        imagemagick libsm-dev libdmtx-dev libdmtx0a libmagickwand-dev \
+        && \
+    apt-get -y --quiet install git supervisor nginx
 
-CMD bash
\ No newline at end of file
+WORKDIR /app
+
+ADD environment.yml /app/environment.yml
+RUN conda env create
+
+# From https://medium.com/@chadlagore/conda-environments-with-docker-82cdc9d25754
+RUN echo "source activate $(head -1 /app/environment.yml | cut -d' ' -f2)" > ~/.bashrc
+ENV PATH /opt/conda/envs/$(head -1 /app/environment.yml | cut -d' ' -f2)/bin:$PATH
+
+RUN rm /app/environment.yml
+
+CMD bash

README.md

@@ -15,10 +15,11 @@ Install Miniconda by following the instructions on this page:
 
 https://conda.io/miniconda.html
 
-Create a Conda environment that you will use for installing all
-of zesje's dependencies:
+Make sure you cloned this repository and enter its directory. Then 
+create a Conda environment that will automatically install all 
+of zesje's Python dependencies:
 
-    conda create -c conda-forge -n zesje-dev python=3.6 yarn
+    conda env create  # Creates an environment from environment.yml 
 
 Then, *activate* the conda environment:
 
@@ -31,10 +32,6 @@ Install all of the Javascript dependencies:
 
     yarn install
 
-Install all of the Python dependencies:
-
-    pip install -r requirements.txt -r requirements-dev.txt
-
 Unfortunately there is also another dependency that must be installed
 manually for now (we are working to bring this dependency into the
 Conda ecosystem). You can install this dependency in the following way
@@ -145,10 +142,10 @@ If you use Atom, install the [linter-js-standard-engine](https://atom.io/package
 ### Adding dependencies
 
 #### Server-side
-If you start using a new Python library, be sure to add it to `requirements.txt`. Python libraries for the testing are in `requirements-dev.txt`.
-The packages can be installed and updated in your environment by `pip` using
+If you start using a new Python library, be sure to add it to `environment.yml`.
+The packages can be installed and updated in your environment by `conda` using
 
-    pip install -r requirements.txt -r requirements-dev.txt
+    conda env update
 
 
 #### Client-side

barcode_example_generator.py

+
+import sys
+import os
 from io import BytesIO
 
 from reportlab.pdfgen import canvas
-import PIL
 from wand.image import Image
 from wand.color import Color
-from pystrich.datamatrix import DataMatrixEncoder
 
+sys.path.append(os.getcwd())
 
-def generate_datamatrix(exam_id, page_num, copy_num):
-    data = f'{exam_id}/{copy_num:04d}/{page_num:02d}'
+from zesje.pdf_generation import generate_datamatrix  # noqa: E402
+from zesje.database import token_length  # noqa: E402
 
-    image_bytes = DataMatrixEncoder(data).get_imagedata(cellsize=2)
-    return PIL.Image.open(BytesIO(image_bytes))
 
+exam_token = "A" * token_length
+copy_num = 1559
+page_num = 0
 
-datamatrix = generate_datamatrix(0, 0, 0)
-datamatrix_x = datamatrix_y = 0
-fontsize = 8
-margin = 3
+fontsize = 12
+datamatrix_x = 0
+datamatrix_y = fontsize
 
-datamatrix = generate_datamatrix(0, 0, 0000)
-imagesize = (datamatrix.width, 3 + fontsize + datamatrix.height)
+datamatrix = generate_datamatrix(exam_token, page_num, copy_num)
+imagesize = (datamatrix.width, fontsize + datamatrix.height)
 
 result_pdf = BytesIO()
 canv = canvas.Canvas(result_pdf, pagesize=imagesize)
 
-canv.drawInlineImage(datamatrix, 0, 3 + fontsize)
+canv.drawInlineImage(datamatrix, datamatrix_x, datamatrix_y)
 
 canv.setFont('Helvetica', fontsize)
-canv.drawString(0, 3, f"  # 1519")
+canv.drawString(datamatrix_x, datamatrix_y - (fontsize * 0.66),
+                f" # {copy_num}")
 
 canv.showPage()
 canv.save()
@@ -36,7 +39,7 @@ canv.save()
 result_pdf.seek(0)
 
 # From https://stackoverflow.com/questions/27826854/python-wand-convert-pdf-to-png-disable-transparent-alpha-channel
-with Image(file=result_pdf, resolution=80) as img:
-    with Image(width=img.width, height=img.height, background=Color("white")) as bg:
+with Image(file=result_pdf, resolution=72) as img:
+    with Image(width=imagesize[0], height=imagesize[1], background=Color("white")) as bg:
         bg.composite(img, 0, 0)
         bg.save(filename="client/components/barcode_example.png")

client/views/ExamEditor.jsx

@@ -112,6 +112,7 @@ class ExamEditor extends React.Component {
         api.post('problems', formData).then(result => {
           widgetData.id = result.widget_id
           problemData.id = result.id
+          problemData.name = result.problem_name
           widgetData.problem = problemData
 
           this.props.createNewWidget(widgetData)

environment.yml

+name: zesje-dev
+channels:
+    - conda-forge
+    - anaconda
+dependencies:
+    - python=3.6 
+    - yarn
+    - gunicorn
+    - redis
+    - pip
+    - pip:
+        # Core components
+        - flask
+        - flask_restful
+        - flask_sqlalchemy
+        - sqlalchemy
+        - Flask-Migrate
+        - alembic
+        - pyyaml
+        - celery
+        - redis
+
+        # General utilities
+        - numpy==1.15.4
+        - scipy==1.3.0
+
+        # summary plot generation
+        - matplotlib
+        - seaborn
+
+        # PDF generation
+        - pdfrw
+        - reportlab
+        - Wand
+        - Pillow  # also scan processing
+
+        # Scan processing
+        - opencv-python
+        - pikepdf
+        - pylibdmtx
+        - pdfminer3
+
+        # Exporting
+        - pandas
+        - openpyxl  # required for writing dataframes as Excel spreadsheets
+
+        #
+        # Development dependencies
+        #
+
+        # Tests
+        - pytest
+        - pyssim
+        - pytest-cov
+
+        # Linting
+        - flake8

package.json

@@ -3,7 +3,7 @@
   "main": "index.js",
   "license": "AGPL-3.0",
   "scripts": {
-    "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON,CELERY\" --prefix-colors \"bgBlue.bold,bgGreen.bold,bgRed.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg celery -A zesje.celery worker -l info --autoscale=4,1 --max-tasks-per-child=16\"",
+    "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON,CELERY,REDIS\" --prefix-colors \"bgBlue.bold,bgGreen.bold,bgRed.bold,bgYellow.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg celery -A zesje.celery worker -l info --autoscale=4,1 --max-tasks-per-child=16\" \"redis-server redis.conf\"",
     "build": "webpack --config webpack.prod.js",
     "ci": "yarn lint; yarn test",
     "lint": "yarn lint:js; yarn lint:py",

redis.conf

+port 6479
+loglevel notice
\ No newline at end of file

requirements-dev.txt

-# Tests
-pytest
-pyssim
-pytest-cov
-
-# Linting
-flake8

requirements.txt

-# Core components
-flask
-flask_restful
-flask_sqlalchemy
-sqlalchemy
-Flask-Migrate
-alembic
-pyyaml
-celery
-redis
-
-# General utilities
-numpy
-scipy
-
-# summary plot generation
-matplotlib
-seaborn
-
-# PDF generation
-pdfrw
-reportlab
-Wand
-Pillow  # also scan processing
-pyStrich  # TODO: can we replace this with stuff from pylibdmtx?
-
-# Scan processing
-opencv-python
-git+https://github.com/mstamy2/PyPDF2
-pylibdmtx
-
-# Exporting
-pandas
-openpyxl  # required for writing dataframes as Excel spreadsheets

tests/conftest.py

 import os
 
 import pytest
-
 from flask import Flask
 from zesje.api import api_bp
 from zesje.database import db
@@ -40,3 +39,12 @@ def test_client(app):
     with app.app_context():
         db.drop_all()
         db.create_all()
+
+
+@pytest.fixture
+def empty_app(app):
+    with app.app_context():
+        db.drop_all()
+        db.create_all()
+
+    return app

tests/test_database.py

 import pytest
 from flask import Flask
 
-from zesje.database import db, Exam, _generate_exam_token
+from zesje.database import db, _generate_exam_token, Exam, Problem, ProblemWidget, Solution
+from zesje.database import Submission, Scan, Page, ExamWidget, FeedbackOption
 
 
 @pytest.mark.parametrize('duplicate_count', [
@@ -32,3 +33,141 @@ def test_exam_generate_token_length_uppercase(duplicate_count, monkeypatch):
         id = _generate_exam_token()
         assert len(id) == 12
         assert id.isupper()
+
+
+def test_cascades_exam(empty_app, exam, problem, submission, scan, exam_widget):
+    """Tests the cascades defined for an exam
+
+    Tests the cascades for the following relations:
+    - Exam -> Submission
+    - Exam -> Problem
+    - Exam -> Scan
+    - Exam -> ExamWidget
+    """
+    empty_app.app_context().push()
+    exam.problems = [problem]
+    exam.scans = [scan]
+    exam.submissions = [submission]
+    exam.widgets = [exam_widget]
+
+    db.session.add(exam)
+    db.session.commit()
+
+    assert problem in db.session
+    assert submission in db.session
+    assert scan in db.session
+    assert exam_widget in db.session
+
+    db.session.delete(exam)
+    db.session.commit()
+
+    assert problem not in db.session
+    assert submission not in db.session
+    assert scan not in db.session
+    assert exam_widget not in db.session
+
+
+def test_cascades_problem(empty_app, exam, problem, submission, solution, problem_widget, feedback_option):
+    """Tests the cascades defined for a problem
+
+    Tests the cascades for the following relations:
+    - Problem -> Solution
+    - Problem -> ProblemWidget
+    - Problem -> FeedbackOption
+    """
+    empty_app.app_context().push()
+
+    exam.problems = [problem]
+    exam.submissions = [submission]
+    solution.submission = submission
+    problem.widget = problem_widget
+    problem.solutions = [solution]
+    problem.feedback_options = [feedback_option]
+
+    db.session.add_all([exam, problem, submission])
+    db.session.commit()
+
+    assert solution in db.session
+    assert problem_widget in db.session
+    assert feedback_option in db.session
+
+    db.session.delete(problem)
+    db.session.commit()
+
+    assert solution not in db.session
+    assert problem_widget not in db.session
+    assert feedback_option not in db.session
+
+
+def test_cascades_submission(empty_app, exam, problem, submission, solution, page):
+    """Tests the cascades defined for a submission
+
+    Tests the cascades for the following relations:
+    - Submission -> Solution
+    - Submission -> Page
+    """
+    empty_app.app_context().push()
+
+    exam.problems = [problem]
+    exam.submissions = [submission]
+
+    solution.problem = problem
+    solution.submission = submission
+    page.submission = submission
+
+    db.session.add_all([exam, problem, submission])
+    db.session.commit()
+
+    assert solution in db.session
+    assert page in db.session
+
+    db.session.delete(submission)
+    db.session.commit()
+
+    assert solution not in db.session
+    assert page not in db.session
+
+
+@pytest.fixture
+def exam():
+    return Exam(name='')
+
+
+@pytest.fixture
+def problem():
+    return Problem(name='')
+
+
+@pytest.fixture
+def problem_widget():
+    return ProblemWidget(name='', page=0, x=0, y=0, width=0, height=0)
+
+
+@pytest.fixture
+def exam_widget():
+    return ExamWidget(name='', x=0, y=0)
+
+
+@pytest.fixture
+def submission():
+    return Submission(copy_number=0)
+
+
+@pytest.fixture
+def solution():
+    return Solution()
+
+
+@pytest.fixture
+def scan():
+    return Scan(name='', status='')
+
+
+@pytest.fixture
+def page():
+    return Page(path='', number=0)
+
+
+@pytest.fixture
+def feedback_option():
+    return FeedbackOption(text='')

tests/test_images.py

+import cv2
+import os
+import numpy as np
+import pytest
+
+from zesje.images import get_delta, get_corner_marker_sides, fix_corner_markers, add_tup, sub_tup
+from zesje.scans import find_corner_marker_keypoints
+
+
+@pytest.mark.parametrize(
+    'shape,corners,expected',
+    [((240, 200, 3), [(120, 50), (50, 200), (120, 200)], (50, 50)),
+     ((240, 200, 3), [(50, 50), (50, 200), (120, 200)], (120, 50)),
+     ((240, 200, 3), [(50, 50), (120, 50), (120, 200)], (50, 200)),
+     ((240, 200, 3), [(50, 50), (120, 50), (50, 200)], (120, 200))],
+    ids=["missing top left", "missing top right", "missing bottom left", "missing bottom right"])
+def test_three_straight_corners(shape, corners, expected):
+    corner_markers = fix_corner_markers(corners, shape)
+    assert expected in corner_markers
+
+
+def test_pdf(datadir):
+    # Max deviation of inferred corner marker and actual location
+    epsilon = 2
+
+    # Scan rotated image with 4 corner markers
+    image_filename1 = 'a4-rotated.png'
+    image_path = os.path.join(datadir, 'cornermarkers', image_filename1)
+    page_img = cv2.imread(image_path)
+
+    corners1 = find_corner_marker_keypoints(page_img)
+
+    # Scan the same image with 3 corner markers
+    image_filename2 = 'a4-rotated-3-markers.png'
+    image_path = os.path.join(datadir, 'cornermarkers', image_filename2)
+    page_img = cv2.imread(image_path)
+
+    corners2 = find_corner_marker_keypoints(page_img)
+
+    # Get marker that was removed
+    diff = [corner for corner in corners1 if corner not in corners2]
+    diff_marker = min(diff)
+
+    fixed_corners2 = fix_corner_markers(corners2, page_img.shape)
+    added_marker = [corner for corner in fixed_corners2 if corner not in corners1][0]
+
+    # Check if 'inferred' corner marker is not too far away
+    dist = np.linalg.norm(np.subtract(added_marker, diff_marker))
+
+    assert dist < epsilon
+
+
+@pytest.mark.parametrize(
+    'inputs,expected',
+    [
+        (((0, 1), (1, 1), (0, 0), None), (0, 1)),
+        (((0, 1), None, (0, 0), (0, 1)), (0, 1)),
+        (((1, 1), (2, 1), None, (2, 2)), (0, -1)),
+        ((None, (1, 1), (1, 2), (2, 2)), (-1, -1))
+    ],
+    ids=["missing bottom right", "missing top right", "missing bottom left", "missing top left"]
+)
+def test_get_delta(inputs, expected):
+    # unpack inputs so that the individual elements are paramaters
+    delta = get_delta(*inputs)
+
+    assert delta == expected
+
+
+def test_get_corner_marker_sides_all_four():
+    shape = (100, 100)
+    corner_markers = [(0, 0), (100, 0), (0, 100), (100, 100)]
+
+    assert tuple(corner_markers) == get_corner_marker_sides(corner_markers, shape)
+
+
+def test_get_corner_markers_three():
+    shape = (100, 100)
+    corner_markers = [(0, 0), (0, 100), (100, 0)]
+
+    top_left, top_right, bottom_left, bottom_right = get_corner_marker_sides(corner_markers, shape)
+
+    assert not bottom_right
+
+
+def test_add_tup():
+    tup1 = tup2 = (1, 1)
+
+    assert add_tup(tup1, tup2) == (2, 2)
+
+
+def test_sub_tup():
+    tup1 = tup2 = (1, 1)
+
+    assert sub_tup(tup1, tup2) == (0, 0)

tests/test_scans.py

@@ -8,6 +8,7 @@ from tempfile import NamedTemporaryFile
 from flask import Flask
 from io import BytesIO
 import wand.image
+from pikepdf import Pdf
 
 from zesje.scans import decode_barcode, ExamMetadata, ExtractedBarcode
 from zesje.database import db, _generate_exam_token
@@ -277,6 +278,24 @@ def test_all_effects(
         assert success is expected, reason
 
 
+@pytest.mark.parametrize('filename,expected', [
+    ['blank-a4-2pages.pdf', AttributeError],
+    ['single-image-a4.pdf', ValueError],
+    ['two-images-a4.pdf', ValueError],
+    ['flattened-a4-2pages.pdf', None]],
+    ids=['blank pdf', 'single image', 'two images', 'flattened pdf'])
+def test_image_extraction_pike(datadir, filename, expected):
+    file = os.path.join(datadir, filename)
+    with Pdf.open(file) as pdf_reader:
+        for pagenr in range(len(pdf_reader.pages)):
+            if expected is not None:
+                with pytest.raises(expected):
+                    scans.extract_image_pikepdf(pagenr, pdf_reader)
+            else:
+                img = scans.extract_image_pikepdf(pagenr, pdf_reader)
+                assert img is not None
+
+
 @pytest.mark.parametrize('filename', [
     'blank-a4-2pages.pdf',
     'flattened-a4-2pages.pdf'],

tests/test_three_corners.py

-import cv2
-import os
-import numpy as np
-import pytest
-
-from zesje.images import fix_corner_markers
-from zesje.scans import find_corner_marker_keypoints
-
-
-@pytest.mark.parametrize(
-    'shape,corners,expected',
-    [((240, 200, 3), [(50, 50), (120, 50), (50, 200)], (120, 200)),
-        ((240, 200, 3), [(120, 50), (50, 200), (120, 200)], (50, 50))],
-    ids=["", ""])
-def test_three_straight_corners(shape, corners, expected):
-    corner_markers = fix_corner_markers(corners, shape)
-    assert expected in corner_markers
-
-
-def test_pdf(datadir):
-    # Max deviation of inferred corner marker and actual location
-    epsilon = 2
-
-    # Scan rotated image with 4 corner markers
-    image_filename1 = 'a4-rotated.png'
-    image_path = os.path.join(datadir, 'cornermarkers', image_filename1)
-    page_img = cv2.imread(image_path)
-
-    corners1 = find_corner_marker_keypoints(page_img)
-
-    # Scan the same image with 3 corner markers
-    image_filename2 = 'a4-rotated-3-markers.png'
-    image_path = os.path.join(datadir, 'cornermarkers', image_filename2)
-    page_img = cv2.imread(image_path)
-
-    corners2 = find_corner_marker_keypoints(page_img)
-
-    # Get marker that was removed
-    diff = [corner for corner in corners1 if corner not in corners2]
-    diff_marker = min(diff)
-
-    fixed_corners2 = fix_corner_markers(corners2, page_img.shape)
-    added_marker = [corner for corner in fixed_corners2 if corner not in corners1][0]
-
-    # Check if 'inferred' corner marker is not too far away
-    dist = np.linalg.norm(np.subtract(added_marker, diff_marker))
-
-    assert dist < epsilon

zesje/api/exams.py

@@ -9,11 +9,9 @@ from flask_restful import Resource, reqparse
 from werkzeug.datastructures import FileStorage
 from sqlalchemy.orm import selectinload
 
-
-from ..pdf_generation import PAGE_FORMATS, generate_pdfs, output_pdf_filename_format
-from ..pdf_generation import join_pdfs, page_is_size, make_pages_even
-
-from ..database import db, Exam, ExamWidget, Submission
+from ..pdf_generation import generate_pdfs, output_pdf_filename_format, join_pdfs
+from ..pdf_generation import page_is_size, make_pages_even, PAGE_FORMATS
+from ..database import db, Exam, ExamWidget, Submission, token_length
 
 
 def _get_exam_dir(exam_id):
@@ -67,19 +65,7 @@ class Exams(Resource):
         elif Submission.query.filter(Submission.exam_id == exam.id).count():
             return dict(status=500, message='Exam is not finalized but already has submissions.'), 500
         else:
-            # Delete any scans that were wrongly uploaded to this exam
-            for scan in exam.scans:
-                db.session.delete(scan)
-
-            for widget in exam.widgets:
-                db.session.delete(widget)
-
-            for problem in exam.problems:
-                for fb_option in problem.feedback_options:
-                    db.session.delete(fb_option)
-                db.session.delete(problem.widget)
-                db.session.delete(problem)
-
+            # All corresponding solutions, scans and problems are automatically deleted
             db.session.delete(exam)
             db.session.commit()
 
@@ -533,8 +519,8 @@ class ExamPreview(Resource):
         cb_data = get_cb_data_for_exam(exam)
         generate_pdfs(
             exam_path,
-            exam.token[:5] + 'PREVIEW',
-            [1519],
+            "A" * token_length,
+            [1559],
             [output_file],
             student_id_widget.x, student_id_widget.y,
             barcode_widget.x, barcode_widget.y,

zesje/api/problems.py

 """ REST api for problems """
 
-from flask_restful import Resource, reqparse
+from flask_restful import Resource, reqparse, current_app
 
 from ..database import db, Exam, Problem, ProblemWidget, Solution
 
+from zesje.pdf_reader import get_problem_title
+
 
 class Problems(Resource):
     """ List of problems associated with a particular exam_id """
@@ -58,11 +60,18 @@ class Problems(Resource):
             db.session.commit()
             widget.name = f'problem_{problem.id}'
 
+            app_config = current_app.config
+            data_dir = app_config.get('DATA_DIRECTORY', 'data')
+            page_format = app_config.get('PAGE_FORMAT', 'A4')
+
+            problem.name = get_problem_title(problem, data_dir, page_format)
+
             db.session.commit()
 
             return {
                 'id': problem.id,
                 'widget_id': widget.id,
+                'problem_name': problem.name
             }
 
     put_parser = reqparse.RequestParser()
@@ -105,13 +114,11 @@ class Problems(Resource):
         if any([sol.graded_by is not None for sol in problem.solutions]):
             return dict(status=403, message=f'Problem has already been graded'), 403
         else:
-            # Delete all solutions associated with this problem
-            for sol in problem.solutions:
-                db.session.delete(sol)
-            # Delete all multiple choice options associated with this problem
+            # delete mc options
             for mc_option in problem.mc_options:
                 db.session.delete(mc_option)
-            db.session.delete(problem.widget)
+
+            # The widget and all associated solutions are automatically deleted
             db.session.delete(problem)
             db.session.commit()
             return dict(status=200, message="ok"), 200