Merge branch 'box-loc-db' into 'master'

Add multiple choice checkbox location to database

See merge request !5

.gitlab-ci.yml

@@ -16,6 +16,10 @@ stages:
   before_script:
     - yarn install --cache-folder .yarn-cache
 
+.python_packages: &python_packages
+  before_script:
+    - pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+
 build:
   <<: *node_modules
   stage: build
@@ -33,6 +37,7 @@ test_js:
   script: yarn test:js
 
 test_py:
+  <<: *python_packages
   stage: test
   script: yarn test:py
 
@@ -44,6 +49,7 @@ lint_js:
     - yarn lint:js
 
 lint_py:
+  <<: *python_packages
   stage: test
   allow_failure: true
   script:

README.md

@@ -64,6 +64,26 @@ You can run the tests by running
 
     yarn test
 
+#### Policy errors
+
+If you encounter PolicyErrors related to ImageMagick in any of the previous steps, please
+try the instructions listed
+[here](https://alexvanderbist.com/posts/2018/fixing-imagick-error-unauthorized) as
+a first resort.
+
+### Database modifications
+
+Zesje uses Flask-Migrate and Alembic for database versioning and migration. Flask-Migrate is an extension that handles SQLAlchemy database migrations for Flask applications using Alembic. 
+
+To change something in the database schema, simply add this change to `zesje/database.py`. After that run the following command to prepare a new migration:
+
+    yarn prepare-migration
+
+This uses Flask-Migrate to make a new migration script in `migrations/versions` which needs to be reviewed and edited. Please suffix the name of this file with something distinctive and add a short description at the top of the file. To apply the database migration run:
+
+    yarn migrate:dev # (for the development database)
+    yarn migrate # (for the production database)
+
 ### Building and running the production version
 
 

migrations/alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

migrations/env.py

+from __future__ import with_statement
+
+import logging
+import sys
+import os
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+
+from flask import current_app
+
+sys.path.append(os.getcwd())
+from zesje.database import db  # noqa: E402
+
+config = context.config
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+config.set_main_option('sqlalchemy.url',
+                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+
+target_metadata = db.metadata
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    engine = engine_from_config(config.get_section(config.config_ini_section),
+                                prefix='sqlalchemy.',
+                                poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(connection=connection,
+                      target_metadata=target_metadata,
+                      process_revision_directives=process_revision_directives,
+                      **current_app.extensions['migrate'].configure_args)
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    except Exception as exception:
+        logger.error(exception)
+        raise exception
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

migrations/script.py.mako

+""" ${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

migrations/versions/4204f4a83863_pony_to_sqlalchemy.py

+""" Pony or empty database to SQLAlchemy
+
+Revision ID: 4204f4a83863
+Revises:
+
+"""
+import shutil
+from alembic import op
+from flask import current_app
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '4204f4a83863'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    db_url = current_app.config.get('SQLALCHEMY_DATABASE_URI')
+    engine = sa.create_engine(db_url)
+    empty_database = not engine.dialect.has_table(engine, 'Exam')
+
+    if not empty_database:
+        # Make backup of sqlite file since no downgrade is supported
+        db_path = db_url.replace('sqlite:///', '')
+        shutil.copy2(db_path, db_path + '.pony')
+
+        # Remove old indices
+        op.drop_index('idx_scan__exam', table_name='Scan')
+        op.drop_index('idx_feedbackoption__problem', table_name='FeedbackOption')
+        op.drop_index('idx_submission__exam', table_name='Submission')
+        op.drop_index('idx_submission__student', table_name='Submission')
+        op.drop_index('idx_problem__exam', table_name='Problem')
+        op.drop_index('idx_problem__widget', table_name='Problem')
+        op.drop_index('idx_page__submission', table_name='Page')
+        op.drop_index('idx_widget__exam', table_name='Widget')
+        op.drop_index('idx_solution__graded_by', table_name='Solution')
+        op.drop_index('idx_solution__problem', table_name='Solution')
+        op.drop_index('idx_feedbackoption_solution', table_name='FeedbackOption_Solution')
+
+        # Temporarily prefix old table names with 'Pony'
+        table_names = [
+            'Exam',
+            'FeedbackOption',
+            'FeedbackOption_Solution',
+            'Grader',
+            'Page',
+            'Problem',
+            'Scan',
+            'Solution',
+            'Student',
+            'Submission',
+            'Widget',
+        ]
+        for table_name in table_names:
+            op.rename_table(table_name, 'Pony' + table_name)
+
+    # Create new tables
+    op.create_table(
+        'exam',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.Text(), nullable=False),
+        sa.Column('token', sa.String(length=12), nullable=True),
+        sa.Column('finalized', sa.Boolean(), server_default='f', nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('token')
+    )
+    op.create_table(
+        'grader',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.Text(), nullable=False),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'student',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('first_name', sa.Text(), nullable=False),
+        sa.Column('last_name', sa.Text(), nullable=False),
+        sa.Column('email', sa.Text(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('email')
+    )
+    op.create_table(
+        'widget',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.Text(), nullable=True),
+        sa.Column('x', sa.Integer(), nullable=False),
+        sa.Column('y', sa.Integer(), nullable=False),
+        sa.Column('type', sa.String(length=20), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'exam_widget',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('exam_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['exam_id'], ['exam.id'], ),
+        sa.ForeignKeyConstraint(['id'], ['widget.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'problem',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.Text(), nullable=False),
+        sa.Column('exam_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['exam_id'], ['exam.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'scan',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('exam_id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.Text(), nullable=False),
+        sa.Column('status', sa.Text(), nullable=False),
+        sa.Column('message', sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(['exam_id'], ['exam.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'submission',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('copy_number', sa.Integer(), nullable=False),
+        sa.Column('exam_id', sa.Integer(), nullable=False),
+        sa.Column('student_id', sa.Integer(), nullable=True),
+        sa.Column('signature_validated', sa.Boolean(), server_default='f', nullable=False),
+        sa.ForeignKeyConstraint(['exam_id'], ['exam.id'], ),
+        sa.ForeignKeyConstraint(['student_id'], ['student.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'feedback_option',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('problem_id', sa.Integer(), nullable=True),
+        sa.Column('text', sa.Text(), nullable=False),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('score', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['problem_id'], ['problem.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'page',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('path', sa.Text(), nullable=False),
+        sa.Column('submission_id', sa.Integer(), nullable=True),
+        sa.Column('number', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['submission_id'], ['submission.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'problem_widget',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('problem_id', sa.Integer(), nullable=False),
+        sa.Column('page', sa.Integer(), nullable=True),
+        sa.Column('width', sa.Integer(), nullable=True),
+        sa.Column('height', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['id'], ['widget.id'], ),
+        sa.ForeignKeyConstraint(['problem_id'], ['problem.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'solution',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('submission_id', sa.Integer(), nullable=False),
+        sa.Column('problem_id', sa.Integer(), nullable=False),
+        sa.Column('grader_id', sa.Integer(), nullable=True),
+        sa.Column('graded_at', sa.DateTime(), nullable=True),
+        sa.Column('remarks', sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(['grader_id'], ['grader.id'], ),
+        sa.ForeignKeyConstraint(['problem_id'], ['problem.id'], ),
+        sa.ForeignKeyConstraint(['submission_id'], ['submission.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'solution_feedback',
+        sa.Column('solution_id', sa.Integer(), nullable=False),
+        sa.Column('feedback_option_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['feedback_option_id'], ['feedback_option.id'], ),
+        sa.ForeignKeyConstraint(['solution_id'], ['solution.id'], ),
+        sa.PrimaryKeyConstraint('solution_id', 'feedback_option_id')
+    )
+
+    if not empty_database:
+        # Move data from old tables to new tables
+        # exam
+        op.execute('INSERT INTO exam (id, name, token, finalized) ' +
+                   'SELECT id, name, token, finalized FROM PonyExam;')
+
+        # widget
+        op.execute('INSERT INTO widget (id, name, x, y, type) ' +
+                   'SELECT id, name, x, y, CASE ' +
+                   'WHEN classtype = "ExamWidget" THEN "exam_widget" ' +
+                   'WHEN classtype = "ProblemWidget" THEN "problem_widget" ' +
+                   'END AS type FROM PonyWidget;')
+
+        # exam_widget
+        op.execute('INSERT INTO exam_widget (id, exam_id) ' +
+                   'SELECT id, exam FROM PonyWidget WHERE classtype = "ExamWidget"')
+
+        # problem_widget
+        op.execute('INSERT INTO problem_widget (id, problem_id, page, width, height) ' +
+                   'SELECT PonyWidget.id, PonyProblem.id, page, width, height FROM PonyWidget ' +
+                   'JOIN PonyProblem ON PonyWidget.id = PonyProblem.widget WHERE classtype = "ProblemWidget"')
+
+        # feedback_option
+        op.execute('INSERT INTO feedback_option (id, problem_id, text, description, score) ' +
+                   'SELECT id, problem, text, description, score FROM PonyFeedbackOption')
+
+        # grader
+        op.execute('INSERT INTO grader (id, name) ' +
+                   'SELECT id, name FROM PonyGrader')
+
+        # page
+        op.execute('INSERT INTO page (id, path, submission_id, number) ' +
+                   'SELECT id, path, submission, number FROM PonyPage')
+
+        # problem
+        op.execute('INSERT INTO problem (id, name, exam_id) ' +
+                   'SELECT id, name, exam FROM PonyProblem')
+
+        # scan
+        op.execute('INSERT INTO scan (id, exam_id, name, status, message)  ' +
+                   'SELECT id, exam, name, status, message FROM PonyScan')
+
+        # solution
+        op.execute('INSERT INTO solution (submission_id, problem_id, grader_id, graded_at, remarks) ' +
+                   'SELECT submission, problem, graded_by, graded_at, remarks FROM PonySolution')
+
+        # student
+        op.execute('INSERT INTO student (id, first_name, last_name, email) ' +
+                   'SELECT id, first_name, last_name, email FROM PonyStudent')
+
+        # submission
+        op.execute('INSERT INTO submission (id, copy_number, exam_id, student_id, signature_validated) ' +
+                   'SELECT id, copy_number, exam, student, signature_validated FROM PonySubmission')
+
+        # solution_feedback
+        op.execute('INSERT INTO solution_feedback (solution_id, feedback_option_id) ' +
+                   'SELECT solution.id, PonyFeedbackOption_Solution.feedbackoption ' +
+                   'FROM PonyFeedbackOption_Solution JOIN solution ON ' +
+                   'solution.submission_id = PonyFeedbackOption_Solution.solution_submission AND ' +
+                   'solution.problem_id = PonyFeedbackOption_Solution.solution_problem')
+
+        # Remove old tables
+        for table_name in table_names:
+            op.drop_table('Pony' + table_name)
+
+
+def downgrade():
+    # No support for downgrading to Pony
+    pass

migrations/versions/f97aa3c73453_.py

+""" empty message
+
+Revision ID: f97aa3c73453
+Revises: 4204f4a83863
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f97aa3c73453'
+down_revision = '4204f4a83863'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('mc_option',
+                    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+                    sa.Column('x', sa.Integer(), nullable=False),
+                    sa.Column('y', sa.Integer(), nullable=False),
+                    sa.Column('page', sa.Integer(), nullable=False),
+                    sa.Column('label', sa.String(), nullable=True),
+                    sa.Column('problem_id', sa.Integer(), nullable=False),
+                    sa.Column('feedback_id', sa.Integer(), nullable=True),
+                    sa.ForeignKeyConstraint(['feedback_id'], ['feedback_option.id'], ),
+                    sa.ForeignKeyConstraint(['problem_id'], ['solution.id'], ),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+
+
+def downgrade():
+    op.drop_table('mc_option')

package.json

@@ -3,7 +3,7 @@
   "main": "index.js",
   "license": "AGPL-3.0",
   "scripts": {
-    "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON\" --prefix-colors \"bgBlue.bold,bgGreen.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\"",
+    "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON,CELERY\" --prefix-colors \"bgBlue.bold,bgGreen.bold,bgRed.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg celery -A zesje.celery worker\"",
     "build": "webpack --config webpack.prod.js",
     "ci": "yarn lint; yarn test",
     "lint": "yarn lint:js; yarn lint:py",
@@ -13,7 +13,10 @@
     "lint:py": "flake8",
     "test:py": "python3 -m pytest -v -W error::RuntimeWarning",
     "start": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje",
-    "analyze": "webpack --config webpack.prod.js --profile --json > stats.json; webpack-bundle-analyzer stats.json zesje/static"
+    "analyze": "webpack --config webpack.prod.js --profile --json > stats.json; webpack-bundle-analyzer stats.json zesje/static",
+    "migrate:dev": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg FLASK_APP=zesje/__init__.py flask db upgrade",
+    "migrate": "FLASK_APP=zesje/__init__.py flask db upgrade",
+    "prepare-migration": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg FLASK_APP=zesje/__init__.py flask db migrate"
   },
   "standard": {
     "parser": "babel-eslint",

requirements.txt

 # Core components
 flask
 flask_restful
-pony
+flask_sqlalchemy
+sqlalchemy
+Flask-Migrate
+alembic
 pyyaml
+celery
+redis
 
 # General utilities
 numpy

tests/test_database.py

-import random
-
 import pytest
+from flask import Flask
+
+from zesje.database import db, Exam, _generate_exam_token
+
+
+@pytest.mark.parametrize('duplicate_count', [
+    0, 1],
+    ids=['No existing token', 'Existing token'])
+def test_exam_generate_token_length_uppercase(duplicate_count, monkeypatch):
+    class MockQuery:
+        def __init__(self):
+            self.duplicates = duplicate_count + 1
 
-from zesje.database import Exam, _generate_exam_token
+        def filter(self, *args):
+            return self
 
+        def first(self):
+            self.duplicates -= 1
+            return None if self.duplicates else True
 
-# I couldn't figure out how to make the mock return True on the first call and False on the second call. Therefore,
-# I just used randomness and run the test 50 times to make sure that that scenario occurs.
-@pytest.mark.parametrize('execution_number', range(50))
-def test_exam_generate_token_length_uppercase(execution_number, monkeypatch):
-    def mock_select_return(f):
-        class MockQuery:
-            def exists():
-                return random.choice([True, False])
-        return MockQuery
+    app = Flask(__name__, static_folder=None)
+    app.config.update(
+        SQLALCHEMY_DATABASE_URI='sqlite:///:memory:',
+        SQLALCHEMY_TRACK_MODIFICATIONS=False  # Suppress future deprecation warning
+    )
+    db.init_app(app)
 
-    monkeypatch.setattr(Exam, 'select', mock_select_return)
+    with app.app_context():
+        monkeypatch.setattr(Exam, 'query', MockQuery())
 
-    id = _generate_exam_token()
-    assert len(id) == 12
-    assert id.isupper()
+        id = _generate_exam_token()
+        assert len(id) == 12
+        assert id.isupper()

tests/test_scans.py

@@ -4,7 +4,7 @@ import pytest
 import numpy as np
 import PIL.Image
 from tempfile import NamedTemporaryFile
-from pony.orm import db_session
+from flask import Flask
 from io import BytesIO
 import wand.image
 
@@ -27,21 +27,22 @@ def mock_get_box_return_original(monkeypatch, datadir):
 # Module scope ensures it is ran only once
 @pytest.fixture(scope="module")
 def db_setup():
-    try:
-        db.bind('sqlite', ':memory:')
-    except TypeError:
-        pass
-    else:
-        db.generate_mapping(check_tables=False)
-    db.drop_all_tables(with_all_data=True)
-    db.create_tables()
+    app = Flask(__name__, static_folder=None)
+    app.config.update(
+        SQLALCHEMY_DATABASE_URI='sqlite:///:memory:',
+        SQLALCHEMY_TRACK_MODIFICATIONS=False  # Suppress future deprecation warning
+    )
+    db.init_app(app)
+    return app
 
 
 # Fixture which empties the database
 @pytest.fixture
 def db_empty(db_setup):
-    db.drop_all_tables(with_all_data=True)
-    db.create_tables()
+    with db_setup.app_context():
+        db.drop_all()
+        db.create_all()
+    return db_setup
 
 
 # Tests whether the output of calc angle is correct
@@ -96,15 +97,20 @@ def new_exam(db_empty):
     This needs to be ran at the start of every pipeline test
     TODO: rewrite to a fixture
     """
-    with db_session:
+    with db_empty.app_context():
         token = _generate_exam_token()
         e = Exam(name="testExam", token=token)
-        Submission(copy_number=145, exam=e)
-        ExamWidget(exam=e, name='student_id_widget', x=0, y=0)
+        sub = Submission(copy_number=145, exam=e)
+        widget = ExamWidget(exam=e, name='student_id_widget', x=0, y=0)
         exam_config = ExamMetadata(
             token=token,
             barcode_coords=[40, 90, 510, 560],  # in points (not pixels!)
         )
+        db.session.add_all([e, sub, widget])
+        db.session.commit()
+
+    # Push the current app context for all tests so the database can be used
+    db_empty.app_context().push()
     return exam_config
 
 

zesje/__init__.py

 """ Init file that starts a Flask dev server and opens db """
 
 import os
-from os.path import abspath, dirname
 
-from flask import Flask
 from werkzeug.exceptions import NotFound
 
+from .factory import create_app, make_celery
 from .api import api_bp
-from .database import db
+
 from ._version import __version__
 
 
 __all__ = ['__version__', 'app']
 
-STATIC_FOLDER_PATH = os.path.join(abspath(dirname(__file__)), 'static')
+app = create_app()
 
-app = Flask(__name__, static_folder=STATIC_FOLDER_PATH)
 app.register_blueprint(api_bp, url_prefix='/api')
 
-if 'ZESJE_SETTINGS' in os.environ:
-    app.config.from_envvar('ZESJE_SETTINGS')
-
-# Default settings
-app.config.update(
-    DATA_DIRECTORY=abspath(app.config.get('DATA_DIRECTORY', 'data')),
-)
-
-# These reference DATA_DIRECTORY, so they need to be in a separate update
-app.config.update(
-    SCAN_DIRECTORY=os.path.join(app.config['DATA_DIRECTORY'], 'scans'),
-    DB_PATH=os.path.join(app.config['DATA_DIRECTORY'], 'course.sqlite'),
-)
-
-
-@app.before_first_request
-def setup():
-    os.makedirs(app.config['DATA_DIRECTORY'], exist_ok=True)
-    os.makedirs(app.config['SCAN_DIRECTORY'], exist_ok=True)
+os.makedirs(app.config['DATA_DIRECTORY'], exist_ok=True)
+os.makedirs(app.config['SCAN_DIRECTORY'], exist_ok=True)
 
-    db.bind('sqlite', app.config['DB_PATH'], create_db=True)
-    db.generate_mapping(create_tables=True)
+celery = make_celery(app)
 
 
 @app.route('/')

zesje/api/__init__.py

@@ -11,6 +11,7 @@ from .feedback import Feedback
 from .solutions import Solutions
 from .widgets import Widgets
 from .emails import EmailTemplate, RenderedEmailTemplate, Email
+from .mult_choice import MultipleChoice
 from . import signature
 from . import images
 from . import summary_plot
@@ -18,14 +19,7 @@ from . import export
 
 api_bp = Blueprint(__name__, __name__)
 
-errors = {
-    'ObjectNotFound': {
-        'status': 404,
-        'message': 'Resource with that ID does not exist',
-     },
-}
-
-api = Api(api_bp, errors=errors)
+api = Api(api_bp)
 
 api.add_resource(Graders, '/graders')
 api.add_resource(Exams, '/exams', '/exams/<int:exam_id>', '/exams/<int:exam_id>/<string:attr>')
@@ -55,6 +49,7 @@ api.add_resource(RenderedEmailTemplate,
 api.add_resource(Email,
                  '/email/<int:exam_id>',
                  '/email/<int:exam_id>/<int:student_id>')
+api.add_resource(MultipleChoice, '/mult-choice/<int:id>')
 
 
 # Other resources that don't return JSON

zesje/api/emails.py

@@ -8,8 +8,6 @@ import werkzeug.exceptions
 from flask import current_app as app
 from flask_restful import Resource, reqparse
 
-from pony import orm
-
 from .. import emails
 from ..database import Exam, Student
 from ._helpers import abort
@@ -59,8 +57,12 @@ def render_email(exam_id, student_id, template):
 
 
 def build_email(exam_id, student_id, template, attach, from_address, copy_to=None):
-    with orm.db_session:
-        student = Student[student_id]
+    student = Student.query.get(student_id)
+    if student is None:
+        abort(
+            404,
+            message=f"Student #{student_id} does not exist"
+        )
     if not student.email:
         abort(
             409,
@@ -145,13 +147,19 @@ class Email(Resource):
                 message="Not CC-ing all emails from the exam."
             )
 
-        with orm.db_session:
-            if not all(Exam[exam_id].submissions.signature_validated):
-                abort(
-                    409,
-                    message="All submissions must be validated before "
-                            "sending emails."
-                )
+        exam = Exam.query.get(exam_id)
+        if exam is None:
+            abort(
+                404,
+                message="Exam does not exist"
+            )
+
+        if not all(sub.signature_validated for sub in exam.submissions):
+            abort(
+                409,
+                message="All submissions must be validated before "
+                        "sending emails."
+            )
 
         if student_id is not None:
             return self._send_single(exam_id, student_id, template, attach, copy_to)
@@ -190,8 +198,13 @@ class Email(Resource):
                 500,
                 message='Sending email is not configured'
             )
-        with orm.db_session:
-            student_ids = set(Exam[exam_id].submissions.student.id)
+        exam = Exam.query.get(exam_id)
+        if exam is None:
+            abort(
+                404,
+                message="Exam does not exist"
+            )
+        student_ids = set(sub.student.id for sub in exam.submissions if sub.student)
 
         failed_to_build = list()
         to_send = dict()

zesje/api/exams.py

@@ -7,11 +7,10 @@ from tempfile import TemporaryFile
 from flask import current_app as app, send_file, request
 from flask_restful import Resource, reqparse
 from werkzeug.datastructures import FileStorage
-
-from pony import orm
+from sqlalchemy.orm import selectinload
 
 from ..pdf_generation import generate_pdfs, output_pdf_filename_format, join_pdfs, page_is_size
-from ..database import db, Exam, ExamWidget
+from ..database import db, Exam, ExamWidget, Submission
 
 PAGE_FORMATS = {
     "A4": (595.276, 841.89),
@@ -34,9 +33,8 @@ class Exams(Resource):
         else:
             return self._get_all()
 
-    @orm.db_session
     def delete(self, exam_id):
-        exam = Exam.get(id=exam_id)
+        exam = Exam.query.get(exam_id)
         if exam is None:
             return dict(status=404, message='Exam does not exist.'), 404
         elif exam.finalized:
@@ -44,7 +42,6 @@ class Exams(Resource):
         else:
             exam.delete()
 
-    @orm.db_session
     def _get_all(self):
         """get list of uploaded exams.
 
@@ -62,12 +59,11 @@ class Exams(Resource):
             {
                 'id': ex.id,
                 'name': ex.name,
-                'submissions': ex.submissions.count()
+                'submissions': len(ex.submissions)
             }
-            for ex in Exam.select().order_by(Exam.id)
+            for ex in Exam.query.order_by(Exam.id).all()
         ]
 
-    @orm.db_session
     def _get_single(self, exam_id):
         """Get detailed information about a single exam
 
@@ -89,7 +85,9 @@ class Exams(Resource):
         widgets
             list of widgets in this exam
         """
-        exam = Exam[exam_id]
+        # Load exam using the following most efficient strategy
+        exam = Exam.query.options(selectinload(Exam.submissions).
+                                  subqueryload(Submission.solutions)).get(exam_id)
 
         if exam is None:
             return dict(status=404, message='Exam does not exist.'), 404
@@ -115,8 +113,8 @@ class Exams(Resource):
                             'feedback': [
                                 fb.id for fb in sol.feedback
                             ],
-                            'remark': sol.remarks
-                        } for sol in sub.solutions.order_by(lambda s: s.problem.id)
+                            'remark': sol.remarks if sol.remarks else ""
+                        } for sol in sub.solutions  # Sorted by sol.problem_id
                     ],
                 } for sub in exam.submissions
         ]
@@ -142,10 +140,10 @@ class Exams(Resource):
                             'name': fb.text,
                             'description': fb.description,
                             'score': fb.score,
-                            'used': fb.solutions.count()
+                            'used': len(fb.solutions)
                         }
                         for fb
-                        in prob.feedback_options.order_by(lambda f: f.id)
+                        in prob.feedback_options  # Sorted by fb.id
                     ],
                     'page': prob.widget.page,
                     'widget': {
@@ -157,7 +155,7 @@ class Exams(Resource):
                         'height': prob.widget.height,
                     },
                     'graded': any([sol.graded_by is not None for sol in prob.solutions])
-                } for prob in exam.problems.order_by(lambda p: p.id)
+                } for prob in exam.problems  # Sorted by prob.id
             ],
             'widgets': [
                 {
@@ -165,7 +163,7 @@ class Exams(Resource):
                     'name': widget.name,
                     'x': widget.x,
                     'y': widget.y,
-                } for widget in exam.widgets.order_by(lambda w: w.id)
+                } for widget in exam.widgets  # Sorted by widget.id
             ],
             'finalized': exam.finalized,
         }
@@ -174,7 +172,6 @@ class Exams(Resource):
     post_parser.add_argument('pdf', type=FileStorage, required=True, location='files')
     post_parser.add_argument('exam_name', type=str, required=True, location='form')
 
-    @orm.db_session
     def post(self):
         """Add a new exam.
 
@@ -223,7 +220,8 @@ class Exams(Resource):
             ),
         ]
 
-        db.commit()  # so exam gets an id
+        db.session.add(exam)
+        db.session.commit()  # so exam gets an id
 
         exam_dir = _get_exam_dir(exam.id)
         pdf_path = os.path.join(exam_dir, 'exam.pdf')
@@ -238,13 +236,16 @@ class Exams(Resource):
             'id': exam.id
         }
 
-    @orm.db_session
     def put(self, exam_id, attr):
         if attr == 'finalized':
-            exam = Exam[exam_id]
+            exam = Exam.query.get(exam_id)
+            if exam is None:
+                return dict(status=404, message='Exam does not exist.'), 404
+
             bodyStr = request.data.decode('utf-8')
             if bodyStr == 'true':
                 exam.finalized = True
+                db.session.commit()
             elif bodyStr == 'false':
                 if exam.finalized:
                     return dict(status=403, message=f'Exam already finalized'), 403
@@ -257,10 +258,12 @@ class Exams(Resource):
 
 class ExamSource(Resource):
 
-    @orm.db_session
     def get(self, exam_id):
 
-        exam = Exam[exam_id]
+        exam = Exam.query.get(exam_id)
+
+        if exam is None:
+            return dict(status=404, message='Exam does not exist.'), 404
 
         exam_dir = _get_exam_dir(exam.id)
 
@@ -323,7 +326,6 @@ class ExamGeneratedPdfs(Resource):
     post_parser.add_argument('copies_start', type=int, required=True)
     post_parser.add_argument('copies_end', type=int, required=True)
 
-    @orm.db_session
     def post(self, exam_id):
         """Generates the exams with corner markers and Widgets.
 
@@ -351,10 +353,9 @@ class ExamGeneratedPdfs(Resource):
         copies_start = args.get('copies_start')
         copies_end = args.get('copies_end')
 
-        try:
-            exam = Exam[exam_id]
-        except orm.ObjectNotFound:
-            return dict(status=404, message=f'Exam not found'), 404
+        exam = Exam.query.get(exam_id)
+        if exam is None:
+            return dict(status=404, message='Exam does not exist.'), 404
 
         exam_dir = _get_exam_dir(exam_id)
         generated_pdfs_dir = self._get_generated_exam_dir(exam_dir)
@@ -389,17 +390,15 @@ class ExamGeneratedPdfs(Resource):
     get_parser.add_argument('copies_end', type=int, required=True)
     get_parser.add_argument('type', type=str, required=True)
 
-    @orm.db_session
     def get(self, exam_id):
         args = self.get_parser.parse_args()
 
         copies_start = args['copies_start']
         copies_end = args['copies_end']
 
-        try:
-            exam = Exam[exam_id]
-        except orm.ObjectNotFound:
-            return dict(status=404, message=f'Exam not found'), 404
+        exam = Exam.query.get(exam_id)
+        if exam is None:
+            return dict(status=404, message='Exam does not exist.'), 404
 
         exam_dir = _get_exam_dir(exam_id)
         generated_pdfs_dir = self._get_generated_exam_dir(exam_dir)
@@ -445,12 +444,10 @@ class ExamGeneratedPdfs(Resource):
 
 class ExamPreview(Resource):
 
-    @orm.db_session
     def get(self, exam_id):
-        try:
-            exam = Exam[exam_id]
-        except orm.ObjectNotFound:
-            return dict(status=404, message=f'Exam not found'), 404
+        exam = Exam.query.get(exam_id)
+        if exam is None:
+            return dict(status=404, message='Exam does not exist.'), 404
 
         output_file = BytesIO()
 

zesje/api/export.py

 from io import BytesIO
 
-from flask import abort, send_file, send_from_directory, current_app as app
+from flask import abort, send_file, current_app as app
 
 from ..statistics import full_exam_data
 
@@ -13,9 +13,8 @@ def full():
     response : flask Response
         response containing the ``course.sqlite``
     """
-    return send_from_directory(
-        app.config['DATA_DIRECTORY'],
-        'course.sqlite',
+    return send_file(
+        app.config['DB_PATH'],
         as_attachment=True,
         mimetype="application/x-sqlite3",
         cache_timeout=0,
@@ -52,7 +51,7 @@ def exam(file_format, exam_id):
     if file_format == 'dataframe':
         extension = 'pd'
         mimetype = 'application/python-pickle'
-        data.to_pickle(serialized)
+        data.to_pickle(serialized, compression=None)
     else:
         extension = 'xlsx'
         mimetype = (

zesje/api/feedback.py

@@ -2,15 +2,12 @@
 
 from flask_restful import Resource, reqparse
 
-from pony import orm
-
-from ..database import Problem, FeedbackOption, Solution
+from ..database import db, Problem, FeedbackOption, Solution
 
 
 class Feedback(Resource):
     """ List of feedback options of a problem """
 
-    @orm.db_session
     def get(self, problem_id):
         """get list of feedback connected to a specific problem
 
@@ -24,7 +21,9 @@ class Feedback(Resource):
             used: int
         """
 
-        problem = Problem[problem_id]
+        problem = Problem.query.get(problem_id)
+        if problem is None:
+            return dict(status=404, message=f"Problem with id #{problem_id} does not exist"), 404
 
         return [
             {
@@ -32,9 +31,9 @@ class Feedback(Resource):
                 'name': fb.text,
                 'description': fb.description,
                 'score': fb.score,
-                'used': fb.solutions.count()
+                'used': len(fb.solutions)
             }
-            for fb in FeedbackOption.select(lambda fb: fb.problem == problem)
+            for fb in FeedbackOption.query.filter(FeedbackOption.problem == problem)
         ]
 
     post_parser = reqparse.RequestParser()
@@ -42,7 +41,6 @@ class Feedback(Resource):
     post_parser.add_argument('description', type=str, required=False)
     post_parser.add_argument('score', type=int, required=False)
 
-    @orm.db_session
     def post(self, problem_id):
         """Post a new feedback option
 
@@ -53,12 +51,15 @@ class Feedback(Resource):
             score: int
         """
 
-        problem = Problem[problem_id]
+        problem = Problem.query.get(problem_id)
+        if problem is None:
+            return dict(status=404, message=f"Problem with id #{problem_id} does not exist"), 404
 
         args = self.post_parser.parse_args()
 
         fb = FeedbackOption(problem=problem, text=args.name, description=args.description, score=args.score)
-        orm.commit()
+        db.session.add(fb)
+        db.session.commit()
 
         return {
             'id': fb.id,
@@ -73,7 +74,6 @@ class Feedback(Resource):
     put_parser.add_argument('description', type=str, required=False)
     put_parser.add_argument('score', type=int, required=False)
 
-    @orm.db_session
     def put(self, problem_id):
         """Modify an existing feedback option
 
@@ -87,9 +87,15 @@ class Feedback(Resource):
 
         args = self.put_parser.parse_args()
 
-        fb = FeedbackOption.get(id=args.id)
-        if fb:
-            fb.set(text=args.name, description=args.description, score=args.score)
+        fb = FeedbackOption.query.get(args.id)
+        if fb is None:
+            return dict(status=404, message=f"Feedback option with id #{args.id} does not exist"), 404
+
+        fb.text = args.name
+        fb.description = args.description
+        fb.score = args.score
+
+        db.session.commit()
 
         return {
             'id': fb.id,
@@ -98,7 +104,6 @@ class Feedback(Resource):
             'score': fb.score
         }
 
-    @orm.db_session
     def delete(self, problem_id, feedback_id):
         """Delete an existing feedback option
 
@@ -114,20 +119,22 @@ class Feedback(Resource):
         We use the problem id for extra safety check that we don't corrupt
         things accidentally.
         """
-        fb = FeedbackOption.get(id=feedback_id)
-        problem = fb.problem
+        fb = FeedbackOption.query.get(feedback_id)
         if fb is None:
-            return dict(status=404, message="Feedback with this id does not exist"), 404
-        elif problem.id != problem_id:
+            return dict(status=404, message=f"Feedback option with id #{feedback_id} does not exist"), 404
+        problem = fb.problem
+        if problem.id != problem_id:
             return dict(status=409, message="Feedback does not match the problem."), 409
-        else:
-            fb.delete()
+
+        db.session.delete(fb)
 
         # If there are submissions with no feedback, we should mark them as
         # ungraded.
-        to_mark_ungraded = Solution.select(
-            lambda s: s.problem == problem and not len(s.feedback) and
-            s.graded_at is not None
-        )
-        for solution in to_mark_ungraded:
-            solution.graded_at = solution.graded_by = None
+        solutions = Solution.query.filter(Solution.problem_id == problem_id,
+                                          Solution.grader_id is not None).all()
+        for solution in solutions:
+            if solution.feedback_count == 0:
+                solution.grader_id = None
+                solution.graded_at = None
+
+        db.session.commit()

zesje/api/graders.py

@@ -3,7 +3,7 @@
 from flask import abort
 from flask_restful import Resource, reqparse
 
-from pony import orm
+from ..database import db
 
 from ._helpers import required_string
 from ..database import Grader
@@ -15,7 +15,6 @@ from ..database import Grader
 class Graders(Resource):
     """ Graders that are able to use the software, also logged during grading """
 
-    @orm.db_session
     def get(self):
         """get all graders.
 
@@ -30,13 +29,12 @@ class Graders(Resource):
                 'id': g.id,
                 'name': g.name
             }
-            for g in Grader.select()
+            for g in Grader.query.all()
         ]
 
     post_parser = reqparse.RequestParser()
     required_string(post_parser, 'name')
 
-    @orm.db_session
     def post(self):
         """add a grader.
 
@@ -53,8 +51,8 @@ class Graders(Resource):
         args = self.post_parser.parse_args()
 
         try:
-            Grader(name=args['name'])
-            orm.commit()
+            db.session.add(Grader(name=args['name']))
+            db.session.commit()
         except KeyError as error:
             abort(400, error)
 

zesje/api/images.py

 from flask import abort, Response
 
-from pony import orm
-
 import numpy as np
 import cv2
 
@@ -9,7 +7,6 @@ from ..images import get_box
 from ..database import Exam, Submission, Problem, Page
 
 
-@orm.db_session
 def get(exam_id, problem_id, submission_id, full_page=False):
     """get image for the given problem.
 
@@ -27,12 +24,18 @@ def get(exam_id, problem_id, submission_id, full_page=False):
     -------
     Image (JPEG mimetype)
     """
-    try:
-        exam = Exam[exam_id]
-        submission = Submission.get(exam=exam, copy_number=submission_id)
-        problem = Problem[problem_id]
-    except (KeyError, ValueError):
-        abort(404)
+    exam = Exam.query.get(exam_id)
+    if exam is None:
+        abort(404, 'Exam does not exist.')
+
+    problem = Problem.query.get(problem_id)
+    if problem is None:
+        abort(404, 'Problem does not exist.')
+
+    sub = Submission.query.filter(Submission.exam_id == exam.id,
+                                  Submission.copy_number == submission_id).one_or_none()
+    if sub is None:
+        abort(404, 'Submission does not exist.')
 
     widget_area = np.asarray([
         problem.widget.y,  # top
@@ -45,7 +48,12 @@ def get(exam_id, problem_id, submission_id, full_page=False):
     widget_area_in = widget_area / 72
 
     #  get the page
-    page_path = Page.get(submission=submission, number=problem.widget.page).path
+    page = Page.query.filter(Page.submission_id == sub.id, Page.number == problem.widget.page).first()
+
+    if page is None:
+        abort(404, f'Page #{problem.widget.page} is missing for copy #{submission_id}.')
+
+    page_path = page.path
 
     page_im = cv2.imread(page_path)
     if not full_page: