diff --git a/.gitignore b/.gitignore
index e5e7d5c88021dc5025a3b76c80f9a00e5b1f7f99..f2904ee0ce6fae47dda031658f26062a275604bf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,8 +89,19 @@ build/
# development data
data-dev
+# test data
+tests/data/submissions
+
+# redis dump
+dump.rdb
+
# webpack analyze data
stats.json
# JetBrains IDE folders
.idea/
+
+# pytest coverage reports
+.coverage
+cov.xml
+cov.html/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00a36093c8acd358c7f35e9199689550973372e2..3dd7a4f92e594733aa4de766ecda691c86a82601 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,5 @@
-
# This base image can be found in 'Dockerfile'
-image: zesje/base
+image: gitlab.kwant-project.org:5005/zesje/zesje/test:latest
stages:
- build
@@ -14,11 +13,13 @@ stages:
paths:
- .yarn-cache
before_script:
+ - source activate zesje-dev
- yarn install --cache-folder .yarn-cache
.python_packages: &python_packages
before_script:
- - pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+ - source activate zesje-dev
+ - conda env update
build:
<<: *node_modules
@@ -36,11 +37,6 @@ test_js:
stage: test
script: yarn test:js
-test_py:
- <<: *python_packages
- stage: test
- script: yarn test:py
-
lint_js:
<<: *node_modules
stage: test
@@ -54,3 +50,13 @@ lint_py:
allow_failure: true
script:
- yarn lint:py
+
+test_py:
+ <<: *python_packages
+ stage: test
+ script:
+ - yarn test:py:cov
+ artifacts:
+ paths:
+ - cov.html/
+ expire_in: 1 week
diff --git a/AUTHORS.md b/AUTHORS.md
index 1b470d040e35f336136f9efa9f1fde981c77d570..2cf605693e74665cac0720f82b93d3320f3c65ec 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -6,3 +6,15 @@
* Justin van der Krieken
* Jamy Mahabier
* Nick Cleintuar
+* Hugo Kerstens
+* Stefan Hugtenburg
+* Hidde Leistra
+* Pim Otte
+* Luc Enthoven
+
+<!--
+Execute
+git shortlog -s | sed -e "s/^ *[0-9\t ]*//"| xargs -i sh -c 'grep -q "{}" AUTHORS.md || echo "{}"'
+
+To check if any authors are missing from this list.
+ -->
diff --git a/Dockerfile b/Dockerfile
index d8b2a61a1a6abc723efa5ed86cb7eeec37290fa5..a31c6c6c81a23c7ec16d87236f0fc68227b6bf8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,15 +1,24 @@
-FROM archlinux/base
+FROM continuumio/miniconda3
-## Install packages and clear the cache after installation. Yarn is fixed at 1.6.0 untill 1.8.0 is released due to a critical bug.
-RUN pacman -Sy --noconfirm nodejs python-pip git libdmtx libsm libxrender libxext gcc libmagick6 imagemagick ghostscript; \
- pacman -U --noconfirm https://archive.archlinux.org/packages/y/yarn/yarn-1.6.0-1-any.pkg.tar.xz
+RUN apt-get update -y && apt-get install -y libdmtx0a libmagickwand-dev
-WORKDIR ~
-ADD requirements*.txt ./
-#ADD package.json .
-RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt;
-#RUN yarn install; \
-# yarn cache clean; \
-# rm package.json
+RUN apt-get update && \
+ apt-get install -y \
+ curl \
+ poppler-utils build-essential libgl1-mesa-glx \
+ imagemagick libsm-dev libdmtx-dev libdmtx0a libmagickwand-dev \
+ && \
+ apt-get -y --quiet install git supervisor nginx
-CMD bash
\ No newline at end of file
+WORKDIR /app
+
+ADD environment.yml /app/environment.yml
+RUN conda env create
+
+# From https://medium.com/@chadlagore/conda-environments-with-docker-82cdc9d25754
+RUN echo "source activate $(head -1 /app/environment.yml | cut -d' ' -f2)" > ~/.bashrc
+ENV PATH /opt/conda/envs/$(head -1 /app/environment.yml | cut -d' ' -f2)/bin:$PATH
+
+RUN rm /app/environment.yml
+
+CMD bash
diff --git a/README.md b/README.md
index 1155b27641ad15059efc00aa53c26cbfa4241072..eb20409b37e4df36944a5d15108c78309c5bd97e 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+[](https://gitlab.kwant-project.org/zesje/zesje/commits/master)
+
# Welcome to Zesje
Zesje is an online grading system for written exams.
@@ -13,10 +15,11 @@ Install Miniconda by following the instructions on this page:
https://conda.io/miniconda.html
-Create a Conda environment that you will use for installing all
-of zesje's dependencies:
+Make sure you cloned this repository and enter its directory. Then
+create a Conda environment that will automatically install all
+of zesje's Python dependencies:
- conda create -c conda-forge -n zesje-dev python=3.6 yarn
+ conda env create # Creates an environment from environment.yml
Then, *activate* the conda environment:
@@ -29,10 +32,6 @@ Install all of the Javascript dependencies:
yarn install
-Install all of the Python dependencies:
-
- pip install -r requirements.txt -r requirements-dev.txt
-
Unfortunately there is also another dependency that must be installed
manually for now (we are working to bring this dependency into the
Conda ecosystem). You can install this dependency in the following way
@@ -63,6 +62,33 @@ or `zesje/`.
You can run the tests by running
yarn test
+
+#### Viewing test coverage
+
+As a test coverage tool for Python tests, `pytest-cov` is used.
+
+To view test coverage, run
+
+ yarn test:py:cov
+
+A coverage report is now generated in the terminal, as an XML file, and in HTML format.
+The HTML file shows an overview of untested code in red.
+
+##### Viewing coverage in Visual Studio Code
+
+There is a plugin called Coverage Gutter that will highlight which lines of code are covered.
+Simply install Coverage Gutter, after which a watch button appears in the colored box at the bottom of your IDE.
+When you click watch, green and red lines appear next to the line numbers indicating if the code is covered.
+
+Coverage Gutter uses the XML which is produced by `yarn test:py:cov`, called `cov.xml`. This file should be located in the main folder.
+
+##### Viewing coverage in PyCharm
+To view test coverage in PyCharm, run `yarn test:py:cov` to generate the coverage report XML file `cov.xml` if it is not present already.
+
+Next, open up PyCharm and in the top bar go to **Run -> Show Code Coverage Data** (Ctrl + Alt + F6).
+
+Press **+** and add the file `cov.xml` that is in the main project directory.
+A code coverage report should now appear in the side bar on the right.
#### Policy errors
@@ -116,10 +142,10 @@ If you use Atom, install the [linter-js-standard-engine](https://atom.io/package
### Adding dependencies
#### Server-side
-If you start using a new Python library, be sure to add it to `requirements.txt`. Python libraries for the testing are in `requirements-dev.txt`.
-The packages can be installed and updated in your environment by `pip` using
+If you start using a new Python library, be sure to add it to `environment.yml`.
+The packages can be installed and updated in your environment by `conda` using
- pip install -r requirements.txt -r requirements-dev.txt
+ conda env update
#### Client-side
diff --git a/barcode_example_generator.py b/barcode_example_generator.py
index 79a33c221a61a5f1b2d84eb7875defb84c11a277..3d338a909978a41ff1645d86a7b2f44340356b00 100644
--- a/barcode_example_generator.py
+++ b/barcode_example_generator.py
@@ -1,34 +1,37 @@
+
+import sys
+import os
from io import BytesIO
from reportlab.pdfgen import canvas
-import PIL
from wand.image import Image
from wand.color import Color
-from pystrich.datamatrix import DataMatrixEncoder
+sys.path.append(os.getcwd())
-def generate_datamatrix(exam_id, page_num, copy_num):
- data = f'{exam_id}/{copy_num:04d}/{page_num:02d}'
+from zesje.pdf_generation import generate_datamatrix # noqa: E402
+from zesje.database import token_length # noqa: E402
- image_bytes = DataMatrixEncoder(data).get_imagedata(cellsize=2)
- return PIL.Image.open(BytesIO(image_bytes))
+exam_token = "A" * token_length
+copy_num = 1559
+page_num = 0
-datamatrix = generate_datamatrix(0, 0, 0)
-datamatrix_x = datamatrix_y = 0
-fontsize = 8
-margin = 3
+fontsize = 12
+datamatrix_x = 0
+datamatrix_y = fontsize
-datamatrix = generate_datamatrix(0, 0, 0000)
-imagesize = (datamatrix.width, 3 + fontsize + datamatrix.height)
+datamatrix = generate_datamatrix(exam_token, page_num, copy_num)
+imagesize = (datamatrix.width, fontsize + datamatrix.height)
result_pdf = BytesIO()
canv = canvas.Canvas(result_pdf, pagesize=imagesize)
-canv.drawInlineImage(datamatrix, 0, 3 + fontsize)
+canv.drawInlineImage(datamatrix, datamatrix_x, datamatrix_y)
canv.setFont('Helvetica', fontsize)
-canv.drawString(0, 3, f" # 1519")
+canv.drawString(datamatrix_x, datamatrix_y - (fontsize * 0.66),
+ f" # {copy_num}")
canv.showPage()
canv.save()
@@ -36,7 +39,7 @@ canv.save()
result_pdf.seek(0)
# From https://stackoverflow.com/questions/27826854/python-wand-convert-pdf-to-png-disable-transparent-alpha-channel
-with Image(file=result_pdf, resolution=80) as img:
- with Image(width=img.width, height=img.height, background=Color("white")) as bg:
+with Image(file=result_pdf, resolution=72) as img:
+ with Image(width=imagesize[0], height=imagesize[1], background=Color("white")) as bg:
bg.composite(img, 0, 0)
bg.save(filename="client/components/barcode_example.png")
diff --git a/client/components/Hero.jsx b/client/components/Hero.jsx
index 0b60fba63dbff092c83e8457777f5a8be31a9dac..62e6b696beda295eade8e932eacfed03b7358fff 100644
--- a/client/components/Hero.jsx
+++ b/client/components/Hero.jsx
@@ -2,7 +2,7 @@ import React from 'react'
const Hero = (props) => {
return (
- <section className='hero is-primary is-info'>
+ <section className='hero is-primary is-info is-small'>
<div className='hero-body'>
<div className='container'>
<h1 className='title'>
diff --git a/client/components/barcode_example.png b/client/components/barcode_example.png
index ab3b2d57e3e12e9f96e8501a36085787208a9634..fa6b7ceb68015b4a5cd038577d5d9cc320b72609 100644
Binary files a/client/components/barcode_example.png and b/client/components/barcode_example.png differ
diff --git a/client/views/grade/EditPanel.jsx b/client/components/feedback/EditPanel.jsx
similarity index 84%
rename from client/views/grade/EditPanel.jsx
rename to client/components/feedback/EditPanel.jsx
index 9683af57f759431c7d8fbf1554dd374348c57e0f..63eb950ab7b52b8f1f9cd6bb810675d5ab001638 100644
--- a/client/views/grade/EditPanel.jsx
+++ b/client/components/feedback/EditPanel.jsx
@@ -40,16 +40,19 @@ class EditPanel extends React.Component {
}
static getDerivedStateFromProps (nextProps, prevState) {
+ // In case nothing is set, use an empty function that no-ops
+ const updateCallback = nextProps.updateCallback || (_ => {})
if (nextProps.feedback && prevState.id !== nextProps.feedback.id) {
const fb = nextProps.feedback
return {
id: fb.id,
name: fb.name,
description: fb.description,
- score: fb.score
+ score: fb.score,
+ updateCallback: updateCallback
}
}
- return null
+ return {updateCallback: updateCallback}
}
changeText = (event) => {
@@ -84,10 +87,15 @@ class EditPanel extends React.Component {
if (this.state.id) {
fb.id = this.state.id
api.put(uri, fb)
- .then(() => this.props.goBack())
+ .then(() => {
+ this.state.updateCallback(fb)
+ this.props.goBack()
+ })
} else {
api.post(uri, fb)
- .then(() => {
+ .then((response) => {
+ // Response is the feedback option
+ this.state.updateCallback(response)
this.setState({
id: null,
name: '',
@@ -101,14 +109,20 @@ class EditPanel extends React.Component {
deleteFeedback = () => {
if (this.state.id) {
api.del('feedback/' + this.props.problemID + '/' + this.state.id)
- .then(() => this.props.goBack())
+ .then(() => {
+ this.state.updateCallback({
+ id: this.state.id,
+ deleted: true
+ })
+ this.props.goBack()
+ })
}
}
render () {
return (
- <nav className='panel'>
- <p className='panel-heading'>
+ <React.Fragment>
+ <p className={this.props.grading ? 'panel-heading' : 'panel-heading is-radiusless'}>
Manage feedback
</p>
@@ -154,7 +168,7 @@ class EditPanel extends React.Component {
<div className='panel-block'>
<BackButton onClick={this.props.goBack} />
<SaveButton onClick={this.saveFeedback} exists={this.props.feedback}
- disabled={!this.state.name || !this.state.score || isNaN(parseInt(this.state.score))} />
+ disabled={!this.state.name || (!this.state.score && this.state.score !== 0) || isNaN(parseInt(this.state.score))} />
<DeleteButton onClick={() => { this.setState({deleting: true}) }} exists={this.props.feedback} />
<ConfirmationModal
headerText={`Do you want to irreversibly delete feedback option "${this.state.name}"?`}
@@ -168,7 +182,7 @@ class EditPanel extends React.Component {
onCancel={() => { this.setState({deleting: false}) }}
/>
</div>
- </nav>
+ </React.Fragment>
)
}
}
diff --git a/client/views/grade/FeedbackBlock.jsx b/client/components/feedback/FeedbackBlock.jsx
similarity index 84%
rename from client/views/grade/FeedbackBlock.jsx
rename to client/components/feedback/FeedbackBlock.jsx
index 7bc7ba60798917605cb34fa14e589de666679d28..af1429fd855a49d6aad28bd3a5fe9d39f68f058b 100644
--- a/client/views/grade/FeedbackBlock.jsx
+++ b/client/components/feedback/FeedbackBlock.jsx
@@ -32,7 +32,9 @@ class FeedbackBlock extends React.Component {
render () {
const shortcut = (this.props.index < 11 ? '' : 'shift + ') + this.props.index % 10
return (
- <a className='panel-block is-active' onClick={this.toggle}
+ <a
+ className={this.props.grading ? 'panel-block is-active' : 'panel-block'}
+ onClick={this.props.grading ? this.toggle : this.props.editFeedback}
style={this.props.selected ? {backgroundColor: '#209cee'} : {}}
>
<span
@@ -40,7 +42,9 @@ class FeedbackBlock extends React.Component {
? ' tooltip is-tooltip-active is-tooltip-left' : '')}
data-tooltip={shortcut}
>
- <i className={'fa fa-' + (this.props.checked ? 'check-square-o' : 'square-o')} />
+ {this.props.grading &&
+ <i className={'fa fa-' + (this.props.checked ? 'check-square-o' : 'square-o')} />
+ }
</span>
<span style={{ width: '80%' }}>
{this.props.feedback.name}
diff --git a/client/views/grade/FeedbackPanel.jsx b/client/components/feedback/FeedbackPanel.jsx
similarity index 77%
rename from client/views/grade/FeedbackPanel.jsx
rename to client/components/feedback/FeedbackPanel.jsx
index ee5266e6f25447fdb6dcbc3040666bb411f09e98..e2e1ce622ea373b7772b6f14deb5df8f1f47177c 100644
--- a/client/views/grade/FeedbackPanel.jsx
+++ b/client/components/feedback/FeedbackPanel.jsx
@@ -4,7 +4,7 @@ import Notification from 'react-bulma-notification'
import * as api from '../../api.jsx'
-import withShortcuts from '../../components/ShortcutBinder.jsx'
+import withShortcuts from '../ShortcutBinder.jsx'
import FeedbackBlock from './FeedbackBlock.jsx'
class FeedbackPanel extends React.Component {
@@ -35,7 +35,7 @@ class FeedbackPanel extends React.Component {
static getDerivedStateFromProps (nextProps, prevState) {
if (prevState.problemID !== nextProps.problem.id || prevState.submissionID !== nextProps.submissionID) {
return {
- remark: nextProps.solution.remark,
+ remark: nextProps.grading && nextProps.solution.remark,
problemID: nextProps.problem.id,
submissionID: nextProps.submissionID,
selectedFeedbackIndex: null
@@ -90,29 +90,34 @@ class FeedbackPanel extends React.Component {
const blockURI = this.props.examID + '/' + this.props.submissionID + '/' + this.props.problem.id
let totalScore = 0
- for (let i = 0; i < this.props.solution.feedback.length; i++) {
- const probIndex = this.props.problem.feedback.findIndex(fb => fb.id === this.props.solution.feedback[i])
- if (probIndex >= 0) totalScore += this.props.problem.feedback[probIndex].score
+ if (this.props.grading) {
+ for (let i = 0; i < this.props.solution.feedback.length; i++) {
+ const probIndex = this.props.problem.feedback.findIndex(fb => fb.id === this.props.solution.feedback[i])
+ if (probIndex >= 0) totalScore += this.props.problem.feedback[probIndex].score
+ }
}
let selectedFeedbackId = this.state.selectedFeedbackIndex !== null &&
this.props.problem.feedback[this.state.selectedFeedbackIndex].id
return (
- <nav className='panel'>
- <p className='panel-heading'>
- Total: <b>{totalScore}</b>
- </p>
+ <React.Fragment>
+ {this.props.grading &&
+ <p className='panel-heading'>
+ Total: <b>{totalScore}</b>
+ </p>}
{this.props.problem.feedback.map((feedback, index) =>
<FeedbackBlock key={feedback.id} uri={blockURI} graderID={this.props.graderID}
- feedback={feedback} checked={this.props.solution.feedback.includes(feedback.id)}
+ feedback={feedback} checked={this.props.grading && this.props.solution.feedback.includes(feedback.id)}
editFeedback={() => this.props.editFeedback(feedback)} updateSubmission={this.props.updateSubmission}
- ref={(selectedFeedbackId === feedback.id) ? this.feedbackBlock : null}
+ ref={(selectedFeedbackId === feedback.id) ? this.feedbackBlock : null} grading={this.props.grading}
selected={selectedFeedbackId === feedback.id} showIndex={this.props.showTooltips} index={index + 1} />
)}
- <div className='panel-block'>
- <textarea className='textarea' rows='2' placeholder='remark' value={this.state.remark} onBlur={this.saveRemark} onChange={this.changeRemark} />
- </div>
+ {this.props.grading &&
+ <div className='panel-block'>
+ <textarea className='textarea' rows='2' placeholder='remark' value={this.state.remark} onBlur={this.saveRemark} onChange={this.changeRemark} />
+ </div>
+ }
<div className='panel-block'>
<button className='button is-link is-outlined is-fullwidth' onClick={() => this.props.editFeedback()}>
<span className='icon is-small'>
@@ -121,7 +126,7 @@ class FeedbackPanel extends React.Component {
<span>option</span>
</button>
</div>
- </nav>
+ </React.Fragment>
)
}
}
diff --git a/client/views/Exam.jsx b/client/views/Exam.jsx
index 60d0c33a5f54df9d4324b01e573898102bd32752..b7f7f0ad241509311604e69d15f9683c3bf6ac6e 100644
--- a/client/views/Exam.jsx
+++ b/client/views/Exam.jsx
@@ -10,6 +10,8 @@ import ExamEditor from './ExamEditor.jsx'
import update from 'immutability-helper'
import ExamFinalizeMarkdown from './ExamFinalize.md'
import ConfirmationModal from '../components/ConfirmationModal.jsx'
+import FeedbackPanel from '../components/feedback/FeedbackPanel.jsx'
+import EditPanel from '../components/feedback/EditPanel.jsx'
import * as api from '../api.jsx'
@@ -17,6 +19,9 @@ class Exams extends React.Component {
state = {
examID: null,
page: 0,
+ editActive: false,
+ feedbackToEdit: null,
+ problemIdToEditFeedbackOf: null,
numPages: null,
selectedWidgetId: null,
changedWidgetId: null,
@@ -38,7 +43,8 @@ class Exams extends React.Component {
id: problem.id,
page: problem.page,
name: problem.name,
- graded: problem.graded
+ graded: problem.graded,
+ feedback: problem.feedback || []
}
}
})
@@ -57,7 +63,6 @@ class Exams extends React.Component {
previewing: false
}
}
-
return null
}
@@ -69,6 +74,10 @@ class Exams extends React.Component {
// The onBlur event is not fired when the input field is being disabled
if (prevState.selectedWidgetId !== this.state.selectedWidgetId) {
this.saveProblemName()
+ this.setState({
+ editActive: false,
+ problemIdToEditFeedbackOf: false
+ })
}
}
@@ -80,7 +89,41 @@ class Exams extends React.Component {
// This might try to save the name unnecessary, but better twice than never.
this.saveProblemName()
// Force an update of the upper exam state, since this component does not update and use that correctly
- this.props.updateExam(this.props.examID)
+ if (!this.state.deletingExam) {
+ this.props.updateExam(this.props.examID)
+ }
+ }
+
+ editFeedback = (feedback) => {
+ this.setState({
+ editActive: true,
+ feedbackToEdit: feedback,
+ problemIdToEditFeedbackOf: this.state.selectedWidgetId
+ })
+ }
+
+ updateFeedback = (feedback) => {
+ var widgets = this.state.widgets
+ const idx = widgets[this.state.selectedWidgetId].problem.feedback.findIndex(e => { return e.id === feedback.id })
+ if (idx === -1) widgets[this.state.selectedWidgetId].problem.feedback.push(feedback)
+ else {
+ if (feedback.deleted) widgets[this.state.selectedWidgetId].problem.feedback.splice(idx, 1)
+ else widgets[this.state.selectedWidgetId].problem.feedback[idx] = feedback
+ }
+ this.setState({
+ widgets: widgets
+ })
+ }
+
+ backToFeedback = () => {
+ this.props.updateExam(this.props.exam.id)
+ this.setState({
+ editActive: false
+ })
+ }
+
+ isProblemWidget = (widget) => {
+ return widget && this.state.widgets[widget].problem
}
saveProblemName = () => {
@@ -110,6 +153,8 @@ class Exams extends React.Component {
selectedWidgetId: null,
changedWidgetId: null,
deletingWidget: false,
+ editActive: false,
+ problemIdToEditFeedbackOf: null,
widgets: update(prevState.widgets, {
$unset: [widgetId]
})
@@ -306,6 +351,20 @@ class Exams extends React.Component {
)}
</div>
</div>
+ {this.isProblemWidget(selectedWidgetId) &&
+ <React.Fragment>
+ <div className='panel-block'>
+ {!this.state.editActive && <label className='label'>Feedback options</label>}
+ </div>
+ {this.state.editActive
+ ? <EditPanel problemID={props.problem.id} feedback={this.state.feedbackToEdit}
+ goBack={this.backToFeedback} updateCallback={this.updateFeedback} />
+ : <FeedbackPanel examID={this.props.examID} problem={props.problem}
+ editFeedback={this.editFeedback} showTooltips={this.state.showTooltips}
+ grading={false}
+ />}
+ </React.Fragment>
+ }
<div className='panel-block'>
<button
disabled={props.disabledDelete}
diff --git a/client/views/ExamEditor.jsx b/client/views/ExamEditor.jsx
index 89cf1d8d163c9ace55510395de54147c9d694b96..5f46964741c7bbd62f490fee6f39765ece99074f 100644
--- a/client/views/ExamEditor.jsx
+++ b/client/views/ExamEditor.jsx
@@ -86,7 +86,8 @@ class ExamEditor extends React.Component {
if (selectionBox.width >= this.props.problemMinWidth && selectionBox.height >= this.props.problemMinHeight) {
const problemData = {
name: 'New problem', // TODO: Name
- page: this.props.page
+ page: this.props.page,
+ feedback: []
}
const widgetData = {
x: Math.round(selectionBox.left),
diff --git a/client/views/Grade.jsx b/client/views/Grade.jsx
index dc2b81f453787d5436edec0ddc394e99c5813473..e98e1143859bf16e46ededf86bb32188cf50951e 100644
--- a/client/views/Grade.jsx
+++ b/client/views/Grade.jsx
@@ -2,9 +2,9 @@ import React from 'react'
import Hero from '../components/Hero.jsx'
-import FeedbackPanel from './grade/FeedbackPanel.jsx'
+import FeedbackPanel from '../components/feedback/FeedbackPanel.jsx'
import ProblemSelector from './grade/ProblemSelector.jsx'
-import EditPanel from './grade/EditPanel.jsx'
+import EditPanel from '../components/feedback/EditPanel.jsx'
import SearchBox from '../components/SearchBox.jsx'
import ProgressBar from '../components/ProgressBar.jsx'
import withShortcuts from '../components/ShortcutBinder.jsx'
@@ -12,6 +12,7 @@ import withShortcuts from '../components/ShortcutBinder.jsx'
import * as api from '../api.jsx'
import 'bulma-tooltip/dist/css/bulma-tooltip.min.css'
+import './grade/Grade.css'
class Grade extends React.Component {
state = {
@@ -209,17 +210,18 @@ class Grade extends React.Component {
<div className='column is-one-quarter-desktop is-one-third-tablet'>
<ProblemSelector problems={exam.problems} changeProblem={this.changeProblem}
current={this.state.pIndex} showTooltips={this.state.showTooltips} />
- {this.state.editActive
- ? <EditPanel problemID={problem.id} feedback={this.state.feedbackToEdit}
- goBack={this.backToFeedback} />
- : <FeedbackPanel examID={exam.id} submissionID={submission.id}
- problem={problem} solution={solution} graderID={this.props.graderID}
- editFeedback={this.editFeedback} showTooltips={this.state.showTooltips}
- updateSubmission={() => {
- this.props.updateSubmission(this.state.sIndex)
- }
- } />
- }
+ <nav className='panel'>
+ {this.state.editActive
+ ? <EditPanel problemID={problem.id} feedback={this.state.feedbackToEdit}
+ goBack={this.backToFeedback} />
+ : <FeedbackPanel examID={exam.id} submissionID={submission.id}
+ problem={problem} solution={solution} graderID={this.props.graderID}
+ editFeedback={this.editFeedback} showTooltips={this.state.showTooltips}
+ updateSubmission={() => {
+ this.props.updateSubmission(this.state.sIndex)
+ }} grading />
+ }
+ </nav>
</div>
<div className='column'>
@@ -259,9 +261,13 @@ class Grade extends React.Component {
renderSuggestion={(submission) => {
const stud = submission.student
return (
- <div>
- <b>{`${stud.firstName} ${stud.lastName}`}</b>
- <i style={{float: 'right'}}>({stud.id})</i>
+ <div className='flex-parent'>
+ <b className='flex-child truncated'>
+ {`${stud.firstName} ${stud.lastName}`}
+ </b>
+ <i className='flex-child fixed'>
+ ({stud.id})
+ </i>
</div>
)
}}
@@ -294,7 +300,7 @@ class Grade extends React.Component {
</article> : null
}
- <p className='box'>
+ <p className={'box' + (solution.graded_at ? ' is-graded' : '')}>
<img src={exam.id ? ('api/images/solutions/' + exam.id + '/' +
problem.id + '/' + submission.id + '/' + (this.state.fullPage ? '1' : '0')) + '?' +
this.getLocationHash(problem) : ''} alt='' />
diff --git a/client/views/grade/Grade.css b/client/views/grade/Grade.css
new file mode 100644
index 0000000000000000000000000000000000000000..8b88019a647828998811e4a4df4a7a6dec7559c8
--- /dev/null
+++ b/client/views/grade/Grade.css
@@ -0,0 +1,19 @@
+.box.is-graded {
+ box-shadow: 0px 0px 6px #23d160, 0 0 0 1px rgba(10, 10, 10, 0.1);
+}
+
+.flex-parent {
+ display: flex;
+ align-items: center;
+}
+
+.flex-child.truncated {
+ flex: 1;
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+.flex-child.fixed {
+ white-space: nowrap;
+}
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2e3d07767494c43a2de7e09d122b2aff02be22a7
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,56 @@
+name: zesje-dev
+channels:
+ - conda-forge
+ - anaconda
+dependencies:
+ - python=3.6
+ - yarn
+ - gunicorn
+ - redis
+ - pip
+ - pip:
+ # Core components
+ - flask
+ - flask_restful
+ - flask_sqlalchemy
+ - sqlalchemy
+ - Flask-Migrate
+ - alembic
+ - pyyaml
+ - celery
+ - redis
+
+ # General utilities
+ - numpy==1.15.4
+ - scipy==1.3.0
+
+ # summary plot generation
+ - matplotlib
+ - seaborn
+
+ # PDF generation
+ - pdfrw
+ - reportlab
+ - Wand
+ - Pillow # also scan processing
+
+ # Scan processing
+ - opencv-python
+ - pikepdf
+ - pylibdmtx
+
+ # Exporting
+ - pandas
+ - openpyxl # required for writing dataframes as Excel spreadsheets
+
+ #
+ # Development dependencies
+ #
+
+ # Tests
+ - pytest
+ - pyssim
+ - pytest-cov
+
+ # Linting
+ - flake8
diff --git a/package.json b/package.json
index ee31829f380dca39acc022f51e243514aa6fcc16..819fd9b6d9110f8eed25a04130ba3cdb287f7017 100644
--- a/package.json
+++ b/package.json
@@ -3,7 +3,7 @@
"main": "index.js",
"license": "AGPL-3.0",
"scripts": {
- "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON,CELERY\" --prefix-colors \"bgBlue.bold,bgGreen.bold,bgRed.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg celery -A zesje.celery worker\"",
+ "dev": "concurrently --kill-others --names \"WEBPACK,PYTHON,CELERY,REDIS\" --prefix-colors \"bgBlue.bold,bgGreen.bold,bgRed.bold,bgYellow.bold\" \"webpack-dev-server --hot --inline --progress --config webpack.dev.js\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg python3 zesje\" \"ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg celery -A zesje.celery worker -l info --autoscale=4,1 --max-tasks-per-child=16\" \"redis-server redis.conf\"",
"build": "webpack --config webpack.prod.js",
"ci": "yarn lint; yarn test",
"lint": "yarn lint:js; yarn lint:py",
@@ -16,7 +16,9 @@
"analyze": "webpack --config webpack.prod.js --profile --json > stats.json; webpack-bundle-analyzer stats.json zesje/static",
"migrate:dev": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg FLASK_APP=zesje/__init__.py flask db upgrade",
"migrate": "FLASK_APP=zesje/__init__.py flask db upgrade",
- "prepare-migration": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg FLASK_APP=zesje/__init__.py flask db migrate"
+ "prepare-migration": "ZESJE_SETTINGS=$(pwd)/zesje.dev.cfg FLASK_APP=zesje/__init__.py flask db migrate",
+ "test:py:cov": "python3 -m pytest -v -W error::RuntimeWarning --cov=zesje --cov-report=xml:cov.xml --cov-report=html:cov.html --cov-report=term tests/",
+ "migrate-down": "FLASK_APP=zesje/__init__.py flask db downgrade"
},
"standard": {
"parser": "babel-eslint",
diff --git a/redis.conf b/redis.conf
new file mode 100644
index 0000000000000000000000000000000000000000..cc274107936dfea9d0a98702eb9f714df3c1e1a6
--- /dev/null
+++ b/redis.conf
@@ -0,0 +1,2 @@
+port 6479
+loglevel notice
\ No newline at end of file
diff --git a/requirements-dev.txt b/requirements-dev.txt
deleted file mode 100644
index 729f564bb1f31c49a88e5ae1b4bb8540e47abd8d..0000000000000000000000000000000000000000
--- a/requirements-dev.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# Tests
-pytest
-pyssim
-
-# Linting
-flake8
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 5bf41ebd1370c15a8e8876d3826330021a0b1fd5..0000000000000000000000000000000000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-# Core components
-flask
-flask_restful
-flask_sqlalchemy
-sqlalchemy
-Flask-Migrate
-alembic
-pyyaml
-celery
-redis
-
-# General utilities
-numpy
-scipy
-
-# summary plot generation
-matplotlib
-seaborn
-
-# PDF generation
-pdfrw
-reportlab
-Wand
-Pillow # also scan processing
-pyStrich # TODO: can we replace this with stuff from pylibdmtx?
-
-# Scan processing
-opencv-python
-git+https://github.com/mstamy2/PyPDF2
-pylibdmtx
-
-# Exporting
-pandas
-openpyxl # required for writing dataframes as Excel spreadsheets
diff --git a/tests/conftest.py b/tests/conftest.py
index 0c373c16efa7c85627e2ba0efc09afa8b578987b..932c59914e744129b0ea99b0285692e8ef239052 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,9 +1,39 @@
import os
import pytest
+from flask import Flask
+from zesje.api import api_bp
+from zesje.database import db
# Adapted from https://stackoverflow.com/a/46062148/1062698
@pytest.fixture
def datadir():
return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data')
+
+
+@pytest.fixture(scope="module")
+def app():
+ app = Flask(__name__, static_folder=None)
+
+ app.config.update(
+ SQLALCHEMY_DATABASE_URI='sqlite:///:memory:',
+ SQLALCHEMY_TRACK_MODIFICATIONS=False # Suppress future deprecation warning
+ )
+ db.init_app(app)
+
+ with app.app_context():
+ db.create_all()
+
+ app.register_blueprint(api_bp, url_prefix='/api')
+
+ return app
+
+
+@pytest.fixture
+def empty_app(app):
+ with app.app_context():
+ db.drop_all()
+ db.create_all()
+
+ return app
diff --git a/tests/data/flattened-a4-2pages.pdf b/tests/data/flattened-a4-2pages.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f3fd0b348aae99c435ed85f1192d51dccac4c15c
Binary files /dev/null and b/tests/data/flattened-a4-2pages.pdf differ
diff --git a/tests/data/scanned_pdfs/blank.jpg b/tests/data/scanned_pdfs/blank.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..23b5d6f755eac86469fe9d8cbe697e8d2d199f1a
Binary files /dev/null and b/tests/data/scanned_pdfs/blank.jpg differ
diff --git a/tests/data/scanned_pdfs/jamy1.jpg b/tests/data/scanned_pdfs/jamy1.jpg
deleted file mode 100644
index c28501f2392f36cbfc40c074f8f5dba5a524d706..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/jamy1.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/jamy2.jpg b/tests/data/scanned_pdfs/jamy2.jpg
deleted file mode 100644
index c28501f2392f36cbfc40c074f8f5dba5a524d706..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/jamy2.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex1.jpg b/tests/data/scanned_pdfs/latex1.jpg
deleted file mode 100644
index 9c8830c177544d722fc9bc6b963b74595c5d7ecf..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex1.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex2.jpg b/tests/data/scanned_pdfs/latex2.jpg
deleted file mode 100644
index 3cbc8508dde10104caf8f7786e8c1231a2fe12a1..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex2.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex3.jpg b/tests/data/scanned_pdfs/latex3.jpg
deleted file mode 100644
index 9157dd64c8afecf78c6c1d446ab0bff80f35551e..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex3.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex4.jpg b/tests/data/scanned_pdfs/latex4.jpg
deleted file mode 100644
index a44f1af340656ba8a91bbe0e95df26cdc3200fe5..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex4.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex5.jpg b/tests/data/scanned_pdfs/latex5.jpg
deleted file mode 100644
index a90248dbea353f628e727070664d294f996de95f..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex5.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/latex6.jpg b/tests/data/scanned_pdfs/latex6.jpg
deleted file mode 100644
index fca94498196d8499f50bd58da91ccf0a4bcb3163..0000000000000000000000000000000000000000
Binary files a/tests/data/scanned_pdfs/latex6.jpg and /dev/null differ
diff --git a/tests/data/scanned_pdfs/messy_three_corners.jpg b/tests/data/scanned_pdfs/messy_three_corners.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..83232b7e0040f45d5510d61ffbe769d06b448ace
Binary files /dev/null and b/tests/data/scanned_pdfs/messy_three_corners.jpg differ
diff --git a/tests/data/scanned_pdfs/missing_two_corners.jpg b/tests/data/scanned_pdfs/missing_two_corners.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5cdbf9521137dd4a846c3b1c9750c8e869b8a0ce
Binary files /dev/null and b/tests/data/scanned_pdfs/missing_two_corners.jpg differ
diff --git a/tests/data/scanned_pdfs/sample_exam.jpg b/tests/data/scanned_pdfs/sample_exam.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6cd6fd878db6c7b81208f6e019645aca89cba8b0
Binary files /dev/null and b/tests/data/scanned_pdfs/sample_exam.jpg differ
diff --git a/tests/data/scanned_pdfs/shifted.jpg b/tests/data/scanned_pdfs/shifted.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c2931bc5f58e36793abef4590e828474d8739534
Binary files /dev/null and b/tests/data/scanned_pdfs/shifted.jpg differ
diff --git a/tests/data/scanned_pdfs/tilted.jpg b/tests/data/scanned_pdfs/tilted.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6230dc73c40d0eb5af6e22160b4f88de62ea8201
Binary files /dev/null and b/tests/data/scanned_pdfs/tilted.jpg differ
diff --git a/tests/data/single-image-a4.pdf b/tests/data/single-image-a4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..da8a0fe09595151a98a9eae3d3376b1bf34eb8da
Binary files /dev/null and b/tests/data/single-image-a4.pdf differ
diff --git a/tests/data/two-images-a4.pdf b/tests/data/two-images-a4.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..87af190fc6bfba91cc5156e43c128db51f4d1417
Binary files /dev/null and b/tests/data/two-images-a4.pdf differ
diff --git a/tests/test_database.py b/tests/test_database.py
index c890a1880b17706ca7d80f152ba80ab788cab21c..fc93348a3c441b9b36e4f5890c41e79c6f86f322 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -1,7 +1,8 @@
import pytest
from flask import Flask
-from zesje.database import db, Exam, _generate_exam_token
+from zesje.database import db, _generate_exam_token, Exam, Problem, ProblemWidget, Solution
+from zesje.database import Submission, Scan, Page, ExamWidget, FeedbackOption
@pytest.mark.parametrize('duplicate_count', [
@@ -32,3 +33,141 @@ def test_exam_generate_token_length_uppercase(duplicate_count, monkeypatch):
id = _generate_exam_token()
assert len(id) == 12
assert id.isupper()
+
+
+def test_cascades_exam(empty_app, exam, problem, submission, scan, exam_widget):
+ """Tests the cascades defined for an exam
+
+ Tests the cascades for the following relations:
+ - Exam -> Submission
+ - Exam -> Problem
+ - Exam -> Scan
+ - Exam -> ExamWidget
+ """
+ empty_app.app_context().push()
+ exam.problems = [problem]
+ exam.scans = [scan]
+ exam.submissions = [submission]
+ exam.widgets = [exam_widget]
+
+ db.session.add(exam)
+ db.session.commit()
+
+ assert problem in db.session
+ assert submission in db.session
+ assert scan in db.session
+ assert exam_widget in db.session
+
+ db.session.delete(exam)
+ db.session.commit()
+
+ assert problem not in db.session
+ assert submission not in db.session
+ assert scan not in db.session
+ assert exam_widget not in db.session
+
+
+def test_cascades_problem(empty_app, exam, problem, submission, solution, problem_widget, feedback_option):
+ """Tests the cascades defined for a problem
+
+ Tests the cascades for the following relations:
+ - Problem -> Solution
+ - Problem -> ProblemWidget
+ - Problem -> FeedbackOption
+ """
+ empty_app.app_context().push()
+
+ exam.problems = [problem]
+ exam.submissions = [submission]
+ solution.submission = submission
+ problem.widget = problem_widget
+ problem.solutions = [solution]
+ problem.feedback_options = [feedback_option]
+
+ db.session.add_all([exam, problem, submission])
+ db.session.commit()
+
+ assert solution in db.session
+ assert problem_widget in db.session
+ assert feedback_option in db.session
+
+ db.session.delete(problem)
+ db.session.commit()
+
+ assert solution not in db.session
+ assert problem_widget not in db.session
+ assert feedback_option not in db.session
+
+
+def test_cascades_submission(empty_app, exam, problem, submission, solution, page):
+ """Tests the cascades defined for a submission
+
+ Tests the cascades for the following relations:
+ - Submission -> Solution
+ - Submission -> Page
+ """
+ empty_app.app_context().push()
+
+ exam.problems = [problem]
+ exam.submissions = [submission]
+
+ solution.problem = problem
+ solution.submission = submission
+ page.submission = submission
+
+ db.session.add_all([exam, problem, submission])
+ db.session.commit()
+
+ assert solution in db.session
+ assert page in db.session
+
+ db.session.delete(submission)
+ db.session.commit()
+
+ assert solution not in db.session
+ assert page not in db.session
+
+
+@pytest.fixture
+def exam():
+ return Exam(name='')
+
+
+@pytest.fixture
+def problem():
+ return Problem(name='')
+
+
+@pytest.fixture
+def problem_widget():
+ return ProblemWidget(name='', page=0, x=0, y=0, width=0, height=0)
+
+
+@pytest.fixture
+def exam_widget():
+ return ExamWidget(name='', x=0, y=0)
+
+
+@pytest.fixture
+def submission():
+ return Submission(copy_number=0)
+
+
+@pytest.fixture
+def solution():
+ return Solution()
+
+
+@pytest.fixture
+def scan():
+ return Scan(name='', status='')
+
+
+@pytest.fixture
+def page():
+ return Page(path='', number=0)
+
+
+@pytest.fixture
+def feedback_option():
+ return FeedbackOption(text='')
diff --git a/tests/test_rotation_scan.py b/tests/test_rotation_scan.py
index 0025d96cadbbea23c3d3511928cf126d9665de53..15b3e55ee27f14cbe537f8dd333edd9fbfb344c1 100644
--- a/tests/test_rotation_scan.py
+++ b/tests/test_rotation_scan.py
@@ -1,7 +1,6 @@
import math
import os
-import cv2
import numpy as np
import PIL
import pytest
@@ -17,12 +16,12 @@ def distance(keyp1, keyp2):
# Given a name of a exam image and the location it is stored, retrieves the
# image and converts it to binary image
-def generate_binary_image(name, datadir):
+def generate_image(name, datadir):
pdf_path = os.path.join(datadir, 'scanned_pdfs', f'{name}')
pil_im = PIL.Image.open(pdf_path)
- opencv_im = cv2.cvtColor(np.array(pil_im), cv2.COLOR_RGB2BGR)
- _, bin_im = cv2.threshold(opencv_im, 150, 255, cv2.THRESH_BINARY)
- return bin_im
+ pil_im = pil_im.convert('RGB')
+ image_array = np.array(pil_im)
+ return image_array
# Tests
@@ -41,15 +40,22 @@ def test_calc_angle(test_input1, test_input2, expected):
# Tests whether the amount of cornermakers is enough to calculate the angle and
# whether it is lower than 5 as we only add 4 corner markers per page.
-@pytest.mark.parametrize('name', os.listdir(
- os.path.join('tests',
- 'data', 'scanned_pdfs')),
- ids=os.listdir(
- os.path.join('tests', 'data', 'scanned_pdfs')))
-def test_detect_enough_cornermarkers(name, datadir):
- bin_im = generate_binary_image(name, datadir)
- keypoints = scans.find_corner_marker_keypoints(bin_im)
- assert(len(keypoints) >= 2 & len(keypoints) <= 4)
+test_args = [
+ ('missing_two_corners.jpg', 2),
+ ('sample_exam.jpg', 4),
+ ('shifted.jpg', 4),
+ ('tilted.jpg', 4),
+ ('messy_three_corners.jpg', 3),
+ ('blank.jpg', 0)]
+
+
+@pytest.mark.parametrize(
+ 'name,expected', test_args,
+ ids=list(map(lambda e: f"{e[0]} ({e[1]} markers)", test_args)))
+def test_detect_enough_cornermarkers(name, expected, datadir):
+ image = generate_image(name, datadir)
+ keypoints = scans.find_corner_marker_keypoints(image)
+ assert(len(keypoints) == expected)
# Tests whether the detected keypoints are actually corner markers.
@@ -62,10 +68,10 @@ def test_detect_enough_cornermarkers(name, datadir):
ids=os.listdir(
os.path.join('tests', 'data', 'scanned_pdfs')))
def test_detect_valid_cornermarkers(name, datadir):
- bin_im = generate_binary_image(name, datadir)
- keypoints = scans.find_corner_marker_keypoints(bin_im)
+ image = generate_image(name, datadir)
+ keypoints = scans.find_corner_marker_keypoints(image)
- h, w, *_ = bin_im.shape
+ h, w, *_ = image.shape
(xmm, ymm) = (210, 297)
(xcorner, ycorner) = (round(30 * w / xmm), round(30 * h / ymm))
maxdist = math.hypot(xcorner, ycorner)
diff --git a/tests/test_scans.py b/tests/test_scans.py
index 115d2c5b06dfdb3499cfb104585febddd0fe2995..e4f012f95202d5402d482861dcaabc051c9219bd 100644
--- a/tests/test_scans.py
+++ b/tests/test_scans.py
@@ -7,6 +7,7 @@ from tempfile import NamedTemporaryFile
from flask import Flask
from io import BytesIO
import wand.image
+from pikepdf import Pdf
from zesje.scans import decode_barcode, ExamMetadata, ExtractedBarcode
from zesje.database import db, _generate_exam_token
@@ -265,3 +266,36 @@ def test_all_effects(
# image.show()
success, reason = scans.process_page(image, new_exam, datadir)
assert success is expected, reason
+
+
+@pytest.mark.parametrize('filename,expected', [
+ ['blank-a4-2pages.pdf', AttributeError],
+ ['single-image-a4.pdf', ValueError],
+ ['two-images-a4.pdf', ValueError],
+ ['flattened-a4-2pages.pdf', None]],
+ ids=['blank pdf', 'single image', 'two images', 'flattened pdf'])
+def test_image_extraction_pike(datadir, filename, expected):
+ file = os.path.join(datadir, filename)
+ with Pdf.open(file) as pdf_reader:
+ for pagenr in range(len(pdf_reader.pages)):
+ if expected is not None:
+ with pytest.raises(expected):
+ scans.extract_image_pikepdf(pagenr, pdf_reader)
+ else:
+ img = scans.extract_image_pikepdf(pagenr, pdf_reader)
+ assert img is not None
+
+
+@pytest.mark.parametrize('filename', [
+ 'blank-a4-2pages.pdf',
+ 'flattened-a4-2pages.pdf'],
+ ids=['blank pdf', 'flattened pdf'])
+def test_image_extraction(datadir, filename):
+ file = os.path.join(datadir, filename)
+ page = 0
+ for img, pagenr in scans.extract_images(file):
+ page += 1
+ assert pagenr == page
+ assert img is not None
+ assert np.average(np.array(img)) == 255
+ assert page == 2
diff --git a/zesje/api/exams.py b/zesje/api/exams.py
index 3b354f92eb6e1b1b0269d986e9d020dea749a680..d2396658d111ca587d7f8a5cc1305e79ccac126c 100644
--- a/zesje/api/exams.py
+++ b/zesje/api/exams.py
@@ -10,7 +10,7 @@ from werkzeug.datastructures import FileStorage
from sqlalchemy.orm import selectinload
from ..pdf_generation import generate_pdfs, output_pdf_filename_format, join_pdfs, page_is_size
-from ..database import db, Exam, ExamWidget, Submission
+from ..database import db, Exam, ExamWidget, Submission, token_length
PAGE_FORMATS = {
"A4": (595.276, 841.89),
@@ -39,8 +39,14 @@ class Exams(Resource):
return dict(status=404, message='Exam does not exist.'), 404
elif exam.finalized:
return dict(status=409, message='Cannot delete a finalized exam.'), 409
+ elif Submission.query.filter(Submission.exam_id == exam.id).count():
+ return dict(status=500, message='Exam is not finalized but already has submissions.'), 500
else:
- exam.delete()
+ # All corresponding solutions, scans and problems are automatically deleted
+ db.session.delete(exam)
+ db.session.commit()
+
+ return dict(status=200, message="ok"), 200
def _get_all(self):
"""get list of uploaded exams.
@@ -269,6 +275,7 @@ class ExamSource(Resource):
return send_file(
os.path.join(exam_dir, 'exam.pdf'),
+ cache_timeout=0,
mimetype='application/pdf')
@@ -471,8 +478,8 @@ class ExamPreview(Resource):
generate_pdfs(
exam_path,
- exam.token[:5] + 'PREVIEW',
- [1519],
+ "A" * token_length,
+ [1559],
[output_file],
student_id_widget.x, student_id_widget.y,
barcode_widget.x, barcode_widget.y
diff --git a/zesje/api/problems.py b/zesje/api/problems.py
index 41c24f2ed1cd5f7909611e86fe2156efe719cf7a..ee228423f5f8a1bc4d131c303f8f80dd625e2c71 100644
--- a/zesje/api/problems.py
+++ b/zesje/api/problems.py
@@ -105,10 +105,7 @@ class Problems(Resource):
if any([sol.graded_by is not None for sol in problem.solutions]):
return dict(status=403, message=f'Problem has already been graded'), 403
else:
- # Delete all solutions associated with this problem
- for sol in problem.solutions:
- db.session.delete(sol)
- db.session.delete(problem.widget)
+ # The widget and all associated solutions are automatically deleted
db.session.delete(problem)
db.session.commit()
return dict(status=200, message="ok"), 200
diff --git a/zesje/api/scans.py b/zesje/api/scans.py
index 88c42c553430f1bdc2a685277211d457e17a2829..d0eecd400d6da8522737f6337a864e37dc265cfe 100644
--- a/zesje/api/scans.py
+++ b/zesje/api/scans.py
@@ -67,7 +67,7 @@ class Scans(Resource):
return dict(status=404, message='Exam does not exist.'), 404
scan = Scan(exam=exam, name=args['pdf'].filename,
- status='processing', message='importing PDF')
+ status='processing', message='Waiting...')
db.session.add(scan)
db.session.commit()
diff --git a/zesje/database.py b/zesje/database.py
index 7f3d02fd4f8e30cc38fc550e653025cfd0b85ee2..1f1d4ac4bf389404a25084ad092bfba0e11ddbd1 100644
--- a/zesje/database.py
+++ b/zesje/database.py
@@ -62,10 +62,11 @@ class Exam(db.Model):
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(Text, nullable=False)
token = Column(String(token_length), unique=True, default=_generate_exam_token)
- submissions = db.relationship('Submission', backref='exam', lazy=True)
- problems = db.relationship('Problem', backref='exam', order_by='Problem.id', lazy=True)
- scans = db.relationship('Scan', backref='exam', lazy=True)
- widgets = db.relationship('ExamWidget', backref='exam', order_by='ExamWidget.id', lazy=True)
+ submissions = db.relationship('Submission', backref='exam', cascade='all', lazy=True)
+ problems = db.relationship('Problem', backref='exam', cascade='all', order_by='Problem.id', lazy=True)
+ scans = db.relationship('Scan', backref='exam', cascade='all', lazy=True)
+ widgets = db.relationship('ExamWidget', backref='exam', cascade='all',
+ order_by='ExamWidget.id', lazy=True)
finalized = Column(Boolean, default=False, server_default='f')
@@ -75,8 +76,9 @@ class Submission(db.Model):
id = Column(Integer, primary_key=True, autoincrement=True)
copy_number = Column(Integer, nullable=False)
exam_id = Column(Integer, ForeignKey('exam.id'), nullable=False)
- solutions = db.relationship('Solution', backref='submission', order_by='Solution.problem_id', lazy=True)
- pages = db.relationship('Page', backref='submission', lazy=True)
+ solutions = db.relationship('Solution', backref='submission', cascade='all',
+ order_by='Solution.problem_id', lazy=True)
+ pages = db.relationship('Page', backref='submission', cascade='all', lazy=True)
student_id = Column(Integer, ForeignKey('student.id'), nullable=True)
signature_validated = Column(Boolean, default=False, server_default='f', nullable=False)
@@ -96,9 +98,10 @@ class Problem(db.Model):
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(Text, nullable=False)
exam_id = Column(Integer, ForeignKey('exam.id'), nullable=False)
- feedback_options = db.relationship('FeedbackOption', backref='problem', order_by='FeedbackOption.id', lazy=True)
- solutions = db.relationship('Solution', backref='problem', lazy=True)
- widget = db.relationship('ProblemWidget', backref='problem', uselist=False, lazy=True)
+ feedback_options = db.relationship('FeedbackOption', backref='problem', cascade='all',
+ order_by='FeedbackOption.id', lazy=True)
+ solutions = db.relationship('Solution', backref='problem', cascade='all', lazy=True)
+ widget = db.relationship('ProblemWidget', backref='problem', cascade='all', uselist=False, lazy=True)
class FeedbackOption(db.Model):
diff --git a/zesje/emails.py b/zesje/emails.py
index 8e3571608e501a0476cfcb357104d3cf2674184c..aa16ab1a5eed253186590cd076408475b02be0e2 100644
--- a/zesje/emails.py
+++ b/zesje/emails.py
@@ -8,10 +8,12 @@ from email.mime.base import MIMEBase
from email import encoders
import jinja2
-from wand.image import Image
+
+from reportlab.pdfgen import canvas
from .database import Submission
from . import statistics
+from .api.exams import PAGE_FORMATS
def solution_pdf(exam_id, student_id):
@@ -20,17 +22,17 @@ def solution_pdf(exam_id, student_id):
pages = sorted((p for s in subs for p in s.pages), key=(lambda p: p.number))
pages = [p.path for p in pages]
- with Image() as output_pdf:
- for filepath in pages:
- with Image(filename=filepath) as page:
- output_pdf.sequence.append(page)
-
- output_pdf.format = 'pdf'
-
- result = BytesIO()
+ from flask import current_app
+ page_format = current_app.config.get('PAGE_FORMAT', 'A4') # TODO Remove default value
+ page_size = PAGE_FORMATS[page_format]
- output_pdf.save(file=result)
+ result = BytesIO()
+ pdf = canvas.Canvas(result, pagesize=page_size)
+ for page in pages:
+ pdf.drawImage(page, 0, 0, width=page_size[0], height=page_size[1])
+ pdf.showPage()
+ pdf.save()
result.seek(0)
return result
diff --git a/zesje/factory.py b/zesje/factory.py
index 87069fee2402b654f988cedc3fbe6a32841b14a3..f404333c8b8cc18e79b201c5f7ff16c3c3e86527 100644
--- a/zesje/factory.py
+++ b/zesje/factory.py
@@ -33,8 +33,8 @@ def create_app():
)
app.config.update(
- CELERY_BROKER_URL='redis://localhost:6379',
- CELERY_RESULT_BACKEND='redis://localhost:6379'
+ CELERY_BROKER_URL='redis://localhost:6479',
+ CELERY_RESULT_BACKEND='redis://localhost:6479'
)
db.init_app(app)
diff --git a/zesje/pdf_generation.py b/zesje/pdf_generation.py
index f0bc3ebb00843818081e61bf45913cdf8127ff31..98e4b655ba8d2b229a20a6e837821534901241d9 100644
--- a/zesje/pdf_generation.py
+++ b/zesje/pdf_generation.py
@@ -1,9 +1,8 @@
-from io import BytesIO
from tempfile import NamedTemporaryFile
import PIL
from pdfrw import PdfReader, PdfWriter, PageMerge
-from pystrich.datamatrix import DataMatrixEncoder
+from pylibdmtx.pylibdmtx import encode
from reportlab.lib.units import mm
from reportlab.pdfgen import canvas
@@ -16,9 +15,9 @@ def generate_pdfs(exam_pdf_file, exam_id, copy_nums, output_paths, id_grid_x,
"""
Generate the final PDFs from the original exam PDF.
- To maintain a consistent size of the DataMatrix codes, adhere to (# of
- letters in exam ID) + 2 * (# of digits in exam ID) = C for a certain
- constant C. The reason for this is that pyStrich encodes two digits in as
+ To ensure the page information fits into the datamatrix grid, adhere to
+ (# of letters in exam ID) + 2 * (# of digits in exam ID) = C for a certain
+ constant C. The reason for this is that libdmtx encodes two digits in as
much space as one letter.
If maximum interchangeability with version 1 QR codes is desired (error
@@ -155,9 +154,9 @@ def generate_datamatrix(exam_id, page_num, copy_num):
"""
Generates a DataMatrix code to be used on a page.
- To maintain a consistent size of the DataMatrix codes, adhere to (# of
- letters in exam ID) + 2 * (# of digits in exam ID) = C for a certain
- constant C. The reason for this is that pyStrich encodes two digits in as
+ To ensure the page information fits into the datamatrix grid, adhere to
+ (# of letters in exam ID) + 2 * (# of digits in exam ID) = C for a certain
+ constant C. The reason for this is that pylibdmtx encodes two digits in as
much space as one letter.
If maximum interchangeability with version 1 QR codes is desired (error
@@ -182,8 +181,10 @@ def generate_datamatrix(exam_id, page_num, copy_num):
data = f'{exam_id}/{copy_num:04d}/{page_num:02d}'
- image_bytes = DataMatrixEncoder(data).get_imagedata(cellsize=2)
- return PIL.Image.open(BytesIO(image_bytes))
+ encoded = encode(data.encode('utf-8'), size='18x18')
+ datamatrix = PIL.Image.frombytes('RGB', (encoded.width, encoded.height), encoded.pixels)
+ datamatrix = datamatrix.resize((44, 44)).convert('L')
+ return datamatrix
def _generate_overlay(canv, pagesize, exam_id, copy_num, num_pages, id_grid_x,
@@ -192,9 +193,9 @@ def _generate_overlay(canv, pagesize, exam_id, copy_num, num_pages, id_grid_x,
Generates an overlay ('watermark') PDF, which can then be overlaid onto
the exam PDF.
- To maintain a consistent size of the DataMatrix codes in the overlay,
+ To ensure the page information fits into the datamatrix grid in the overlay,
adhere to (# of letters in exam ID) + 2 * (# of digits in exam ID) = C for
- a certain constant C. The reason for this is that pyStrich encodes two
+ a certain constant C. The reason for this is that pylibdmtx encodes two
digits in as much space as one letter.
If maximum interchangeability with version 1 QR codes is desired (error
@@ -223,16 +224,16 @@ def _generate_overlay(canv, pagesize, exam_id, copy_num, num_pages, id_grid_x,
The y coordinate where the DataMatrix codes should be placed
"""
- # Font settings for the copy number (printed under the datamatrix)
- fontsize = 8
- canv.setFont('Helvetica', fontsize)
-
# transform y-cooridate to different origin location
id_grid_y = pagesize[1] - id_grid_y
# ID grid on first page only
generate_id_grid(canv, id_grid_x, id_grid_y)
+ # Font settings for the copy number (printed under the datamatrix)
+ fontsize = 12
+ canv.setFont('Helvetica', fontsize)
+
for page_num in range(num_pages):
_add_corner_markers_and_bottom_bar(canv, pagesize)
@@ -243,7 +244,7 @@ def _generate_overlay(canv, pagesize, exam_id, copy_num, num_pages, id_grid_x,
canv.drawInlineImage(datamatrix, datamatrix_x, datamatrix_y_adjusted)
canv.drawString(
- datamatrix_x, datamatrix_y_adjusted - fontsize,
+ datamatrix_x, datamatrix_y_adjusted - (fontsize * 0.66),
f" # {copy_num}"
)
canv.showPage()
diff --git a/zesje/scans.py b/zesje/scans.py
index ae5da5d2ffc94945824b0fae1c26cb856938d33e..52e8d23e52877033b78c684979d861cd45a67690 100644
--- a/zesje/scans.py
+++ b/zesje/scans.py
@@ -4,12 +4,14 @@ import math
import os
from collections import namedtuple, Counter
from io import BytesIO
+from tempfile import SpooledTemporaryFile
import signal
import cv2
import numpy as np
-import PyPDF2
+from pikepdf import Pdf, PdfImage
from PIL import Image
+from wand.image import Image as WandImage
from pylibdmtx import pylibdmtx
from .database import db, Scan, Exam, Page, Student, Submission, Solution, ExamWidget
@@ -66,6 +68,8 @@ def _process_pdf(scan_id, app_config):
# Raises exception if zero or more than one scans found
scan = Scan.query.filter(Scan.id == scan_id).one()
+ report_progress('Importing PDF')
+
pdf_path = os.path.join(data_directory, 'scans', f'{scan.id}.pdf')
output_directory = os.path.join(data_directory, f'{scan.exam.id}_data')
@@ -75,7 +79,9 @@ def _process_pdf(scan_id, app_config):
report_error(f'Error while reading Exam metadata: {e}')
raise
- total = PyPDF2.PdfFileReader(open(pdf_path, "rb")).getNumPages()
+ with Pdf.open(pdf_path) as pdf_reader:
+ total = len(pdf_reader.pages)
+
failures = []
try:
for image, page in extract_images(pdf_path):
@@ -126,39 +132,129 @@ def exam_metadata(exam_id):
def extract_images(filename):
"""Yield all images from a PDF file.
- Adapted from https://stackoverflow.com/a/34116472/2217463
+ Tries to use PikePDF to extract the images from the given PDF.
+ If PikePDF is not able to extract the image from a page,
+ it continues to use Wand to flatten the rest of the pages.
+ """
+
+ with Pdf.open(filename) as pdf_reader:
+ use_wand = False
+
+ total = len(pdf_reader.pages)
+
+ for pagenr in range(total):
+ if not use_wand:
+ try:
+ # Try to use PikePDF, but catch any error it raises
+ img = extract_image_pikepdf(pagenr, pdf_reader)
+
+ except Exception:
+ # Fallback to Wand if extracting with PikePDF failed
+ use_wand = True
+
+ if use_wand:
+ img = extract_image_wand(pagenr, pdf_reader)
+
+ if img.mode == 'L':
+ img = img.convert('RGB')
+
+ yield img, pagenr+1
+
+
+def extract_image_pikepdf(pagenr, reader):
+ """Extracts an image as an array from the designated page
+
+ This method uses PikePDF to extract the image and only works
+ when there is a single image present on the page with the
+ same aspect ratio as the page.
+
+ We do not check for the actual size of the image on the page,
+ since this size depends on the draw instruction rather than
+ the embedded image object available to pikepdf.
+
+ Raises an error if not exactly image is present or the image
+ does not have the same aspect ratio as the page.
+
+ Parameters
+ ----------
+ pagenr : int
+ Page number to extract
+ reader : pikepdf.Pdf instance
+ The pdf reader to read the page from
+
+ Returns
+ -------
+ img_array : PIL Image
+ The extracted image data
+
+ Raises
+ ------
+ ValueError
+ if not exactly one image is found on the page or the image
+ does not have the same aspect ratio as the page
+ AttributeError
+ if no XObject or MediaBox is present on the page
+ """
+
+ page = reader.pages[pagenr]
+
+ xObject = page.Resources.XObject
+
+ if sum((xObject[obj].Subtype == '/Image')
+ for obj in xObject) != 1:
+ raise ValueError('Not exactly 1 image present on the page')
+
+ for obj in xObject:
+ if xObject[obj].Subtype == '/Image':
+ pdfimage = PdfImage(xObject[obj])
+
+ pdf_width = float(page.MediaBox[2] - page.MediaBox[0])
+ pdf_height = float(page.MediaBox[3] - page.MediaBox[1])
+
+ ratio_width = pdfimage.width / pdf_width
+ ratio_height = pdfimage.height / pdf_height
+
+ # Check if the aspect ratio of the image is the same as the
+ # aspect ratio of the page up to a 3% relative error
+ if abs(ratio_width - ratio_height) > 0.03 * ratio_width:
+ raise ValueError('Image has incorrect dimensions')
+
+ return pdfimage.as_pil_image()
+
+
+def extract_image_wand(pagenr, reader):
+ """Flattens a page from a PDF to an image array
- We raise if there are > 1 images / page
+ This method uses Wand to flatten the page and creates an image.
+
+ Parameters
+ ----------
+ pagenr : int
+ Page number to extract, starting at 0
+ reader : pikepdf.Pdf instance
+ The pdf reader to read the page from
+
+ Returns
+ -------
+ img_array : PIL Image
+ The extracted image data
"""
- reader = PyPDF2.PdfFileReader(open(filename, "rb"))
- total = reader.getNumPages()
- for pagenr in range(total):
- page = reader.getPage(pagenr)
- xObject = page['/Resources']['/XObject'].getObject()
-
- if sum((xObject[obj]['/Subtype'] == '/Image')
- for obj in xObject) > 1:
- raise RuntimeError(f'Page {pagenr + 1} contains more than 1 image,'
- 'likely not a scan')
-
- for obj in xObject:
- if xObject[obj]['/Subtype'] == '/Image':
- data = xObject[obj].getData()
- filter = xObject[obj]['/Filter']
-
- if filter == '/FlateDecode':
- size = (xObject[obj]['/Width'], xObject[obj]['/Height'])
- if xObject[obj]['/ColorSpace'] == '/DeviceRGB':
- mode = "RGB"
- else:
- mode = "P"
- img = Image.frombytes(mode, size, data)
- else:
- img = Image.open(BytesIO(data))
-
- if img.mode == 'L':
- img = img.convert('RGB')
- yield img, pagenr+1
+ page = reader.pages[pagenr]
+
+ page_pdf = Pdf.new()
+ page_pdf.pages.append(page)
+
+ with SpooledTemporaryFile() as page_file:
+
+ page_pdf.save(page_file)
+
+ with WandImage(blob=page_file._file.getvalue(), format='pdf', resolution=300) as page_image:
+ page_image.format = 'jpg'
+ img_array = np.asarray(bytearray(page_image.make_blob(format="jpg")), dtype=np.uint8)
+ img = Image.open(BytesIO(img_array))
+ img.load() # Load the data into the PIL image from the Wand image
+
+ return img
def write_pdf_status(scan_id, status, message):
diff --git a/zesje/statistics.py b/zesje/statistics.py
index a2382447d4c1d3faf8820a5d7bcf4571c9348634..ff74c4e6e9ee3a274e0d3cda0c3d29a609af5093 100644
--- a/zesje/statistics.py
+++ b/zesje/statistics.py
@@ -5,7 +5,7 @@ from sqlalchemy.orm.exc import NoResultFound
import numpy as np
import pandas
-from .database import Exam, Problem, Student
+from .database import Exam, Student
def solution_data(exam_id, student_id):
@@ -68,7 +68,7 @@ def full_exam_data(exam_id):
if not data:
# No students were assigned.
columns = []
- for problem in exam.problems.order_by(Problem.id):
+ for problem in exam.problems: # Sorted by problem.id
if not len(problem.feedback_options):
# There is no possible feedback for this problem.
continue