From ae77c14560d4b7919ebba672a74627768b320815 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 2 Aug 2018 20:45:58 +0200 Subject: [PATCH 1/5] Migrate xml component generation to source This avoids introducing an intermediate format for data, reduces the amount of code duplication, and simplifies the converter. For simplicity, and in view of upcoming changes this also removes the peer assessment. --- .gitlab-ci.yml | 2 - code/edx_components.py | 289 +++++++++++++++++------------ docker/python3.yaml | 3 +- scripts/converter.py | 282 +++++----------------------- w10_extensions/w10_assignments.md | 2 +- w11_extensions2/w11_assignments.md | 2 +- w12_manybody/w12_assignments.md | 2 +- w1_topointro/w1_assignments.md | 2 +- w2_majorana/w2_assignments.md | 4 +- w3_pump_QHE/w3_assignments.md | 4 +- w4_haldane/w4_assignments.md | 4 +- w5_qshe/w5_assignments.md | 4 +- w6_3dti/w6_assignments.md | 2 +- w7_defects/w7_assignments.md | 2 +- w8_general/w8_assignments.md | 2 +- w9_disorder/w9_assignments.md | 2 +- 16 files changed, 236 insertions(+), 372 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 211e35e..c0dcf70 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,6 @@ stages: execute_ipynbs: stage: execute - before_script: - - pip install notedown script: - export PYTHONPATH=$PYTHONPATH:${PWD}/code - export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 MKL_DYNAMIC=FALSE diff --git a/code/edx_components.py b/code/edx_components.py index c4352a1..d01b67e 100644 --- a/code/edx_components.py +++ b/code/edx_components.py @@ -1,20 +1,43 @@ import sys import os +import secrets + +from xml.etree.ElementTree import Element, SubElement +from xml.etree import ElementTree + from hashlib import md5 +from IPython import display +import feedparser + module_dir = os.path.dirname(__file__) sys.path.extend(module_dir) -from IPython import display -import feedparser -__all__ = ['MoocVideo', 'PreprintReference', 'MoocDiscussion', - 'MoocCheckboxesAssessment', 'MoocMultipleChoiceAssessment', - 'MoocPeerAssessment', 'MoocSelfAssessment'] +__all__ = [ + 'MoocVideo', 'PreprintReference', 'MoocDiscussion', + 'MoocCheckboxesAssessment', 'MoocMultipleChoiceAssessment', + 'MoocSelfAssessment' +] + + +def _add_solution(el, text): + """Add a solution xml to a problem.""" + sol = SubElement(el, 'solution') + div = SubElement(sol, 'div') + div.attrib['class'] = 'detailed-solution' + title = SubElement(div, 'p') + title.text = 'Explanation' + content = SubElement(div, 'p') + content.text = text + class MoocComponent: - def __repr__(self): - return '{0}(**{1})'.format(self.__class__.__name__, repr(self.param)) + def _repr_mimebundle_(self, include, exclude): + return { + 'application/vnd.edx.olxml+xml': + ElementTree.tostring(self.xml, encoding='unicode'), + } class MoocVideo(MoocComponent, display.YouTubeVideo): @@ -22,27 +45,34 @@ class MoocVideo(MoocComponent, display.YouTubeVideo): download_track='true', download_video='true', show_captions='true', **kwargs): """A video component of an EdX mooc embeddable in IPython notebook.""" - tmp = locals() - del tmp['kwargs'], tmp['self'], tmp['__class__'] - del tmp['id'], tmp['src_location'], tmp['res'] - kwargs.update(tmp) - kwargs['youtube_id_1_0'] = id - kwargs['youtube'] = "1.00:" + id - - # Add source if provided - loc = ("http://delftxdownloads.tudelft.nl/" - "TOPOCMx-QuantumKnots/TOPOCMx-{0}-video.{1}.mp4") + + self.xml = Element('video', attrib=dict( + youtube=f'1.00:{id}', + youtube_id_1_0=id, + display_name=display_name, + download_track=download_track, + download_video=download_video, + show_captions=show_captions, + **kwargs, + )) + if src_location is not None: - kwargs['source'] = loc.format(src_location, res) + self.xml.attrib['source'] = ( + "http://delftxdownloads.tudelft.nl/TOPOCMx-QuantumKnots/" + f"TOPOCMx-{src_location}-video.{res}.mp4" + ) - self.param = kwargs super().__init__(id, rel=0, cc_load_policy=1) def _repr_html_(self): orig = super()._repr_html_() - return ('
{}
' - .format(orig.replace('{}' + .format(orig.replace( + '' - s += '

http://arxiv.org/abs/%s
' % (ind, ind) + s += f'

http://arxiv.org/abs/%s
' % (ind, ind) s += '

' s += ", ".join(author.name for author in data['entries'][0]['authors']) @@ -78,49 +108,26 @@ class PreprintReference: return s -class MoocPeerAssessment(MoocComponent): - def __init__(self, must_grade=5, must_be_graded_by=3, due=9, review_due=16, - url_name=None, **kwargs): - - self.placeholder = ('

Read one of the above papers and see how it is\n' - 'related to the current topic.

\n' - '

In the live version of the course, you would ' - 'need to write a summary which is then assessed by ' - 'your peers.

') - - tmp = locals() - del tmp['kwargs'], tmp['self'] - kwargs.update(tmp) - - with open(module_dir + '/xmls/openassessment_peer.xml', 'r') as content_file: - openassessment_peer = content_file.read() - - kwargs['openassessment_peer'] = openassessment_peer - - self.param = kwargs - - def _repr_html_(self): - return self.placeholder - - class MoocSelfAssessment(MoocComponent): - def __init__(self, due=9, review_due=16, url_name=None, **kwargs): - - tmp = locals() - del tmp['kwargs'], tmp['self'] - kwargs.update(tmp) + def __init__(self): + self.placeholder = ( + '

MoocSelfAssessment description

\n' + '

In the live version of the course, you would ' + 'need to share your solution and grade yourself.' + '

' + ) - self.placeholder = ('

MoocSelfAssessment description

\n' - '

In the live version of the course, you would ' - 'need to share your solution and grade yourself.' - '

') + content_filename = module_dir + '/xmls/openassessment_self.xml' + with open(content_filename, 'r') as content_file: + self.xml = xml = ElementTree.fromstring(content_file.read()) - with open(module_dir + '/xmls/openassessment_self.xml', 'r') as content_file: - openassessment_self = content_file.read() + assessment = xml.find('assessments').find('assessment') - kwargs['openassessment_self'] = openassessment_self + xml.attrib['submission_start'] = '2001-12-31T10:00:00Z' + xml.attrib['submission_due'] = '2100-12-31T10:00:00Z' - self.param = kwargs + assessment.attrib['start'] = '2001-12-31T10:00:00Z' + assessment.attrib['due'] = '2100-12-31T10:00:00Z' def _repr_html_(self): return self.placeholder @@ -128,7 +135,7 @@ class MoocSelfAssessment(MoocComponent): class MoocCheckboxesAssessment(MoocComponent): def __init__(self, question, answers, correct_answers, max_attempts=2, - display_name="Question", **kwargs): + display_name="Question", explanation=None): """ MoocCheckboxesAssessment component @@ -140,31 +147,51 @@ class MoocCheckboxesAssessment(MoocComponent): correct_answers : list of int """ - tmp = locals() - del tmp['kwargs'], tmp['self'] - kwargs.update(tmp) + self.xml = xml = Element('problem', attrib=dict( + display_name=display_name, + max_attempts=str(max_attempts), + )) + + SubElement(xml, 'p', text='question') + SubElement(xml, 'p', text='Select the answers that match') + + sub = SubElement(xml, 'choiceresponse') + sub = SubElement(sub, 'checkboxgroup') + sub.attrib['label'] = "Select the answers that match" + sub.attrib['direction'] = "vertical" + + for i, ans in enumerate(answers): + choice = SubElement(sub, 'choice') + if i in correct_answers: + choice.attrib['correct'] = 'true' + else: + choice.attrib['correct'] = 'false' + choice.text = ans + + if explanation is not None: + _add_solution(xml, explanation) + + self.question = question + self.answers = answers + self.correct_answers = correct_answers + self.explanation = explanation - self.param = kwargs - - def _get_html_repr(self): - param = self.param - s = '

%s

' % param['question'] + def _repr_html_(self): + s = '

%s

' % self.question s += '
' - s += '
'.join(param['answers']) + s += '
'.join(self.answers) s += '
' - answer = 'The correct answer{0}:
'.format('s are' if - len(param['correct_answers']) > 1 - else ' is') + answer = 'The correct answer{0}:
'.format( + 's are' if len(self.correct_answers) > 1 else ' is' + ) tmp = [] - for i in param['correct_answers']: - tmp.append(param['answers'][i]) + for i in self.correct_answers: + tmp.append(self.answers[i]) answer += ', '.join(t for t in tmp) answer += '.' - try: - answer += '
' + param['explanation'] + '' - except KeyError: - pass + if self.explanation is not None: + answer += '
' + self.explanation + '' s += """
""" - return s.format(md5(repr(self).encode('utf-8')).hexdigest(), answer) - - def _repr_html_(self): - return self._get_html_repr() + return s.format(secrets.token_urlsafe(20), answer) class MoocMultipleChoiceAssessment(MoocComponent): def __init__(self, question, answers, correct_answer, max_attempts=2, - display_name="Question", **kwargs): + display_name="Question", explanation=None): """ MoocMultipleChoiceAssessment component @@ -196,26 +220,46 @@ class MoocMultipleChoiceAssessment(MoocComponent): correct_answers : int """ - tmp = locals() - del tmp['kwargs'], tmp['self'] - kwargs.update(tmp) + self.xml = xml = Element('problem', attrib=dict( + display_name=display_name, + max_attempts=str(max_attempts), + )) + + SubElement(xml, 'p', text=question) + SubElement(xml, 'p', text='Please select correct answer') + + sub = SubElement(xml, 'multiplechoiceresponse') + sub = SubElement(sub, 'choicegroup') + sub.attrib['label'] = "Please select correct answer" + sub.attrib['type'] = "MultipleChoice" + + for i, ans in enumerate(answers): + choice = SubElement(sub, 'choice') + if i == correct_answer: + choice.attrib['correct'] = 'true' + else: + choice.attrib['correct'] = 'false' + choice.text = ans + + if explanation is not None: + _add_solution(xml, explanation) + + self.question = question + self.answers = answers + self.correct_answer = correct_answer + self.explanation = explanation - self.param = kwargs - - def _get_html_repr(self): - param = self.param - s = '

%s

' % param['question'] - s+= '
' - for ans in param['answers']: - s += '' % ans + ans + '
' - s+= '
' + def _repr_html_(self): + s = '

%s

' % self.question + s += '
' + for ans in self.answers: + s += f'{ans}
' + s += '
' answer = 'The correct answer is:
' - answer += param['answers'][param['correct_answer']] - try: - answer += '
' + param['explanation'] + '' - except KeyError: - pass + answer += self.answers[self.correct_answer] + if self.explanation is not None: + answer += f'
' + self.explanation + '' s += """ """ - return s.format(md5(repr(self).encode('utf-8')).hexdigest(), answer) - - def _repr_html_(self): - return self._get_html_repr() + return s.format(secrets.token_urlsafe(20), answer) class MoocDiscussion(MoocComponent): - def __init__(self, discussion_category, discussion_target, display_name=None, - discussion_id=None, **kwargs): - - tmp = locals() - del tmp['kwargs'], tmp['self'] - kwargs.update(tmp) - - if display_name is None: - kwargs['display_name'] = discussion_target - - if discussion_id is None: - kwargs['discussion_id'] = md5(discussion_category.encode('utf-8') + discussion_target.encode('utf-8')).hexdigest() - - self.param = kwargs + def __init__( + self, discussion_category, discussion_target, display_name=None, + discussion_id=None, **kwargs, + ): + discussion_id = md5( + discussion_category.encode('utf-8') + + discussion_target.encode('utf-8') + ).hexdigest() + self.xml = Element('discussion', attrib=dict( + discussion_category=discussion_category, + discussion_target=discussion_target, + display_name=(display_name or discussion_target), + discussion_id=discussion_id, + url_name=discussion_id, + **kwargs + )) + self.display_name = (display_name or discussion_target) def _repr_html_(self): - return "

Discussion entitled '{0}' is available in the online version of the course.

".format(self.param['display_name']) + return ( + f"

Discussion {self.display_name}" + " is available in the EdX version of the course.

" + ) diff --git a/docker/python3.yaml b/docker/python3.yaml index 558c561..290744a 100644 --- a/docker/python3.yaml +++ b/docker/python3.yaml @@ -5,11 +5,12 @@ dependencies: - python=3.6 - matplotlib=2.0.0 - kwant=1.3* # Untill v1.3 is out - - scipy=0.18* + - scipy=1.1* - holoviews=1.7* - feedparser=5.2* - sympy=1.0* - pelican=3.7* + - ipython>=6.1.0 - notebook - markdown - notedown diff --git a/scripts/converter.py b/scripts/converter.py index 11118cb..4b85dc7 100755 --- a/scripts/converter.py +++ b/scripts/converter.py @@ -43,8 +43,7 @@ url = ( "https://cdnjs.cloudflare.com/ajax/libs" "/iframe-resizer/3.5.14/iframeResizer.min.js" ) -response = urllib.request.urlopen(url) -js = response.read().decode('utf-8') +js = urllib.request.urlopen(url).read().decode('utf-8') IFRAME_TEMPLATE = r"""

diff --git a/edx_skeleton/html/subsec_00_01_00_out_00.xml b/edx_skeleton/html/subsec_00_01_00_out_00.xml deleted file mode 100644 index 74d63f6..0000000 --- a/edx_skeleton/html/subsec_00_01_00_out_00.xml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/edx_skeleton/html/subsec_05_03_00_out_00.html b/edx_skeleton/html/subsec_05_03_00_out_00.html deleted file mode 100644 index b940105..0000000 --- a/edx_skeleton/html/subsec_05_03_00_out_00.html +++ /dev/null @@ -1 +0,0 @@ -

diff --git a/edx_skeleton/html/subsec_05_03_00_out_00.xml b/edx_skeleton/html/subsec_05_03_00_out_00.xml deleted file mode 100644 index 76c0fea..0000000 --- a/edx_skeleton/html/subsec_05_03_00_out_00.xml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/edx_skeleton/problem/.keep b/edx_skeleton/problem/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/edx_skeleton/sequential/.keep b/edx_skeleton/sequential/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/edx_skeleton/sequential/subsec_00_01.xml b/edx_skeleton/sequential/subsec_00_01.xml deleted file mode 100644 index 76754da..0000000 --- a/edx_skeleton/sequential/subsec_00_01.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/edx_skeleton/sequential/subsec_05_03.xml b/edx_skeleton/sequential/subsec_05_03.xml deleted file mode 100644 index dea05cb..0000000 --- a/edx_skeleton/sequential/subsec_05_03.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/edx_skeleton/vertical/.keep b/edx_skeleton/vertical/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/edx_skeleton/vertical/subsec_00_01_00.xml b/edx_skeleton/vertical/subsec_00_01_00.xml deleted file mode 100644 index 2ba59c6..0000000 --- a/edx_skeleton/vertical/subsec_00_01_00.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/edx_skeleton/vertical/subsec_05_03_00.xml b/edx_skeleton/vertical/subsec_05_03_00.xml deleted file mode 100644 index dc7bda9..0000000 --- a/edx_skeleton/vertical/subsec_05_03_00.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/edx_skeleton/video/.keep b/edx_skeleton/video/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/converter.py b/scripts/converter.py index 4b85dc7..9a8a982 100755 --- a/scripts/converter.py +++ b/scripts/converter.py @@ -13,7 +13,7 @@ from types import SimpleNamespace import shutil import subprocess import urllib.request -from xml.etree.ElementTree import Element, SubElement +from xml.etree.ElementTree import SubElement from xml.etree import ElementTree from xml.dom import minidom @@ -209,9 +209,6 @@ def prettify(elem): def save_xml(xml, path): - if os.path.exists(path): - print("Not overwriting existing file,", path) - return with io.open(path, 'w', encoding='utf-8') as f: text = prettify(xml) text = re.sub(r"\$\$(.*?)\$\$", r"\[\1\]", text) @@ -268,13 +265,13 @@ def convert_unit(unit, date): if normal_cells: html = convert_normal_cells(normal_cells) - unit_output.append(['html', html]) + unit_output.append(html) normal_cells = [] - unit_output.append([xml.tag, xml]) + unit_output.append(xml) if normal_cells: html = convert_normal_cells(normal_cells) - unit_output.append(['html', html]) + unit_output.append(html) normal_cells = [] return unit_output @@ -296,113 +293,67 @@ def converter(mooc_folder, args, content_folder=None): os.path.join(target, 'html/edx/figures')) html_folder = os.path.join(target, 'html/edx') - # Basic info - info_org = 'DelftX' - info_course = 'TOPOCMx' - info_display_name = 'Topology in Condensed Matter: Tying Quantum Knots' - info_run = 'course' - info_start = "2016-02-08T10:00:00Z" - # Temporary locations - dirpath = tempfile.mkdtemp() + '/' + info_run + dirpath = tempfile.mkdtemp() + '/course' if args.debug: print('Temporary path: ', dirpath) - # Copying edx_skeleton skeleton = mooc_folder + '/edx_skeleton' - if os.path.exists(skeleton): - shutil.copytree(skeleton, dirpath) - else: - raise RuntimeError('No edx skeleton!') + shutil.copytree(skeleton, dirpath) # Loading data from syllabus syllabus_nb = os.path.join(content_folder, 'syllabus.ipynb') data = parse_syllabus(syllabus_nb, content_folder) - # saving syllabus - syllabus = split_into_units(syllabus_nb)[0] - cell = syllabus.cells[1] - cell['source'] = re.sub(r"(? Date: Fri, 3 Aug 2018 16:15:23 +0200 Subject: [PATCH 3/5] remove an obsolete converter option --- .gitlab-ci.yml | 2 +- scripts/converter.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0dcf70..462ff5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,7 @@ mirror to github: edx archive: stage: build - script: ./scripts/converter.py --silent ./generated/with_output + script: ./scripts/converter.py ./generated/with_output artifacts: paths: - generated/import_to_edx.tar.gz diff --git a/scripts/converter.py b/scripts/converter.py index 9a8a982..f35042d 100755 --- a/scripts/converter.py +++ b/scripts/converter.py @@ -390,21 +390,17 @@ def converter(mooc_folder, args, content_folder=None): shutil.rmtree(dirpath) -def warn_about_status(mooc_folder, silent=False): +def warn_about_status(mooc_folder): git = f'git --git-dir={mooc_folder}/.git --work-tree={mooc_folder}/ ' status = subprocess.check_output(git + "status", shell=True).decode('utf-8').split("\n")[0] if "On branch master" not in status: print("Not on master branch, do not upload to edx.\n", "Press Enter to continue.") - if not silent: - input() return if subprocess.check_output(git + "diff", shell=True): print("Some files are modified, do not upload to edx.\n", "Press Enter to continue.") - if not silent: - input() def main(): @@ -415,8 +411,6 @@ def main(): help='debugging flag') parser.add_argument('-o', '--open', action='store_true', help='opening uncompressed folder with files') - parser.add_argument('-s', '--silent', action='store_true', - help='opening uncompressed folder with files') args = parser.parse_args() @@ -426,7 +420,7 @@ def main(): print('Path to mooc folder:', mooc_folder) print('Path to notebooks:', args.source) - warn_about_status(mooc_folder, args.silent) + warn_about_status(mooc_folder) converter(mooc_folder, args, content_folder=args.source) -- GitLab From cc34536c7fc964149e9a4f1989266ce12afd7486 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Fri, 3 Aug 2018 17:41:05 +0200 Subject: [PATCH 4/5] remove survey link --- syllabus.md | 1 - 1 file changed, 1 deletion(-) diff --git a/syllabus.md b/syllabus.md index f0761f2..6dddbb0 100644 --- a/syllabus.md +++ b/syllabus.md @@ -2,7 +2,6 @@ * **Before you begin** * [About this course](w0_background/intro.ipynb) - * Starting survey * **Topology in toy models** * [Hamiltonians, topology, and symmetry](w1_topointro/0d.ipynb) * [Bulk-edge correspondence in the Kitaev chain](w1_topointro/1D.ipynb) -- GitLab From 972a195e11c269bcd8d09153bcbc46dd99320b67 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Fri, 3 Aug 2018 17:41:58 +0200 Subject: [PATCH 5/5] further shrink the converting code --- scripts/converter.py | 109 ++++++--------------------------------- scripts/converter_ocw.py | 17 +++--- 2 files changed, 21 insertions(+), 105 deletions(-) diff --git a/scripts/converter.py b/scripts/converter.py index f35042d..547214b 100755 --- a/scripts/converter.py +++ b/scripts/converter.py @@ -11,11 +11,9 @@ import tempfile from time import strptime from types import SimpleNamespace import shutil -import subprocess import urllib.request from xml.etree.ElementTree import SubElement from xml.etree import ElementTree -from xml.dom import minidom import nbformat from nbformat import v4 as current @@ -88,17 +86,16 @@ def date_to_edx(date, add_days=0): return date -def parse_syllabus(syllabus_file, content_folder='', parse_all=False): +def parse_syllabus(syllabus_file, content_folder=''): # loading raw syllabus - syll = split_into_units(syllabus_file)[0] - cell = syll.cells[1] + syll = nbformat.read(syllabus_file, as_version=4).cells[0].source section = '^\* \*\*(?P
.*)\*\*$' subsection = '^ \* \[(?P.*)\]\((?P<filename>.*)\)$' syllabus_line = section + '|' + subsection syllabus = [] - for line in cell.source.split('\n'): + for line in syll.split('\n'): match = re.match(syllabus_line, line) if match is None: continue @@ -111,10 +108,8 @@ def parse_syllabus(syllabus_file, content_folder='', parse_all=False): data = SimpleNamespace(category='main', chapters=[]) for i, section in enumerate(syllabus): - if not parse_all: - # Don't convert sections with no release date. - if section[1] is None: - continue + if section[1] is None: + continue # creating chapter chapter = SimpleNamespace(category='chapter', sequentials=[]) @@ -138,7 +133,7 @@ def parse_syllabus(syllabus_file, content_folder='', parse_all=False): return data -def split_into_units(nb_name, include_header=True): +def split_into_units(nb_name): """Split notebook into units where top level headings occur.""" nb = nbformat.read(nb_name, as_version=4) @@ -169,8 +164,6 @@ def split_into_units(nb_name, include_header=True): units.append(current.new_notebook(metadata={ 'name': nb_name })) - if include_header: - units[-1].cells.append(cell) else: if not units: # We did not encounter a title yet. continue @@ -179,43 +172,6 @@ def split_into_units(nb_name, include_header=True): return units -def export_unit_to_html(unit, export_html=exportHtml): - """Export unit into html format.""" - (body, resources) = export_html.from_notebook_node(unit) - body = re.sub(r'\\begin\{ *equation *\}', '\[', body) - body = re.sub(r'\\end\{ *equation *\}', '\]', body) - return body - - -def make_filename_valid(string): - cleaned_up_filename = re.sub(r'[/\\:$%*?,"<>| ]', '', string) - return cleaned_up_filename - - -def save_html(body, target_path): - """Save html body into edX course.""" - with io.open(target_path, 'w', encoding='utf-8') as f: - f.write(body) - - -def prettify(elem): - """Return a pretty-printed XML string for the Element.""" - rough_string = ElementTree.tostring(elem, 'utf-8') - reparsed = minidom.parseString(rough_string) - output = reparsed.toprettyxml(indent=" ") - - # output = str(output) - return output[output.find("\n")+1:] - - -def save_xml(xml, path): - with io.open(path, 'w', encoding='utf-8') as f: - text = prettify(xml) - text = re.sub(r"\$\$(.*?)\$\$", r"\[\1\]", text) - text = re.sub(r"\$(.*?)\$", r"\(\1\)", text) - f.write(text) - - def convert_normal_cells(normal_cells): """ Convert normal_cells into html. """ for cell in normal_cells: @@ -223,7 +179,7 @@ def convert_normal_cells(normal_cells): cell.source = re.sub(r'\\begin\{ *equation *\}', '\[', cell.source) cell.source = re.sub(r'\\end\{ *equation *\}', '\]', cell.source) tmp = current.new_notebook(cells=normal_cells) - html = export_unit_to_html(tmp) + html = exportHtml.from_notebook_node(tmp)[0] return html @@ -295,8 +251,6 @@ def converter(mooc_folder, args, content_folder=None): # Temporary locations dirpath = tempfile.mkdtemp() + '/course' - if args.debug: - print('Temporary path: ', dirpath) skeleton = mooc_folder + '/edx_skeleton' shutil.copytree(skeleton, dirpath) @@ -328,8 +282,7 @@ def converter(mooc_folder, args, content_folder=None): elif chapter.url != 'sec_00': sequential_xml.attrib['format'] = "Self-check" - units = split_into_units(sequential.source_notebook, - include_header=False) + units = split_into_units(sequential.source_notebook) for i, unit in enumerate(units): vertical_url = sequential.url + f'_{i:02}' @@ -352,16 +305,15 @@ def converter(mooc_folder, args, content_folder=None): html_path = os.path.join(html_folder, out_url + '.html') - save_html(out, html_path) + with open(html_path, 'w') as f: + f.write(out) html_path = os.path.join(dirpath, 'html', out_url + '.html') - save_html( - IFRAME_TEMPLATE.format( + with open(html_path, 'w') as f: + f.write(IFRAME_TEMPLATE.format( id=out_url, url=url, js=js - ), - html_path - ) + )) else: # adding video subelement @@ -369,7 +321,8 @@ def converter(mooc_folder, args, content_folder=None): if 'url_name' not in out.attrib: out.attrib['url_name'] = out_url - save_xml(xml_course, course_xml_path) + with open(course_xml_path, 'w') as f: + f.write(ElementTree.tostring(xml_course, encoding='unicode')) # Creating tar tar_filepath = os.path.join(target, 'import_to_edx.tar.gz') @@ -377,50 +330,18 @@ def converter(mooc_folder, args, content_folder=None): tar.add(dirpath, arcname='') tar.close() - # Some debugging - if args.debug: - shutil.copytree(dirpath, target + '/files') - if args.open: - if not args.debug: - print('--open flag works only with debug') - else: - subprocess.check_call(['nautilus', '--', target + '/files']) - # Cleaning shutil.rmtree(dirpath) -def warn_about_status(mooc_folder): - git = f'git --git-dir={mooc_folder}/.git --work-tree={mooc_folder}/ ' - status = subprocess.check_output(git + "status", - shell=True).decode('utf-8').split("\n")[0] - if "On branch master" not in status: - print("Not on master branch, do not upload to edx.\n", - "Press Enter to continue.") - return - if subprocess.check_output(git + "diff", shell=True): - print("Some files are modified, do not upload to edx.\n", - "Press Enter to continue.") - - def main(): mooc_folder = os.path.join(os.path.dirname(__file__), os.path.pardir) parser = argparse.ArgumentParser() parser.add_argument('source', nargs='?', help='folder to convert') - parser.add_argument('-d', '--debug', action='store_true', - help='debugging flag') - parser.add_argument('-o', '--open', action='store_true', - help='opening uncompressed folder with files') - args = parser.parse_args() - if args.debug: - msg = 'Debug mode : folder {} will contain uncompressed data.' - print(msg.format(mooc_folder + '/generated/files')) - print('Path to mooc folder:', mooc_folder) print('Path to notebooks:', args.source) - warn_about_status(mooc_folder) converter(mooc_folder, args, content_folder=args.source) diff --git a/scripts/converter_ocw.py b/scripts/converter_ocw.py index a669af8..38b07af 100644 --- a/scripts/converter_ocw.py +++ b/scripts/converter_ocw.py @@ -7,10 +7,8 @@ import shutil from traitlets.config import Config from nbconvert import HTMLExporter from nbconvert.filters.markdown import markdown2html_pandoc -from converter import (export_unit_to_html, - mooc_folder, +from converter import (mooc_folder, parse_syllabus, - save_html, scripts_path, split_into_units, url) @@ -55,24 +53,21 @@ generated_ipynbs = os.path.join(mooc_folder, 'generated/with_output') # Loading data from syllabus syllabus_nb = os.path.join(generated_ipynbs, 'syllabus.ipynb') -data = parse_syllabus(syllabus_nb, generated_ipynbs, parse_all=True) - -# saving syllabus -syllabus = split_into_units(syllabus_nb)[0] +data = parse_syllabus(syllabus_nb, generated_ipynbs) for chapter in data.chapters: chap_num = int(chapter.url[-2:]) for sequential in chapter.sequentials: - units = split_into_units(sequential.source_notebook, - include_header=False) + units = split_into_units(sequential.source_notebook) folder, fname = sequential.source_notebook.split('/')[-2:] for i, unit in enumerate(units): fname = fname.replace('.ipynb', '') new_fname = '{}_{}'.format(fname, i) new_path = os.path.join(mooc_folder, output_dir, folder, new_fname + '.html') os.makedirs(os.path.dirname(new_path), exist_ok=True) - html = export_unit_to_html(unit, exportHtml) - save_html(html, new_path) + html = exportHtml.from_notebook_node(unit)[0] + with open(new_path, 'w') as f: + f.write(html) with open('website_assets/iframes.txt', 'a') as f: ID = '{}_{}'.format(folder, new_fname) -- GitLab