From 2ad63c8253974d72d0e1c2f9f1520731760347e1 Mon Sep 17 00:00:00 2001 From: Davide Olianas Date: Sun, 17 Jan 2016 21:45:50 +0100 Subject: [PATCH 1/2] Allow loading LaTeX macros from text files --- Readme.md | 16 +++++++-- math.py | 76 +++++++++++++++++++++++++++++++++++++++++ mathjax_script_template | 2 +- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index bbe9020..490bb95 100644 --- a/Readme.md +++ b/Readme.md @@ -52,6 +52,14 @@ the math output in the summary. To restore math, [BeautifulSoup4](https://pypi.python.org/pypi/beautifulsoup4/4.4.0) is used. If it is not installed, no summary processing will happen. +### Load custom LaTeX macros + +If you use the same macros over and over, it's a good idea to not repeat yourself defining them in multiple Markdown or reStructuredText documents. What you can do instead is tell the plugin absolute paths for text files containing macro definitions. + +If the same macro name has multiple definitions, the last one is used and a warning is printed to stdout. + +See below in the Usage section for examples. + Usage ----- ### Templates @@ -99,11 +107,15 @@ Requires [BeautifulSoup4](http://www.crummy.com/software/BeautifulSoup/bs4/doc/) **Default Value**: `False` * `message_style`: [string] This value controls the verbosity of the messages in the lower left-hand corner. Set it to `None` to eliminate all messages. **Default Value**: normal +* `macros`: [list] each element of the list is a [string] containing the absolute path to a file with macro definitions. +**Default Value**: `[]` #### Settings Examples -Make math render in blue and displaymath align to the left: +Make math render in blue, displaymath align to the left and load macros from `/home/user/latex-macros.tex`: + + macros = ['/home/user/latex-macros.tex'] + MATH_JAX = {'color': 'blue', 'align': 'left', 'macros': macros} - MATH_JAX = {'color':'blue','align':left} Use the [color](http://docs.mathjax.org/en/latest/tex.html#color) and [mhchem](http://docs.mathjax.org/en/latest/tex.html#mhchem) extensions: diff --git a/math.py b/math.py index f1c492e..f532af1 100644 --- a/math.py +++ b/math.py @@ -71,6 +71,7 @@ def process_settings(pelicanobj): mathjax_settings['process_summary'] = BeautifulSoup is not None # will fix up summaries if math is cut off. Requires beautiful soup mathjax_settings['force_tls'] = 'false' # will force mathjax to be served by https - if set as False, it will only use https if site is served using https mathjax_settings['message_style'] = 'normal' # This value controls the verbosity of the messages in the lower left-hand corner. Set it to "none" to eliminate all messages + mathjax_settings['macros'] = '{}' # Source for MathJax mathjax_settings['source'] = "'//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML'" @@ -192,8 +193,83 @@ def process_settings(pelicanobj): mathjax_settings[key] = value + if key == 'macros': + text_lines = [] + macros = parse_tex_macro(*value) + for macro in macros: + if 'args' in macro.keys(): + # number of arguments > 1 + text_lines.append("{0}: ['{1}', {2}]".format(macro['name'], macro['definition'], macro['args'])) + else: + text_lines.append("{0}: '{1}'".format(macro['name'], macro['definition'])) + mathjax_settings[key] = '{' + ", ".join(text_lines) + '}' + + return mathjax_settings +def parse_tex_macro(*args): + """Returns a list of from input files. + + Each argument has to be an absolute path to a file. TeX macro definitions + are read and each one is translated to a dictionary containing the name + without backslash and the definition; if arguments are present, their + number is added too. + + If a macro is defined multiple times, a warning is printed to stdout. + The last definition is used. + + Backslashes in the definition are added in order to ensure the proper + form in the final html page. + + Example: + > [{'name': 'pd', + 'definition': '\\\\\\\\frac{\\\\\\\\partial #1}{\\\\\\\\partial #2}', + 'args': 2}, + {'name': 'R', 'definition': '\\\\\\\\mathbb{R}'}] + """ + temp_macros = [] + for arg in args: + with open(arg, 'rt') as input_file: + lines = input_file.read().splitlines() + for i, command in enumerate(lines): + splitted = command.split('{') + name_number = splitted[1].split('}') + name = name_number[0].strip('\\') + # for the definition, remove the last character from the last string which is } + # remember that strings are immutable objects in python + last_def_token = splitted[-1][:-1] + splitted_def = splitted[2:-1] + [last_def_token] + complete_def = '{'.join(splitted_def).replace('\\','\\\\\\\\') + final_command = {'line': i+1, 'file': arg, 'name': name, 'definition': complete_def} + if name_number[1]: + # the number of arguments is defined, therefore name_number[1] is not null string + args_number = name_number[1].lstrip('[').rstrip(']') + final_command['args'] = args_number + temp_macros.append(final_command) + names = [] + for macro in temp_macros: + names.append(macro['name']) + import collections + seen = set() + duplicate_indices = [names.index(item) for item, count in collections.Counter(names).items() if count > 1] + if len(duplicate_indices) > 0: + duplicates = [] + for i in duplicate_indices: + name = temp_macros[i]['name'] + duplicate = {'name': name, 'where':[]} + for j in temp_macros: + if j['name'] == name: + duplicate['where'].append((j['line'], j['file'])) + duplicates.append(duplicate) + exception_text = "WARNING: macros where defined more than once, the last definition is used\n" + for dup in duplicates: + exception_text += "Macro {} defined in\n".format(dup['name'].strip('\\')) + for place in dup['where']: + exception_text += "{}, line {}\n".format(place[1], place[0]) + print(exception_text) + # remove line and file keys from temp_macros (added for debug in case of duplicates) + return [{k: v for k, v in elem.items() if k in ['name', 'definition', 'args']} for elem in temp_macros] + def process_summary(article): """Ensures summaries are not cut off. Also inserts mathjax script so that math will be rendered""" diff --git a/mathjax_script_template b/mathjax_script_template index 590755b..cfafaca 100644 --- a/mathjax_script_template +++ b/mathjax_script_template @@ -18,7 +18,7 @@ if (!document.getElementById('mathjaxscript_pelican_#%@#$@#')) {{ mathjaxscript[(window.opera ? "innerHTML" : "text")] = "MathJax.Hub.Config({{" + " config: ['MMLorHTML.js']," + - " TeX: {{ extensions: ['AMSmath.js','AMSsymbols.js','noErrors.js','noUndefined.js'{tex_extensions}], equationNumbers: {{ autoNumber: 'AMS' }} }}," + + " TeX: {{ extensions: ['AMSmath.js','AMSsymbols.js','noErrors.js','noUndefined.js'{tex_extensions}], equationNumbers: {{ autoNumber: 'AMS' }}, Macros: {macros} }}," + " jax: ['input/TeX','input/MathML','output/HTML-CSS']," + " extensions: ['tex2jax.js','mml2jax.js','MathMenu.js','MathZoom.js']," + " displayAlign: '"+ align +"'," + From 7dd529fe57365aa92a06a9cf1604fdaacc5cf0cd Mon Sep 17 00:00:00 2001 From: Davide Olianas Date: Wed, 27 Jan 2016 13:49:08 +0100 Subject: [PATCH 2/2] math.py rename and tests for macro loading functions math.py is renamed to avoid collisions with the standard library module. The function handling macros has been refactored for better testability and maintainability. --- Contribute.md | 7 ++ __init__.py | 2 +- devrequirements.txt | 11 +++ latex-commands-example.tex | 3 + math.py => render_math.py | 139 +++++++++++++++++++++++++------------ test_math.py | 84 ++++++++++++++++++++++ 6 files changed, 199 insertions(+), 47 deletions(-) create mode 100644 Contribute.md create mode 100644 devrequirements.txt create mode 100644 latex-commands-example.tex rename math.py => render_math.py (81%) create mode 100644 test_math.py diff --git a/Contribute.md b/Contribute.md new file mode 100644 index 0000000..f735753 --- /dev/null +++ b/Contribute.md @@ -0,0 +1,7 @@ +Local development +================= + +1. Create a virtual environment and install packages via `pip` from `devrequirements.txt` +2. Add your tests to `test_math.py` +3. In the CLI run `python -m unittest discover -t ..` + diff --git a/__init__.py b/__init__.py index 2ac15dd..2eb93c0 100644 --- a/__init__.py +++ b/__init__.py @@ -1 +1 @@ -from .math import * +from .render_math import * diff --git a/devrequirements.txt b/devrequirements.txt new file mode 100644 index 0000000..a6dbf44 --- /dev/null +++ b/devrequirements.txt @@ -0,0 +1,11 @@ +Jinja2==2.8 +MarkupSafe==0.23 +Pygments==2.1 +Unidecode==0.04.19 +blinker==1.4 +docutils==0.12 +feedgenerator==1.7 +pelican==3.6.3 +python-dateutil==2.4.2 +pytz==2015.7 +six==1.10.0 diff --git a/latex-commands-example.tex b/latex-commands-example.tex new file mode 100644 index 0000000..add904a --- /dev/null +++ b/latex-commands-example.tex @@ -0,0 +1,3 @@ +\newcommand{\pp}[2]{\frac{\partial #1}{\partial #2}} +\newcommand{\bb}{\pi R} +\newcommand{\bc}{\pi r} \ No newline at end of file diff --git a/math.py b/render_math.py similarity index 81% rename from math.py rename to render_math.py index f532af1..c5cf33c 100644 --- a/math.py +++ b/render_math.py @@ -30,6 +30,7 @@ the math. See README for more details. """ +import collections import os import sys @@ -195,7 +196,7 @@ def process_settings(pelicanobj): if key == 'macros': text_lines = [] - macros = parse_tex_macro(*value) + macros = parse_tex_macros(value) for macro in macros: if 'args' in macro.keys(): # number of arguments > 1 @@ -204,60 +205,48 @@ def process_settings(pelicanobj): text_lines.append("{0}: '{1}'".format(macro['name'], macro['definition'])) mathjax_settings[key] = '{' + ", ".join(text_lines) + '}' - return mathjax_settings -def parse_tex_macro(*args): - """Returns a list of from input files. - - Each argument has to be an absolute path to a file. TeX macro definitions - are read and each one is translated to a dictionary containing the name - without backslash and the definition; if arguments are present, their - number is added too. +def _load_macro_definitions(*args): + """Returns list of lines from files, use absolute path. - If a macro is defined multiple times, a warning is printed to stdout. - The last definition is used. - - Backslashes in the definition are added in order to ensure the proper - form in the final html page. - - Example: - > [{'name': 'pd', - 'definition': '\\\\\\\\frac{\\\\\\\\partial #1}{\\\\\\\\partial #2}', - 'args': 2}, - {'name': 'R', 'definition': '\\\\\\\\mathbb{R}'}] - """ - temp_macros = [] + Example: [{'filename': '/home/user/defs.text', 'line_num': 1, + 'def': '\newcommand{\circ}{2 \pi R}'}]""" + output_lines = [] for arg in args: with open(arg, 'rt') as input_file: lines = input_file.read().splitlines() - for i, command in enumerate(lines): - splitted = command.split('{') - name_number = splitted[1].split('}') - name = name_number[0].strip('\\') - # for the definition, remove the last character from the last string which is } - # remember that strings are immutable objects in python - last_def_token = splitted[-1][:-1] - splitted_def = splitted[2:-1] + [last_def_token] - complete_def = '{'.join(splitted_def).replace('\\','\\\\\\\\') - final_command = {'line': i+1, 'file': arg, 'name': name, 'definition': complete_def} - if name_number[1]: - # the number of arguments is defined, therefore name_number[1] is not null string - args_number = name_number[1].lstrip('[').rstrip(']') - final_command['args'] = args_number - temp_macros.append(final_command) + for index, value in enumerate(lines): + line_num = index + 1 + line = {'filename': arg, 'line_num': line_num, 'def': value} + output_lines.append(line) + return output_lines + +def _filter_duplicates(*macros): + """Returns a modified copy of the input list of macros by keeping + only the last definition of each duplicate item. Also, if a macro is + defined multiple times, a warning is printed to stdout. + Unique items are left untouched. + """ names = [] - for macro in temp_macros: + for macro in macros: names.append(macro['name']) - import collections + seen = set() - duplicate_indices = [names.index(item) for item, count in collections.Counter(names).items() if count > 1] + # duplicate_indices contains the index of the first time an element appears + # more than once in names + duplicate_indices = [names.index(item) + for item, count in collections.Counter(names).items() + if count > 1] + unique_indices = [names.index(item) + for item, count in collections.Counter(names).items() + if count == 1] if len(duplicate_indices) > 0: - duplicates = [] + duplicates = [] for i in duplicate_indices: - name = temp_macros[i]['name'] + name = macros[i]['name'] duplicate = {'name': name, 'where':[]} - for j in temp_macros: + for j in macros: if j['name'] == name: duplicate['where'].append((j['line'], j['file'])) duplicates.append(duplicate) @@ -267,8 +256,66 @@ def parse_tex_macro(*args): for place in dup['where']: exception_text += "{}, line {}\n".format(place[1], place[0]) print(exception_text) - # remove line and file keys from temp_macros (added for debug in case of duplicates) - return [{k: v for k, v in elem.items() if k in ['name', 'definition', 'args']} for elem in temp_macros] + # I need the last definition for each duplicate definition + last_duplicated_indices = [] + for i, v in enumerate(duplicate_indices): + all_indices = [] + for j, name in enumerate(names): + if name == names[v]: + all_indices.append(j) + last = max(all_indices) + last_duplicated_indices.append(last) + + filtered = [elem for i, elem in enumerate(macros) + if i in unique_indices + last_duplicated_indices] + return filtered + +def parse_tex_macros(args): + # ogni arg รจ un file + macros = [] + for arg in args: + lines = _load_macro_definitions(arg) + for line in lines: + macros.append(_parse_macro(line)) + _filter_duplicates(*macros) + # remove line and file keys from temp_macros + # (added for debug in case of duplicates) + return [{k: v for k, v in elem.items() + if k in ['name', 'definition', 'args']} + for elem in macros] + +def _parse_macro(arg): + """Returns a macro from input raw text. + + The TeX macro definition is read and translated to a + dictionary containing the name without backslash and the definition; + if arguments are present, their number is added too. + + Backslashes in the definition are added in order to ensure the proper + form in the final html page. + + Example: + > {'name': 'pd', + 'definition': '\\\\\\\\frac{\\\\\\\\partial #1}{\\\\\\\\partial #2}', + 'args': 2, + 'file': '/home/user/commands.tex', + 'line': 1}""" + splitted = arg['def'].split('{') + name_number = splitted[1].split('}') + name = name_number[0].strip('\\') + # for the definition, remove the last character from the last string which is } + # remember that strings are immutable objects in python + last_def_token = splitted[-1][:-1] + splitted_def = splitted[2:-1] + [last_def_token] + complete_def = '{'.join(splitted_def).replace('\\','\\\\\\\\') + final_command = {'line': arg['line_num'], 'file': arg['filename'], + 'name': name, + 'definition': complete_def} + if name_number[1]: + # the number of arguments is defined, therefore name_number[1] is not null string + args_number = name_number[1].lstrip('[').rstrip(']') + final_command['args'] = args_number + return final_command def process_summary(article): """Ensures summaries are not cut off. Also inserts @@ -327,7 +374,6 @@ def process_mathjax_script(mathjax_settings): with open (os.path.dirname(os.path.realpath(__file__)) + '/mathjax_script_template', 'r') as mathjax_script_template: mathjax_template = mathjax_script_template.read() - return mathjax_template.format(**mathjax_settings) def mathjax_for_markdown(pelicanobj, mathjax_script, mathjax_settings): @@ -425,4 +471,5 @@ def process_rst_and_summaries(content_generators): def register(): """Plugin registration""" signals.initialized.connect(pelican_init) + # repeated signals.all_generators_finalized.connect(process_rst_and_summaries) diff --git a/test_math.py b/test_math.py new file mode 100644 index 0000000..7823ade --- /dev/null +++ b/test_math.py @@ -0,0 +1,84 @@ +import os +import unittest +from render_math import parse_tex_macros, _parse_macro, _filter_duplicates + +class TestParseMacros(unittest.TestCase): + def test_multiple_arguments(self): + """Parse a definition with multiple arguments""" + text = r'\newcommand{\pp}[2]{\frac{ #1}{ #2} \cdot 2}' + line = {'filename': '/home/user/example.tex', 'line_num': 1, 'def': + text} + parsed = _parse_macro(line) + expected = {'name':'pp', + 'definition': '\\\\\\\\frac{ #1}{ #2} \\\\\\\\cdot 2', + 'args': '2', + 'line': 1, + 'file': '/home/user/example.tex'} + self.assertEqual(parsed, expected) + + def test_no_arguments(self): + """Parse a definition without arguments""" + text = r'\newcommand{\circ}{2 \pi R}' + line = {'filename': '/home/user/example.tex', 'line_num': 1, 'def': + text} + parsed = _parse_macro(line) + expected = {'name':'circ', + 'definition': '2 \\\\\\\\pi R', + 'line': 1, + 'file': '/home/user/example.tex' + } + self.assertEqual(parsed, expected) + + def test_repeated_definitions_same_file(self): + """Last definition is used""" + text1 = r'2 \\\\\\\\pi R' + text2 = r'2 \\\\\\\\pi r' + common_file = '/home/user/example.tex' + def1 = {'name': 'circ', 'line': 1, 'definition': text1, + 'file': common_file} + def2 = {'name': 'circ', 'line': 2, 'definition': text2, + 'file': common_file} + expected = [{'name':'circ', + 'definition': r'2 \\\\\\\\pi r', + 'line': 2, + 'file': '/home/user/example.tex' + }] + parsed = _filter_duplicates(def1, def2) + self.assertEqual(parsed, expected) + + def test_repeated_definitions_different_files(self): + """Last definition is used""" + text1 = r'2 \\\\\\\\pi R' + text2 = r'2 \\\\\\\\pi r' + file1 = '/home/user/example1.tex' + file2 = '/home/user/example2.tex' + def1 = {'name': 'circ', 'line': 1, 'definition': text1, + 'file': file1} + def2 = {'name': 'circ', 'line': 1, 'definition': text2, + 'file': file2} + expected = [{'name':'circ', + 'definition': r'2 \\\\\\\\pi r', + 'line': 1, + 'file': '/home/user/example2.tex' + }] + parsed = _filter_duplicates(def1, def2) + self.assertEqual(parsed, expected) + + def test_load_file(self): + cur_dir = os.path.split(os.path.realpath(__file__))[0] + test_fname = os.path.join(cur_dir, "latex-commands-example.tex") + parsed = parse_tex_macros([test_fname]) + expected = [{'name': 'pp', + 'definition': '\\\\\\\\frac{\\\\\\\\partial #1}{' + '\\\\\\\\partial #2}', + 'args': '2'}, + {'name': 'bb', + 'definition': '\\\\\\\\pi R',}, + {'name': 'bc', + 'definition': '\\\\\\\\pi r', + }] + self.maxDiff = None + self.assertEqual(parsed, expected) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file