update mistune to more recent version (#619)

* gitignore ./mypy_cache, .coverage, coverage.xml,pydocstyle_report.txt
* fix mypy-stubs for mistune
* use mypy.ini to share enable_recursive_aliases between mypy and mypyc calls
* split pyproject.toml build-system requirements multi-line for better readability
* add mistune pre-hook to handle problematic markdown lists (relates to lepture/mistune#296)
* add tests for expected schema-salad-doc output
* add make utilities to compute new hash and check html diff
* fix missing newline in code sample & update metaschema-pre.yml with extra newline
* align mismatchign mypy-requirements.txt / pyproject.toml mypy version
* normalize use of get_data() in tests

Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com>

.dockerignore

@@ -1,5 +1,6 @@
 .coverage
 coverage.xml
+pydocstyle_report.txt
 .tox/
 .eggs/
 .vscode/

.gitignore

@@ -2,14 +2,19 @@
 .eggs
 *.egg-info/
 *pyc
+*.so
 
 build/
 dist/
 mypy-stubs/ruamel/yaml
-# virtualenv 
+# virtualenv
 venv/
 .cache/
+.mypy_cache/
 .pytest_cache/
+.coverage
+coverage.xml
+pydocstyle_report.txt
 
 # PyCharm
 .idea/

Makefile

@@ -185,6 +185,23 @@ mypy_3.6: $(filter-out setup.py,$(PYSOURCES))
 mypyc: $(PYSOURCES)
 	MYPYPATH=mypy-stubs SCHEMA_SALAD_USE_MYPYC=1 python setup.py test
 
+mypyi:
+	MYPYPATH=mypy-stubs SCHEMA_SALAD_USE_MYPYC=1 python setup.py install
+
+check-metaschema-diff:
+	docker run \
+		-v "$(realpath ${MODULE}/metaschema/):/tmp/:ro" \
+		"quay.io/commonwl/cwltool_module:latest" \
+		schema-salad-doc /tmp/metaschema.yml \
+		> /tmp/metaschema.orig.html
+	schema-salad-doc \
+		"$(realpath ${MODULE}/metaschema/metaschema.yml)" \
+		> /tmp/metaschema.new.html
+	diff -a --color /tmp/metaschema.orig.html /tmp/metaschema.new.html || true
+
+compute-metaschema-hash:
+	@python -c 'import hashlib; from schema_salad.tests.test_makedoc import generate_doc; hasher = hashlib.sha256(); hasher.update(generate_doc().encode("utf-8")); print(hasher.hexdigest());'
+
 shellcheck: FORCE
 	shellcheck build-schema_salad-docker.sh release-test.sh
 

mypy-stubs/mistune/__init__.pyi

@@ -0,0 +1,28 @@
+from typing import Iterable, Optional, Union
+
+from mistune._types import *
+from mistune.inline_parser import RendererT
+from mistune.markdown import Markdown, ParseHook, RenderHook
+from mistune.plugins import Plugin, PluginName
+from mistune.renderers import BaseRenderer, DataT, HTMLRenderer, HTMLType
+from typing_extensions import Literal
+
+html: Markdown[HTMLType, HTMLRenderer]
+
+RendererRef = Union[Literal["html", "ast"], BaseRenderer[DataT]]
+PluginRef = Union[PluginName, Plugin]  # reference to register a plugin
+
+def create_markdown(
+    escape: bool = False,
+    hard_wrap: bool = False,
+    renderer: Optional[RendererRef[DataT]] = None,
+    plugins: Optional[Iterable[PluginRef]] = None,
+) -> Markdown[DataT, RendererT]: ...
+def markdown(
+    text: str,
+    escape: bool = True,
+    renderer: Optional[BaseRenderer[DataT]] = None,
+    plugins: Optional[Iterable[PluginRef]] = None,
+) -> str: ...
+
+__version__: str

mypy-stubs/mistune/_types.pyi

@@ -0,0 +1,5 @@
+from typing import Any, Dict, List
+
+# type aliases shared across modules
+State = Dict[str, Any]  # extra options that work with a given 'ParsedType'
+Tokens = List[str]