Adding DefinitionListBlock

mistletoe/base_renderer.py

@@ -68,6 +68,9 @@ def __init__(self, *extras, **kwargs):
             'ThematicBreak':  self.render_thematic_break,
             'LineBreak':      self.render_line_break,
             'Document':       self.render_document,
+            'DefinitionList': self.render_definition_list,
+            'DefinitionTerm': self.render_paragraph,
+            'DefinitionDesc': self.render_paragraph,
         }
         self._extras = extras
 
@@ -189,6 +192,9 @@ def render_list(self, token: block_token.List) -> str:
     def render_list_item(self, token: block_token.ListItem) -> str:
         return self.render_inner(token)
 
+    def render_definition_list(self, token: block_token.DefinitionList) -> str:
+        return self.render_inner(token)
+
     def render_table(self, token: block_token.Table) -> str:
         return self.render_inner(token)
 

mistletoe/block_token.py

@@ -18,8 +18,9 @@
 """
 Tokens to be included in the parsing process, in the order specified.
 """
-__all__ = ['BlockCode', 'Heading', 'Quote', 'CodeFence', 'ThematicBreak',
-           'List', 'Table', 'Footnote', 'Paragraph']
+__all__ = ['BlockCode', 'DefinitionList', 'DefinitionTerm', 'DefinitionDesc',
+           'Heading', 'Quote', 'CodeFence', 'ThematicBreak', 'List',
+           'Table', 'Footnote', 'Paragraph']
 
 
 def tokenize(lines):
@@ -313,6 +314,7 @@ def __new__(cls, lines):
     def __init__(self, lines):
         content = ''.join([line.lstrip() for line in lines]).strip()
         super().__init__(content, span_token.tokenize_inner)
+        self.line_number = 1
 
     @staticmethod
     def start(line):
@@ -1099,6 +1101,51 @@ def read(cls, lines):
         return line_buffer
 
 
+class DefinitionList(BlockToken):
+    """
+    Attributes:
+        defs (list): containing
+        * first a DefinitionTerm token
+        * then one or several DefinitionDesc tokens
+    """
+    START_MARKER = ": "
+
+    def __init__(self, defs):
+        self.defs = defs
+        self.children = [token for def_group in self.defs for token in def_group]
+
+    @classmethod
+    def start(cls, line, lines: 'tokenizer.FileReader'):
+        next_line = lines.peek(2) or ""
+        return next_line.startswith(cls.START_MARKER)
+
+    @classmethod
+    def read(cls, lines):
+        defs = [[]]
+        def_index = 0
+        for line in lines:
+            if line.startswith(cls.START_MARKER):
+                line = line[len(cls.START_MARKER):]
+                defs[def_index].append(DefinitionDesc([line]))
+            elif line == '\n':
+                if cls.start(line, lines):
+                    defs.append([])
+                    def_index += 1
+                else:
+                    break
+            else:
+                defs[def_index].append(DefinitionTerm([line]))
+        return defs
+
+
+class DefinitionTerm(Paragraph):
+    pass
+
+
+class DefinitionDesc(Paragraph):
+    pass
+
+
 HTMLBlock = HtmlBlock
 """
 Deprecated name of the `HtmlBlock` class.

mistletoe/block_tokenizer.py

@@ -1,14 +1,14 @@
 """
 Block-level tokenizer for mistletoe.
 """
+from inspect import signature
 
 
-class FileWrapper:
-    def __init__(self, lines, start_line=1):
+class FileReader:
+    def __init__(self, lines, start_line=1, index=-1):
         self.lines = lines if isinstance(lines, list) else list(lines)
         self.start_line = start_line
-        self._index = -1
-        self._anchor = 0
+        self._index = index
 
     def __next__(self):
         if self._index + 1 < len(self.lines):
@@ -27,6 +27,20 @@ def get_pos(self):
         The result is an opaque value which can be passed to `set_pos`."""
         return self._index
 
+    def peek(self, n=1):
+        if self._index + n < len(self.lines):
+            return self.lines[self._index + n]
+        return None
+
+    def line_number(self):
+        return self.start_line + self._index
+
+
+class FileWrapper(FileReader):
+    def __init__(self, lines, start_line=1):
+        super().__init__(lines, start_line)
+        self._anchor = 0
+
     def set_pos(self, pos):
         """Sets the current reading position."""
         self._index = pos
@@ -39,17 +53,16 @@ def reset(self):
         """@deprecated use `set_pos` instead"""
         self.set_pos(self._anchor)
 
-    def peek(self):
-        if self._index + 1 < len(self.lines):
-            return self.lines[self._index + 1]
-        return None
-
     def backstep(self):
         if self._index != -1:
             self._index -= 1
 
-    def line_number(self):
-        return self.start_line + self._index
+    def reader(self):
+        """
+        Return a FileReader with read-only access to the same file,
+        positioned at the index.
+        """
+        return FileReader(self.lines, self.start_line, self._index)
 
 
 def tokenize(iterable, token_types):
@@ -78,7 +91,12 @@ def tokenize_block(iterable, token_types, start_line=1):
     line = lines.peek()
     while line is not None:
         for token_type in token_types:
-            if token_type.start(line):
+            bound_func = token_type.start
+            # We need to support two cases:
+            # * a start() method accepting only a single `line` parameter
+            # * a start() method accepting both a `line` and a `lines` parameter, e.g. DefinitionList
+            is_start = bound_func(line) if len(signature(bound_func).parameters) == 1 else bound_func(line, lines.reader())
+            if is_start:
                 line_number = lines.line_number() + 1
                 result = token_type.read(lines)
                 if result is not None:

mistletoe/html_renderer.py

@@ -160,6 +160,18 @@ def render_list_item(self, token: block_token.ListItem) -> str:
                 inner_template = inner_template[:-1]
         return '<li>{}</li>'.format(inner_template.format(inner))
 
+    def render_definition_list(self, token: block_token.DefinitionList) -> str:
+        html = '<dl>\n'
+        for def_group in token.defs:
+            for j, token in enumerate(def_group):
+                inner = '\n'.join([self.render(child) for child in token.children])
+                if j == 0:  # => term/name
+                    html += '  <dt>' + inner + '</dt>\n'
+                else:  # => description/value, starting with 2nd token
+                    html += '  <dd>' + inner + '</dd>\n'
+        html += '</dl>'
+        return html
+
     def render_table(self, token: block_token.Table) -> str:
         # This is actually gross and I wonder if there's a better way to do it.
         #

mistletoe/markdown_renderer.py

@@ -332,6 +332,21 @@ def render_list_item(
             " " * prepend,
         )
 
+    def render_definition_list(
+        self, token: block_token.DefinitionList, max_line_length: int
+    ) -> Iterable[str]:
+        lines = []
+        last_group_index = len(token.defs) - 1
+        for i, def_group in enumerate(token.defs):
+            for j, token in enumerate(def_group):
+                token_lines = list(self.blocks_to_lines([token], max_line_length=max_line_length))
+                if j:  # => description/value, starting with 2nd token
+                    token_lines[0] = ": " + token_lines[0]
+                lines.extend(token_lines)
+            if i != last_group_index:
+                lines.append("")
+        return lines
+
     def render_table(
         self, token: block_token.Table, max_line_length: int
     ) -> Iterable[str]:

mistletoe/token.py

@@ -72,6 +72,14 @@ def __repr__(self):
         output += " at {:#x}>".format(id(self))
         return output
 
+    def __eq__(self, other):
+        if self.__class__ != other.__class__:
+            return False
+        for attrname in self.repr_attributes:
+            if getattr(self, attrname) != getattr(other, attrname):
+                return False
+        return True
+
     @property
     def parent(self) -> Optional['Token']:
         """Returns the parent token, if there is any."""

test/test_block_token.py

@@ -724,3 +724,22 @@ def test_anchor_reset(self):
         assert next(wrapper) == "somewhat interesting\n"
         wrapper.reset()
         assert next(wrapper) == "somewhat interesting\n"
+
+
+class TestDefinitionList(unittest.TestCase):
+    def test_match(self):
+        token = next(iter(block_token.tokenize([
+            "Term1\n",
+            ": definition1\n",
+            "\n",
+            "Term2\n",
+            ": definition2-A\n",
+            ": definition2-B\n"
+        ])))
+        self.assertIsInstance(token, block_token.DefinitionList)
+        self.assertEqual(len(token.defs), 2)
+        self.assertEqual(token.defs[0][0], block_token.DefinitionTerm(["Term1\n"]))
+        self.assertEqual(token.defs[0][1], block_token.DefinitionDesc(["definition1\n"]))
+        self.assertEqual(token.defs[1][0], block_token.DefinitionTerm(["Term2\n"]))
+        self.assertEqual(token.defs[1][1], block_token.DefinitionDesc(["definition2-A\n"]))
+        self.assertEqual(token.defs[1][2], block_token.DefinitionDesc(["definition2-B\n"]))

test/test_html_renderer.py

@@ -1,3 +1,4 @@
+from textwrap import dedent
 from unittest import TestCase, mock
 from mistletoe import Document
 from mistletoe.html_renderer import HtmlRenderer
@@ -174,3 +175,30 @@ def test_footnote_link(self):
         token = Document(['[name][foo]\n', '\n', '[foo]: target\n'])
         expected = '<p><a href="target">name</a></p>\n'
         self.assertEqual(self.renderer.render(token), expected)
+
+
+class TestHtmlDefinitionList(TestCase):
+    def setUp(self):
+        self.renderer = HtmlRenderer()
+        self.renderer.__enter__()
+        self.addCleanup(self.renderer.__exit__, None, None, None)
+
+    def test_definition_list(self):
+        token = Document([
+            'Term1\n',
+            ': definition1\n',
+            '\n',
+            'Term2\n',
+            ': definition2-A\n',
+            ': definition2-B\n',
+        ])
+        expected = dedent('''
+        <dl>
+          <dt>Term1</dt>
+          <dd>definition1</dd>
+          <dt>Term2</dt>
+          <dd>definition2-A</dd>
+          <dd>definition2-B</dd>
+        </dl>
+        ''').lstrip()
+        self.assertEqual(self.renderer.render(token), expected)

test/test_markdown_renderer.py

@@ -370,6 +370,19 @@ def test_table_with_narrow_column(self):
         ]
         self.assertEqual(output, "".join(expected))
 
+    def test_definition_list(self):
+        input = [
+            "Term1\n",
+            ": definition1\n",
+            "\n",
+            "Term2\n",
+            ": definition2-A\n",
+            ": definition2-B\n",
+        ]
+        output = self.roundtrip(input)
+        print("output:\n", output)
+        self.assertEqual(output, "".join(input))
+
     def test_direct_rendering_of_block_token(self):
         input = [
             "Line 1\n",
@@ -652,4 +665,4 @@ def test_wordwrap_tables(self):
             lines = renderer.render(document)
 
             # then the table is rendered without any word wrapping
-            assert lines == "".join(input)
+            assert lines == "".join(input)