Format nested HTML lists without duplicating items (#1286)

src/python/txtai/pipeline/data/htmltomd.py

@@ -4,6 +4,8 @@ HTMLToMarkdown module
 
 import re
 
+from textwrap import indent
+
 # Conditional import
 try:
     from bs4 import BeautifulSoup, NavigableString
@@ -253,10 +255,19 @@ class HTMLToMarkdown(Pipeline):
             list as markdown
         """
 
-        elements = []
-        for x, element in enumerate(node.find_all("li")):
+        elements, count = [], 0
+        for element in node.find_all(("li", "ul", "ol"), recursive=False):
+            # Nested list that is a direct child of this list
+            if element.name != "li":
+                self.nested(element, article, elements)
+                continue
+
             # Unordered lists use dashes. Ordered lists use numbers.
-            prefix = "-" if node.name == "ul" else f"{x + 1}."
+            count += 1
+            prefix = "-" if node.name == "ul" else f"{count}."
+
+            # Detach nested lists, these are formatted separately
+            nested = [child.extract() for child in element.find_all(("ul", "ol"), recursive=False)]
 
             # List item text
             text = self.process(element, article)
@@ -265,9 +276,28 @@ class HTMLToMarkdown(Pipeline):
             if text:
                 elements.append(f"{prefix} {text}")
 
+            # Add nested lists
+            for child in nested:
+                self.nested(child, article, elements)
+
         # Join elements together as string
         return "\n".join(elements)
 
+    def nested(self, node, article, elements):
+        """
+        Nested list handler. Formats a nested list and adds it to elements indented with a tab,
+        which is preserved by text cleaning.
+
+        Args:
+            node: nested list node
+            article: True if the main section node is an article
+            elements: list of formatted list elements
+        """
+
+        text = self.items(node, article)
+        if text:
+            elements.append(indent(text, "\t"))
+
     def code(self, node):
         """
         Code block handler. This method transforms a HTML pre or code block into a Markdown formatted

test/python/testapi/testapipipeline.py

@@ -316,7 +316,7 @@ class TestPipeline(unittest.TestCase):
         text = self.client.get(f"textract?file={Utils.PATH}/article.pdf").json()
 
         # Check length of text is as expected
-        self.assertEqual(len(text), 2471)
+        self.assertEqual(len(text), 2472)
 
         # Check invalid URLs
         for url in ["http://192.168.1.1/path", "http://127.0.0.1/path", "http://invalid", "/etc/config", "/tmp/txtai-1/test"]:
@@ -331,7 +331,7 @@ class TestPipeline(unittest.TestCase):
         path = Utils.PATH + "/article.pdf"
 
         texts = self.client.post("batchtextract", json=[path, path]).json()
-        self.assertEqual((len(texts[0]), len(texts[1])), (2471, 2471))
+        self.assertEqual((len(texts[0]), len(texts[1])), (2472, 2472))
 
     def testTextToSpeech(self):
         """

test/python/testpipeline/testdata/testtextractor.py

@@ -113,6 +113,19 @@ class TestTextractor(unittest.TestCase):
         self.assertMarkdown("<ul><li>Test1</li><li>Test2</li></ul>", "- Test1\n- Test2")
         self.assertMarkdown("<ol><li>Test1</li><li>Test2</li></ol>", "1. Test1\n2. Test2")
 
+        # Nested lists
+        self.assertMarkdown("<ul><li>Test1<ul><li>Test2</li></ul></li><li>Test3</li></ul>", "- Test1\n\t- Test2\n- Test3")
+        self.assertMarkdown(
+            "<ol><li>Test1<ol><li>Test2</li><li>Test3</li></ol></li><li>Test4</li></ol>", "1. Test1\n\t1. Test2\n\t2. Test3\n2. Test4"
+        )
+        self.assertMarkdown(
+            "<ul><li>Test1<ol><li>Test2<ul><li>Test3</li><li>Test4</li></ul></li><li>Test5</li></ol></li><li>Test6</li></ul>",
+            "- Test1\n\t1. Test2\n\t\t- Test3\n\t\t- Test4\n\t2. Test5\n- Test6",
+        )
+
+        # Nested list that is a direct child of the parent list
+        self.assertMarkdown("<ol><li>Test1</li><ol><li>Test2</li></ol><li>Test3</li></ol>", "1. Test1\n\t1. Test2\n2. Test3")
+
         # Code
         self.assertMarkdown("<code>This is a test</code>", "```\nThis is a test\n```")
         self.assertMarkdown("<pre>This is a test</pre>", "```\nThis is a test\n```")
@@ -198,7 +211,7 @@ class TestTextractor(unittest.TestCase):
         text = textractor(Utils.PATH + "/article.pdf")
 
         # Check length of text is as expected
-        self.assertEqual(len(text), 2471)
+        self.assertEqual(len(text), 2472)
 
     def testTable(self):
         """
@@ -235,7 +248,7 @@ class TestTextractor(unittest.TestCase):
 
         path, text = textractor(Utils.PATH + "/article.pdf")
         self.assertEqual(path, Utils.PATH + "/article.pdf")
-        self.assertEqual(len(text), 2471)
+        self.assertEqual(len(text), 2472)
 
     def testURL(self):
         """