Replace HTML line breaks with newlines in HTMLToMarkdown (#1345)

src/python/txtai/pipeline/data/htmltomd.py

@@ -60,6 +60,14 @@ class HTMLToMarkdown(Pipeline):
         for script in soup.find_all(["script", "style"]):
             script.decompose()
 
+        # Replace line breaks with newlines. Table cells and headings must stay on a single line, use a space there.
+        for br in soup.find_all("br"):
+            parent = br.parent
+            br.replace_with(" " if br.find_parent(["th", "td", "h1", "h2", "h3", "h4", "h5", "h6"]) else "\n")
+
+            # Merge the line break with the text around it
+            parent.smooth()
+
         # Check for article sections
         article = next((x for x in ["article", "main"] if soup.find(x)), None)
 

test/python/testpipeline/testdata/testtextractor.py

@@ -151,6 +151,18 @@ class TestTextractor(unittest.TestCase):
         self.assertMarkdown("<p>This is a <strong><em>test</em></strong></p>", "This is a **test**")
         self.assertMarkdown("<p>This is a <em><strong>test</strong></em></p>", "This is a *test*")
 
+        # Line breaks
+        self.assertMarkdown("<p>Test1<br>Test2</p>", "Test1\nTest2")
+        self.assertMarkdown("<p>This is a <b>test<br/>case</b></p>", "This is a **test\ncase**")
+        self.assertMarkdown("<pre>Test1<br>Test2</pre>", "```\nTest1\nTest2\n```")
+
+        # Line breaks in headings and table cells are replaced with a space
+        self.assertMarkdown("<h1>This is<br>a test</h1>", "# This is a test")
+        self.assertMarkdown(
+            "<table><tr><th>Header1</th><th>Header2</th></tr><tr><td>Test1<br>Test2</td><td>Test3</td></tr></table>",
+            "|Header1|Header2|\n|---|---|\n|Test1 Test2|Test3|",
+        )
+
     def testHTMLMetadataNoContent(self):
         """
         Test HTML with a meta description tag that has no content attribute