From 738df97d5f8117fa3993ae03f03ce47b29f5c395 Mon Sep 17 00:00:00 2001 From: amengnew <82008855+amengnew@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:37:51 +0800 Subject: [PATCH] Fall back to row index when Tabular IDs are NaN (#1366) --- src/python/txtai/pipeline/data/tabular.py | 3 +- .../testpipeline/testdata/testtabular.py | 126 ++++++++++-------- 2 files changed, 75 insertions(+), 54 deletions(-) diff --git a/src/python/txtai/pipeline/data/tabular.py b/src/python/txtai/pipeline/data/tabular.py index ece0007a..58a75c7f 100644 --- a/src/python/txtai/pipeline/data/tabular.py +++ b/src/python/txtai/pipeline/data/tabular.py @@ -106,7 +106,8 @@ class Tabular(Pipeline): # Tuple iteration preserves column types instead of coercing each row to one dtype. for index, values in zip(df.index, df.itertuples(index=False, name=None)): row = dict(zip(df.columns, values)) - uid = row[self.idcolumn] if self.idcolumn is not None else index + # Coerce NaN ids to None so the row-index fallback applies. + uid = self.column(row[self.idcolumn]) if self.idcolumn is not None else index uid = uid if uid is not None else index text = self.concat(row, columns) diff --git a/test/python/testpipeline/testdata/testtabular.py b/test/python/testpipeline/testdata/testtabular.py index fc30568b..dcd9ac80 100644 --- a/test/python/testpipeline/testdata/testtabular.py +++ b/test/python/testpipeline/testdata/testtabular.py @@ -23,53 +23,6 @@ class TestTabular(unittest.TestCase): cls.tabular = Tabular("id", ["text"]) - def testFalsyValues(self): - """ - Test that zero and False are indexed as content - """ - - tabular = Tabular("id", ["quantity", "text"]) - - # A quantity of 0 is real data. Before this was fixed, the truthy check in concat - # dropped it and the row indexed as "Widget", losing the quantity. - rows = tabular([{"id": 1, "quantity": 0, "text": "Widget"}]) - self.assertEqual(rows[0][1], "0. Widget") - - rows = tabular([{"id": 2, "quantity": False, "text": "Widget"}]) - self.assertEqual(rows[0][1], "False. Widget") - - def testNullValues(self): - """ - Test that null and empty values are still skipped - """ - - tabular = Tabular("id", ["quantity", "text"]) - - # NaN is the null marker column() normalizes to None, and an empty string adds - # nothing but a separator. Both stay excluded. - rows = tabular([{"id": 1, "quantity": float("nan"), "text": "Widget"}]) - self.assertEqual(rows[0][1], "Widget") - - rows = tabular([{"id": 2, "quantity": "", "text": "Widget"}]) - self.assertEqual(rows[0][1], "Widget") - - def testNumericTypes(self): - """ - Mixed numeric rows preserve integer IDs and content without float rounding. - """ - - data = [{"id": 2**53 + 1, "count": 2, "value": 1.5}, {"id": 2**53, "count": 3, "value": 2.5}] - for content in (False, True, ["id"]): - with self.subTest(content=content): - rows = Tabular("id", ["count", "value"], content)(data) - step = 2 if content else 1 - self.assertTrue(all(isinstance(row[0], int) for row in rows)) - self.assertEqual([row[0] for row in rows[::step]], [row["id"] for row in data]) - self.assertEqual([row[1] for row in rows[::step]], ["2. 1.5", "3. 2.5"]) - if content: - expected = data if content is True else [{"id": row["id"]} for row in data] - self.assertEqual([row[1] for row in rows[1::2]], expected) - def testContent(self): """ Test parsing additional content @@ -120,6 +73,31 @@ class TestTabular(unittest.TestCase): self.assertEqual(uid, 0) self.assertEqual(text, "This is a test") + def testFalsyIDColumn(self): + """ + Explicit column labels are used for IDs and excluded from default text. + """ + + for column in ("", 0, "id"): + with self.subTest(column=column): + rows = Tabular(idcolumn=column)([{column: "doc-7", "text": "hello"}]) + self.assertEqual(rows, [("doc-7", "hello", None)]) + + def testFalsyValues(self): + """ + Test that zero and False are indexed as content + """ + + tabular = Tabular("id", ["quantity", "text"]) + + # A quantity of 0 is real data. Before this was fixed, the truthy check in concat + # dropped it and the row indexed as "Widget", losing the quantity. + rows = tabular([{"id": 1, "quantity": 0, "text": "Widget"}]) + self.assertEqual(rows[0][1], "0. Widget") + + rows = tabular([{"id": 2, "quantity": False, "text": "Widget"}]) + self.assertEqual(rows[0][1], "False. Widget") + def testInvalid(self): """ Test invalid file paths @@ -157,15 +135,25 @@ class TestTabular(unittest.TestCase): self.assertIsNone(data["metadata"]) - def testFalsyIDColumn(self): + def testMissingIDFallsBack(self): """ - Explicit column labels are used for IDs and excluded from default text. + Rows without an ID (missing key, None or NaN) fall back to the row index. + pandas coerces the missing value to NaN, which previously bypassed the + is-not-None fallback and indexed rows with a NaN uid. """ - for column in ("", 0, "id"): - with self.subTest(column=column): - rows = Tabular(idcolumn=column)([{column: "doc-7", "text": "hello"}]) - self.assertEqual(rows, [("doc-7", "hello", None)]) + tabular = Tabular("id", ["text"]) + + rows = tabular( + [ + {"id": "a", "text": "first"}, + {"text": "second"}, + {"id": None, "text": "third"}, + {"id": float("nan"), "text": "fourth"}, + {"id": "c", "text": "fifth"}, + ] + ) + self.assertEqual([row[0] for row in rows], ["a", 1, 2, 3, "c"]) def testNoColumns(self): """ @@ -178,3 +166,35 @@ class TestTabular(unittest.TestCase): self.assertEqual(uid, 0) self.assertEqual(text, "This is a test. Describes text in more detail") + + def testNullValues(self): + """ + Test that null and empty values are still skipped + """ + + tabular = Tabular("id", ["quantity", "text"]) + + # NaN is the null marker column() normalizes to None, and an empty string adds + # nothing but a separator. Both stay excluded. + rows = tabular([{"id": 1, "quantity": float("nan"), "text": "Widget"}]) + self.assertEqual(rows[0][1], "Widget") + + rows = tabular([{"id": 2, "quantity": "", "text": "Widget"}]) + self.assertEqual(rows[0][1], "Widget") + + def testNumericTypes(self): + """ + Mixed numeric rows preserve integer IDs and content without float rounding. + """ + + data = [{"id": 2**53 + 1, "count": 2, "value": 1.5}, {"id": 2**53, "count": 3, "value": 2.5}] + for content in (False, True, ["id"]): + with self.subTest(content=content): + rows = Tabular("id", ["count", "value"], content)(data) + step = 2 if content else 1 + self.assertTrue(all(isinstance(row[0], int) for row in rows)) + self.assertEqual([row[0] for row in rows[::step]], [row["id"] for row in data]) + self.assertEqual([row[1] for row in rows[::step]], ["2. 1.5", "3. 2.5"]) + if content: + expected = data if content is True else [{"id": row["id"]} for row in data] + self.assertEqual([row[1] for row in rows[1::2]], expected)