create llms.txt (#10097)

docs-website/docs/pipeline-components/classifiers.mdx

@@ -12,4 +12,4 @@ Use Classifiers to classify your documents by specific traits and update the met
 | Classifier                                                                           | Description                                           |
 | --- | --- |
 | [DocumentLanguageClassifier](classifiers/documentlanguageclassifier.mdx)                       | Classify documents by language.                       |
-| [TransformersZeroShotDocumentClassifier](classifiers/transformerszeroshotdocumentclassifier.mdx) | Classifiy the documents based on the provided labels. |
+| [TransformersZeroShotDocumentClassifier](classifiers/transformerszeroshotdocumentclassifier.mdx) | Classify the documents based on the provided labels. |

docs-website/docusaurus.config.js

@@ -96,6 +96,15 @@ const config = {
         lastVersion: '2.20',
       },
     ],
+    [
+      'docusaurus-plugin-generate-llms-txt',
+      {
+        // defaults to "llms.txt", but set explicitly for clarity
+        outputFile: 'llms.txt',
+      },
+    ],
+    // Local plugin to teach Webpack how to handle `.txt` files like `llms.txt`
+    require.resolve('./plugins/txtLoaderPlugin'),
   ],
 
   themeConfig:

docs-website/package.json

@@ -13,7 +13,8 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "update-next-version": "node scripts/update-next-version.js",
-    "create-version": "node scripts/create-new-version.js"
+    "create-version": "node scripts/create-new-version.js",
+    "generate-llms-txt": "docusaurus generate-llms-txt"
   },
   "dependencies": {
     "@docusaurus/core": "^3.8.1",
@@ -22,6 +23,7 @@
     "@docusaurus/preset-classic": "^3.8.1",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.0.0",
+    "docusaurus-plugin-generate-llms-txt": "^0.0.1",
     "prism-react-renderer": "^2.3.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",

docs-website/plugins/txtLoaderPlugin.js

@@ -0,0 +1,21 @@
+export default function txtLoaderPlugin() {
+  return {
+    name: 'txt-loader-plugin',
+    /**
+     * Ensure Webpack can handle plain `.txt` files like `llms.txt` without
+     * trying to parse them as JavaScript.
+     */
+    configureWebpack() {
+      return {
+        module: {
+          rules: [
+            {
+              test: /\.txt$/i,
+              type: 'asset/source',
+            },
+          ],
+        },
+      };
+    },
+  };
+}

docs-website/versioned_docs/version-2.20/pipeline-components/classifiers.mdx

@@ -12,4 +12,4 @@ Use Classifiers to classify your documents by specific traits and update the met
 | Classifier                                                                           | Description                                           |
 | --- | --- |
 | [DocumentLanguageClassifier](classifiers/documentlanguageclassifier.mdx)                       | Classify documents by language.                       |
-| [TransformersZeroShotDocumentClassifier](classifiers/transformerszeroshotdocumentclassifier.mdx) | Classifiy the documents based on the provided labels. |
+| [TransformersZeroShotDocumentClassifier](classifiers/transformerszeroshotdocumentclassifier.mdx) | Classify the documents based on the provided labels. |