chore(tauri-search): started setting up for AST parse of rust

packages/docs/package.json

@@ -25,7 +25,6 @@
     "floating-vue": "^2.0.0-beta.6",
     "inferred-types": "^0.18.4",
     "markdown-it-expandable": "^1.0.2",
-    "meili-searchbar": "^2.1.0",
     "nprogress": "^0.2.0",
     "pinia": "^2.0.11",
     "prism-theme-vars": "^0.2.2",
@@ -37,7 +36,7 @@
   },
   "devDependencies": {
     "@antfu/eslint-config": "^0.16.1",
-    "@iconify/json": "^2.0.35",
+    "@iconify/json": "^2.0.36",
     "@intlify/vite-plugin-vue-i18n": "^3.2.2",
     "@types/markdown-it-link-attributes": "^3.0.1",
     "@types/nprogress": "^0.2.0",
@@ -48,26 +47,27 @@
     "critters": "^0.0.16",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.0",
-    "eslint": "^8.8.0",
+    "eslint": "^8.9.0",
     "eslint-plugin-cypress": "^2.12.1",
     "https-localhost": "^4.7.0",
     "jsdom": "^19.0.0",
     "markdown-it": "^12.3.2",
     "markdown-it-link-attributes": "^4.0.0",
     "markdown-it-prism": "^2.2.2",
+    "meili-searchbar": "^2.2.1",
     "typescript": "^4.5.5",
     "unplugin-auto-import": "^0.5.11",
     "unplugin-icons": "^0.13.1",
     "unplugin-vue-components": "^0.17.18",
-    "vite": "^2.8.0",
-    "vite-plugin-inspect": "^0.3.13",
+    "vite": "^2.8.1",
+    "vite-plugin-inspect": "^0.3.14",
     "vite-plugin-md": "^0.11.8",
     "vite-plugin-pages": "^0.20.2",
     "vite-plugin-pwa": "^0.11.13",
     "vite-plugin-vue-layouts": "^0.6.0",
     "vite-plugin-windicss": "^1.7.0",
     "vite-ssg": "^0.17.10",
-    "vitest": "^0.3.0",
+    "vitest": "^0.3.2",
     "vue-tsc": "^0.31.2"
   }
 }

packages/docs/pnpm-global/5/pnpm-lock.yaml

@@ -1,4 +0,0 @@
-lockfileVersion: 5.3
-
-specifiers:
-  meili-searchbar: ^2.1.0

packages/docs/src/App.vue

@@ -3,10 +3,8 @@
 // you can use this to manipulate the document head in any components,
 // they will be rendered correctly in the html results with vite-ssg
 useHead({
-  title: "Vitesse",
-  meta: [
-    { name: "description", content: "Opinionated Vite Starter Template" },
-  ],
+  title: "Tauri Search",
+  meta: [{ name: "description", content: "documentation for Tauri's search" }],
 });
 </script>
 

packages/docs/src/components/Footer.vue

@@ -1,8 +1,17 @@
 <script setup lang="ts">
 import { isDark, toggleDark } from "~/composables";
+import { SearchBar } from "meili-searchbar";
 
 const { t, availableLocales, locale } = useI18n();
 
+const config = {
+    hostUrl: "http://localhost:7700",
+    apiKey: undefined,
+    index: "consolidated",
+    primaryKey: "objectID",
+    debug: true,
+  };
+
 const toggleLocales = () => {
   // change to some real logic
   const locales = availableLocales;
@@ -123,6 +132,12 @@ const toggleLocales = () => {
         Switch Language for Heading Content
       </template>
     </v-tooltip>
+
+    <div class="px-4 text-gray-200/50 dark:text-gray-300/50 text-light ">
+      |
+    </div>
+
+    <search-bar :config="config" class="w-64 max-w-64" />
   </nav>
 </template>
 

packages/docs/src/components/SearchHit.vue

@@ -5,7 +5,9 @@ import { useSearch } from "~/modules/search";
 
 const props = defineProps({
   document: {
-    type: Object as PropType<GenericDoc>, required: true}
+    type: Object as PropType<GenericDoc>,
+    required: true,
+  },
 });
 
 const s = useSearch();
@@ -14,10 +16,10 @@ const doc = computed(() => props.document);
 const showDetails = ref(false);
 
 const apiKind = computed(() => {
-  if(doc.value._idx !== "api") {
+  if (doc.value._idx !== "api") {
     return "";
   }
-  switch(doc.value.kind) {
+  switch (doc.value.kind) {
     case "Function":
       return "fn";
     case "Namespace":
@@ -33,84 +35,130 @@ const details = () => {
   showDetails.value = !showDetails.value;
 };
 
-const searchable =s.indexSettings[doc.value._idx as any].searchableAttributes as string[];
-
+const searchable = s.indexSettings[doc.value._idx as any]
+  .searchableAttributes as string[];
 </script>
 
 <template>
-<div class="flex flex-col border-1 rounded px-2 py-1 border-gray-500  ">
-  <div class="flex flex-row space-x-1 -mx-2 -my-1 px-2 py-1 items-center place-content-center cursor-pointer hover:bg-gray-100/50 dark:hover:bg-gray-700/50" @click="details">
-    <div v-if="doc._idx === 'repo'" class="flex flex-row flex-grow space-x-1 items-center place-items-center">
-      <mdi:github  class="flex flex-shrink-0" />
-      <div class="name font-semibold flex-shrink-0 pr-2">{{doc.name}}</div>
-      <div class="description flex flex-grow font-light text-sm truncate text-gray-500">{{doc.description}}</div>
-      <link-validation :url="(doc.url as string)" />
-    </div>
-
-    <div v-if="doc._idx === 'api'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
-      <vscode-icons:file-type-typescript-official v-if="doc.language === 'typescript'" class="flex  flex-shrink-0" />
-      <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
-      <span class="flex flex-grow">{{ doc.name }}</span>
-      <div v-if="doc._idx === 'api'" class="flex text-xs font-medium px-1 py-0.5 bg-blue-500 dark:bg-blue-600 text-gray-50 rounded">
-        {{apiKind}}
-      </div>
-      <link-validation :url="(doc.url as string)" />
-    </div>
-
-    <div v-if="doc._idx === 'prose'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
-      <ant-design:file-markdown-outlined  class="flex" />
-      <div class="name flex flex-grow">{{doc.title}}</div>
-      <link-validation :url="(doc.url as string)" />
-    </div>
-
-    <div v-if="doc._idx === 'consolidated'" class="w-full">
-      <div v-if="doc.from === 'api'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
-        <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
-        <vscode-icons:file-type-typescript-official v-else class="flex" />
-
-        <div class="symbolName font-semibold">
-          <span class="font-light">{{doc.hierarchy_lvl1 && doc.hierarchy_lvl1 !== doc.hierarchy_lvl0  ? `${doc.hierarchy_lvl1}::` : ""}}</span>
-          {{doc.hierarchy_lvl0}}
+  <div class="flex flex-col border-1 rounded px-2 py-1 border-gray-500">
+    <div
+      class="flex flex-row space-x-1 -mx-2 -my-1 px-2 py-1 items-center place-content-center cursor-pointer hover:bg-gray-100/50 dark:hover:bg-gray-700/50"
+      @click="details"
+    >
+      <div
+        v-if="doc._idx === 'repo'"
+        class="flex flex-row flex-grow space-x-1 items-center place-items-center"
+      >
+        <mdi:github class="flex flex-shrink-0" />
+        <div class="name font-semibold flex-shrink-0 pr-2">{{ doc.name }}</div>
+        <div class="description flex flex-grow font-light text-sm truncate text-gray-500">
+          {{ doc.description }}
         </div>
-        <div class="text-sm font-light flex flex-grow">{{doc.symbol}}</div>
-        
         <link-validation :url="(doc.url as string)" />
       </div>
 
-      <!-- PROSE -->
-      <div v-if="doc.from === 'prose'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
-        <teenyicons:text-document-solid  class="flex flex-shrink-0" />
-        <div class="title font-semibold flex-shrink-0">{{doc.hierarchy_lvl0}}</div>
-        <div class="title font-light truncate text-gray-500 flex flex-grow">{{doc.hierarchy_lvl1 || doc.content}}</div>
+      <div
+        v-if="doc._idx === 'api'"
+        class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+      >
+        <vscode-icons:file-type-typescript-official
+          v-if="doc.language === 'typescript'"
+          class="flex flex-shrink-0"
+        />
+        <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
+        <span class="flex flex-grow">{{ doc.name }}</span>
+        <div
+          v-if="doc._idx === 'api'"
+          class="flex text-xs font-medium px-1 py-0.5 bg-blue-500 dark:bg-blue-600 text-gray-50 rounded"
+        >
+          {{ apiKind }}
+        </div>
         <link-validation :url="(doc.url as string)" />
       </div>
 
-      <!-- REPOs -->
-      <div v-if="doc.from === 'repo'" class="flex flex-row space-x-2 place-items-start items-start">
-        <mdi:github  class="flex flex-shrink-0 flex-grow-0" />
-        <div class="name font-semibold flex-shrink-0 flex-grow-0">{{doc.hierarchy_lvl0}}</div>
-        <div class="description truncate flex-shrink text-gray-500 font-light flex-grow">{{doc.hierarchy_lvl1}}</div>
+      <div
+        v-if="doc._idx === 'prose'"
+        class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+      >
+        <ant-design:file-markdown-outlined class="flex" />
+        <div class="name flex flex-grow">{{ doc.title }}</div>
         <link-validation :url="(doc.url as string)" />
       </div>
-      
+
+      <div v-if="doc._idx === 'consolidated'" class="w-full">
+        <div
+          v-if="doc.from === 'api'"
+          class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+        >
+          <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
+          <vscode-icons:file-type-typescript-official v-else class="flex" />
+
+          <div class="symbolName font-semibold">
+            <span class="font-light">
+              {{ doc.area !== doc.symbol ? `${doc.area}::` : "" }}
+            </span>
+            <span>{{ doc.symbol }}</span>
+          </div>
+          <div class="text-sm font-light flex flex-grow">
+            {{ doc.kind === "Namespace" ? "module" : doc.kind }}
+          </div>
+
+          <link-validation :url="(doc.url as string)" />
+        </div>
+
+        <!-- PROSE -->
+        <div
+          v-if="doc.from === 'prose'"
+          class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+        >
+          <teenyicons:text-document-solid class="flex flex-shrink-0" />
+          <div class="title font-semibold flex-shrink-0">{{ doc.hierarchy_lvl0 }}</div>
+          <div class="title font-light truncate text-gray-500 flex flex-grow">
+            {{ doc.hierarchy_lvl1 || doc.content }}
+          </div>
+          <link-validation :url="(doc.url as string)" />
+        </div>
+
+        <!-- REPOs -->
+        <div
+          v-if="doc.from === 'repo'"
+          class="flex flex-row space-x-2 place-items-start items-start"
+        >
+          <mdi:github class="flex flex-shrink-0 flex-grow-0" />
+          <div class="name font-semibold flex-shrink-0 flex-grow-0">
+            {{ doc.hierarchy_lvl0 }}
+          </div>
+          <div
+            class="description truncate flex-shrink text-gray-500 font-light flex-grow"
+          >
+            {{ doc.hierarchy_lvl1 }}
+          </div>
+          <link-validation :url="(doc.url as string)" />
+        </div>
+      </div>
     </div>
- 
-  
-  </div>
     <div v-if="showDetails" class="items-start my-2">
       <div
-v-for="key in Object.keys(doc)" :key="key" class="flex items-start justify-start" :class="[
-        searchable?.includes(key) || searchable[0] === '*' || false ? 'text-gray-800 dark:text-gray-200' : 'text-gray-500',
-        String(doc[key]).length > 70 ? 'flex-col space-y-1' : 'flex-row space-x-2'
-      ].join(' ')">
-        <div class="flex font-medium">{{key}}:</div>
+        v-for="key in Object.keys(doc)"
+        :key="key"
+        class="flex items-start justify-start"
+        :class="
+          [
+            searchable?.includes(key) || searchable[0] === '*' || false
+              ? 'text-gray-800 dark:text-gray-200'
+              : 'text-gray-500',
+            String(doc[key]).length > 70 ? 'flex-col space-y-1' : 'flex-row space-x-2',
+          ].join(' ')
+        "
+      >
+        <div class="flex font-medium">{{ key }}:</div>
         <div class="flex font-light place-self-start">
-          {{doc[key]}}
+          {{ doc[key] }}
         </div>
       </div>
       <div class="text-sm italic mt-2">
         key/values in grey are not "searchable" values and do not contribute to ranking.
       </div>
     </div>
-</div>
-</template>
+  </div>
+</template>

packages/docs/vite.config.ts

@@ -127,8 +127,8 @@ export default defineConfig({
       registerType: "autoUpdate",
       includeAssets: ["favicon.svg", "robots.txt", "safari-pinned-tab.svg"],
       manifest: {
-        name: "Vitesse",
-        short_name: "Vitesse",
+        name: "Tauri Search",
+        short_name: "Search",
         theme_color: "#ffffff",
         icons: [
           {

packages/scraper/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "scraper"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

packages/scraper/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

packages/tauri-search/package.json

@@ -37,7 +37,8 @@
     "restart": "docker compose restart",
     "sitemap": "node bin/sitemap.cjs",
     "test": "vitest run",
-    "test:watch": "vitest watch --ui",
+    "test:watch": "vitest",
+    "test:ui": "vitest watch --ui --silent",
     "ts-ast": "node ./bin/ts-ast.cjs",
     "ts-ast-overview": "node ./bin/ts-ast-overview.cjs",
     "watch": "run-p watch:*",
@@ -52,19 +53,19 @@
     "gray-matter": "^4.0.3",
     "html-to-text": "^8.1.0",
     "inferred-types": "^0.18.4",
-    "native-dash": "^1.21.5",
+    "native-dash": "^1.22.0",
     "simple-markdown-2": "^0.7.5"
   },
   "devDependencies": {
     "@octokit/types": "^6.34.0",
     "@type-challenges/utils": "^0.1.1",
     "@types/markdown-it": "^12.2.3",
-    "@types/node": "^14.18.10",
-    "@typescript-eslint/eslint-plugin": "^5.11.0",
-    "@typescript-eslint/parser": "^5.11.0",
-    "@vitest/ui": "^0.3.0",
+    "@types/node": "^14.18.12",
+    "@typescript-eslint/eslint-plugin": "^5.12.0",
+    "@typescript-eslint/parser": "^5.12.0",
+    "@vitest/ui": "^0.3.5",
     "changeset": "^0.2.6",
-    "eslint": "^8.8.0",
+    "eslint": "^8.9.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-cypress": "^2.12.1",
     "eslint-plugin-import": "^2.25.4",
@@ -78,10 +79,10 @@
     "ts-node": "^10.5.0",
     "tsup": "^5.11.13",
     "typescript": "^4.5.5",
-    "vite": "^2.8.0",
+    "vite": "^2.8.2",
     "vite-plugin-dts": "^0.9.9",
-    "vite-plugin-inspect": "^0.3.13",
-    "vitest": "^0.3.0"
+    "vite-plugin-inspect": "^0.3.14",
+    "vitest": "^0.3.5"
   },
   "engines": {
     "node": ">=14",

packages/tauri-search/src/constants.ts

@@ -2,13 +2,12 @@ import { Stage } from "~/types";
 
 export const TAURI_BASE_URL = `https://tauri.studio`;
 export const TAURI_JS_DOCS_URL = `${TAURI_BASE_URL}/docs/api/js`;
-
 export const GITHUB_API_BASE = `https://api.github.com`;
-
 export const REPO_DOCS_CACHE = `src/generated/ast/repo/documents.json`;
 
 export const TS_DOCS_CACHE = `src/generated/ast/api/ts-documents.json`;
 export const TS_AST_CACHE = `src/generated/ast/api/ts-ast.json`;
+export const RUST_AST_FIXTURE = `test/fixtures/rust-ast.json`;
 export const RS_DOCS_CACHE = `src/generated/ast/api/rs-documents.json`;
 
 export const SERVERS: Record<Stage, { url: string; search_key: string }> = {

packages/tauri-search/src/mappers/ConsolidatedMapper.ts

@@ -1,4 +1,5 @@
-import { ModelMapper, isRepoDocument, isApiDocument, isProseDocument } from "~/types";
+import { ModelMapper, isApiDocument, isProseDocument } from "~/types";
+import {ApiToConsolidated, ProseToConsolidated, RepoToConsolidated} from "./consolidated";
 import { IApiModel, IConsolidatedModel, IProseModel, IRepoModel } from "~/models";
 
 export enum IndexRank {
@@ -7,82 +8,12 @@ export enum IndexRank {
   api = 5,
 }
 
+
 export const ConsolidatedMapper: ModelMapper<
   IApiModel | IRepoModel | IProseModel,
   IConsolidatedModel
-> = (i): IConsolidatedModel => ({
-  objectID: i.id,
-  anchor_l2: isProseDocument(i) ? i.headings || null : null,
-  anchor_l3: isProseDocument(i) ? i.subHeadings || null : null,
-  area: isProseDocument(i)
-    ? i.area
-    : isRepoDocument(i)
-    ? i.kind
-    : isApiDocument(i)
-    ? i.module
-    : "",
-  hierarchy_lvl0: isRepoDocument(i)
-    ? i.name
-    : isApiDocument(i)
-    ? i.name || null
-    : i.title,
-  hierarchy_lvl1: isRepoDocument(i)
-    ? i.topics?.join(" ") || null
-    : isApiDocument(i)
-    ? i.module || null
-    : i.tags?.join(" ") || null,
-  hierarchy_lvl2: isRepoDocument(i)
-    ? i.kind === "unknown"
-      ? null
-      : i.kind || null
-    : isApiDocument(i)
-    ? i.language
-    : i.headings?.join(" ") || null,
-  hierarchy_lvl3: isRepoDocument(i)
-    ? i.description || null
-    : isApiDocument(i)
-    ? i.module
-    : i.tags?.join(" ") || null,
-  hierarchy_lvl4: isRepoDocument(i)
-    ? i.language || null
-    : isApiDocument(i)
-    ? null
-    : i.section || null,
-  hierarchy_lvl5: isRepoDocument(i)
-    ? i.license || null
-    : isApiDocument(i)
-    ? null
-    : i.area || null,
-  hierarchy_lvl6: isRepoDocument(i)
-    ? String(i.stars) || null
-    : isApiDocument(i)
-    ? null
-    : i.code?.join(" ") || null,
-  from: isRepoDocument(i) ? "repo" : isApiDocument(i) ? "api" : "prose",
-  symbol: isApiDocument(i) ? i.kind : null,
-  language: isApiDocument(i)
-    ? i.language
-    : isRepoDocument(i)
-    ? i.language
-    : i.code?.pop() || null,
-  tags: isRepoDocument(i) ? i.topics || null : isApiDocument(i) ? null : i.tags || null,
-  content: isRepoDocument(i)
-    ? i.topics?.join(" ") || null
-    : isApiDocument(i)
-    ? i.declaration || null
-    : i.subHeadings?.join(" ") || null,
-
-  text: isApiDocument(i) ? i.comment || null : i.text || null,
-  rank: isRepoDocument(i)
-    ? IndexRank.repo
-    : isApiDocument(i)
-    ? IndexRank.api
-    : IndexRank.prose,
-  // hierarchy_radio_lvl0: null,
-  // hierarchy_radio_lvl1: null,
-  // hierarchy_radio_lvl2: null,
-  // hierarchy_radio_lvl3: null,
-  // hierarchy_radio_lvl4: null,
-  // hierarchy_radio_lvl5: null,
-  url: i.url,
-});
+> = (i) => isApiDocument(i)
+  ? ApiToConsolidated(i)
+  : isProseDocument(i)
+    ? ProseToConsolidated(i)
+    : RepoToConsolidated(i);

packages/tauri-search/src/mappers/consolidated/ApiToConsolidated.ts

@@ -0,0 +1,32 @@
+import { IConsolidatedModel, IApiModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const ApiToConsolidated: ModelMapper<IApiModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "api",
+  rank: IndexRank.repo,
+
+  sections: null,
+  sub_sections: null,
+
+  symbol: i.name,
+  kind: i.kind,
+  area: i.module,
+  language: i.language,
+
+  content: null,
+  tags: i.tags || [],
+  text: i.comment || "",
+
+  // compatibility props
+  hierarchy_lvl0: i.name,
+  hierarchy_lvl1: i.comment || null,
+  hierarchy_lvl2: i.declaration || null,
+  hierarchy_lvl3: null,
+  hierarchy_lvl4: null,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/ProseToConsolidated.ts

@@ -0,0 +1,31 @@
+import { IConsolidatedModel, IProseModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const ProseToConsolidated: ModelMapper<IProseModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "prose",
+  rank: IndexRank.prose,
+
+  sections: i.headings || [],
+  sub_sections: i.subHeadings || [],
+
+  symbol: null,
+  area: i.area || i.section || null,
+  language: i.code?.join(" ") || null,
+
+  content: null,
+  tags: i.tags || [],
+  text: i.text || "",
+
+  // compatibility props
+  hierarchy_lvl0: i.title,
+  hierarchy_lvl1: i.area || null,
+  hierarchy_lvl2: i.parentSection || null,
+  hierarchy_lvl3: i.headings?.join(" ") || null,
+  hierarchy_lvl4: i.text,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/RepoToConsolidated.ts

@@ -0,0 +1,32 @@
+import { IConsolidatedModel, IRepoModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const RepoToConsolidated: ModelMapper<IRepoModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "repo",
+  rank: IndexRank.repo,
+
+  sections: null,
+  sub_sections: null,
+
+  symbol: null,
+  area: i.kind === "unknown" ? null : i.kind || null,
+  language: i.language,
+
+  content: null,
+  tags: i.topics || [],
+  text: i.text,
+
+
+  // compatibility props
+  hierarchy_lvl0: i.name,
+  hierarchy_lvl1: i.description,
+  hierarchy_lvl2: i.kind === "unknown" ? null : i.kind || null,
+  hierarchy_lvl3: null,
+  hierarchy_lvl4: null,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/index.ts

@@ -0,0 +1,3 @@
+export * from "./ApiToConsolidated";
+export * from "./ProseToConsolidated";
+export * from "./RepoToConsolidated";

packages/tauri-search/src/models/ConsolidatedModel.ts

@@ -11,14 +11,42 @@ export type IConsolidatedModel = Omit<
   | "hierarchy_radio_lvl4"
   | "hierarchy_radio_lvl5"
 > & {
-  anchor_l2: string[] | null;
-  anchor_l3: string[] | null;
-  area?: string;
+  /**
+   * top level sections in a document; in a Markdown doc this
+   * would typically be an H2 tag
+   */
+  sections: string[] | null;
+  /**
+   * a subsection of the document; in a Markdown doc this would
+   * typically be an H3 tag
+   */
+  sub_sections: string[] | null;
+
+  kind?: string;
+
+  area: string | null;
+  /**
+   * The source content/index which this document was derived from
+   */
   from: "prose" | "api" | "repo";
   rank: number;
+
+  /**
+   * most useful for API documents where it has the clearest mapping but 
+   * we can sometimes report on this in other docs too
+   */
   symbol: string | null;
+  /**
+   * A great mechanism for content authors to add words that should
+   * be associated with the document in markdown (as frontmatter). 
+   * In repo's we can also add topics of the repo into this property.
+   */
   tags: null | string[];
+  /**
+   * allows association of content to a particular programming language 
+   */
   language: string | null;
+  /** the main body of text  */
   text: string | null;
 };
 
@@ -31,20 +59,24 @@ export const ConsolidatedModel = createModel<IConsolidatedModel>("consolidated",
       js: ["typescript", "javascript", "js"],
       typescript: ["ts", "javascript", "js"],
       javascript: ["ts", "typescript", "js"],
+      
     })
     .filterable("from", "language", "symbol")
     .searchable(
       "area",
+      "symbol",
+      "tags",
       "hierarchy_lvl0",
       "hierarchy_lvl1",
-      "anchor_l2",
-      "symbol",
+      "sections",
       "rank",
-      "tags",
-      "content",
       "hierarchy_lvl2",
       "hierarchy_lvl3",
-      "anchor_l3"
+      "hierarchy_lvl4",
+      "hierarchy_lvl5",
+      "text",
+      "kind",
+      "sub_sections"
     )
     .rankingRules((r) =>
       r.words().typo().sort().attribute().proximity().ASC("rank").exactness()

packages/tauri-search/test/ast/parseRust.test.ts

@@ -1,5 +1,29 @@
+import { readFile } from "fs/promises";
+import { beforeAll } from "vitest";
 import { describe, expect, it } from "vitest";
+import { RUST_AST_FIXTURE } from "~/constants";
 
-describe("rustParser() - AST to List", () => {
-  it.todo("first test", () => {});
+export interface IRustAst {
+  root: string;
+  /** semver string */
+  crate_version: string;
+  includes_private: boolean;
+  index: any;
+  paths: any;
+  external_crates: any;
+  format_version: number;
+}
+
+describe("parseRustAst()", () => {
+  let ast: IRustAst;
+  beforeAll(async () => {
+    ast = JSON.parse(await readFile(RUST_AST_FIXTURE, "utf-8")) as IRustAst;
+  });
+  it("loaded fixture structure", () => {
+    expect(typeof ast.format_version).toEqual("number");
+
+    // crate_version is a semver string
+    expect(typeof ast.crate_version).toEqual("string");
+    expect(ast.crate_version.split(".")).toHaveLength(3);
+  });
 });

packages/tauri-search/test/ast/parseTypescript.test.ts

@@ -4,7 +4,7 @@ import { TypescriptKind } from "~/enums";
 import type { Expect, Equal } from "@type-challenges/utils";
 import { TsDocProject, TypescriptBlock, TypescriptSymbol } from "~/types";
 import { getRepoFile } from "~/utils/github/getRepoFile";
-import { getEnv } from "~/utils/getEnv";
+import { getEnv } from "~/utils/getEnv/esm/getEnv";
 
 let prj: TsDocProject;
 

tauri-search.code-workspace

@@ -11,6 +11,10 @@
     {
       "name": "📚 Docs",
       "path": "packages/docs"
+    },
+    {
+      "name": "🦀 scraper",
+      "path": "packages/scraper"
     }
   ],
   "settings": {