Squashed commit of the following:

commit aa60ea14d3
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 14 12:52:27 2022 -0800

    chore(tauri-search): started setting up for AST parse of rust

commit 035aef7619
Author: Ken Snyder <ken@ken.net>
Date:   Fri Feb 11 22:06:39 2022 -0800

    chore: updated deps and caches

commit 39eb1c04f2
Author: Ken Snyder <ken@ken.net>
Date:   Wed Feb 9 17:16:20 2022 -0800

    chore: improvements to Prose and Consolidated model

commit cf85aece9a
Author: Ken Snyder <ken@ken.net>
Date:   Wed Feb 9 10:37:39 2022 -0800

    fix: moved no-new-privileges setting in docker-compose back to correct settign

commit 6fd1af7fc4
Author: Ken Snyder <ken@ken.net>
Date:   Wed Feb 9 09:57:30 2022 -0800

    chore: remove heirarchy_radio_lvl props to improve interaction with searchbar

    chore: updated deps

commit 266bc604ae
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 17:34:23 2022 -0800

    chore: removed refs to feature branch with TS AST

    chore: added `anchor` property to consolidated index for consistency sake

commit 556e0e29bd
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 14:31:09 2022 -0800

    chore: wire up all inputs to ENV variables

commit 866092e2e0
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 14:20:10 2022 -0800

    Feature/action jackson (#2)

    * chore: checking whether full relative path works

    * chore: add stage input to full-update

    * chore: moved prep action into actions folder

    * chore: checkout in action as well as workflow

    * chore: no name for checkout

    * fix: added checkout to test workflow

    * chore: trying actions/prep/action.yml path

    * chore: added namespace for setup-node

    * chore: added "shell" for CLI commands

    * chore: install tauri-repo deps as separate step

    * chore: use pnpm's action

    * chore: added pnpm version number

    * chore: disable cache for now

    * added some more inputs

    * chore: remove recursive flag in pnpm action

    * added ENV specific scripts for full update

    * chore: move MEILI keys to be stage specific

    * chore: add secrets to ENV in update

    * fix: add defensive code to ESM version of getEnv()

    * fix: more defence with getEnv()

    * chore: getEnv is defensive as hell :)

    * chore: another attempt at avoiding getEnv error

    * chore: picking up secrets in a differnt bucket

    * chore: full-update includes GH_ secrets

commit 6da3e9c4d6
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 11:25:11 2022 -0800

    chore: added full-update action

commit 7164376b38
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 11:18:23 2022 -0800

    chore: composite action's must be called action.yml

commit c781712d1b
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 09:32:54 2022 -0800

    chore: take 2 on test action

commit e841ba2b7f
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 09:12:34 2022 -0800

    chore: take 1 on a build action

commit 977434bd07
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 08:50:34 2022 -0800

    fix: fixed typo in docker-compose

    chore: starting github actions with build, lint, and test

commit 27aba5e5cf
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 8 07:05:38 2022 -0800

    chore: increased patterns to exclude ENV files

commit 4b40fc3a1f
Merge: b153e32 deb541f
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 19:53:51 2022 -0800

    Merge branch 'feature/models-return-function' into develop

commit deb541fafe
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 19:53:40 2022 -0800

    refactor: adjusted both CJS and ESM code to be able to interact better WRT to stage

commit 5aaaafe7ff
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 03:37:33 2022 -0800

    refactor: made createModel return a function that returns ISearchModel; allowing stage based offsets in Meili API

commit b153e3266d
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 01:19:27 2022 -0800

    chore: remove temp file vite.config.ts.js from repo

commit be8e580fe5
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 01:12:23 2022 -0800

    test: fixed tests to adjust to external changes in documentation as well as earlier caching refactor

commit c34a4c693f
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 00:42:46 2022 -0800

    fix: exclude JS API docs from prose docs

commit e3e371d3a8
Author: Ken Snyder <ken@ken.net>
Date:   Mon Feb 7 00:08:42 2022 -0800

    chore: created placeholder files for github actions

    refactor: refactored all CLI and pipeline code to be more consistent and use better abstraction

    feature: added the ability to configure ranking properties in a model

    refactor: reduced the number of cache files and removed those unnecessary for delta updates from git

commit faf048b2cf
Author: Ken Snyder <ken@ken.net>
Date:   Sat Feb 5 15:00:52 2022 -0800

    chore: remove unwanted cache files and make those which have merit binary files

commit b9c05ad560
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 21:14:55 2022 -0800

    chore: tuned properties of "consolidated" index to be more inline with current search

commit f40de2fd32
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 14:49:07 2022 -0800

    fix: fixed CLI to drop indexes (really this time)

commit f78c4070ed
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 14:47:05 2022 -0800

    fix: added appropriate await for dropping indexes

commit 2aaa6ddc69
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 14:45:47 2022 -0800

    chore: added CLI commands for creating and dropping indexes

commit 97384a6e9e
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 13:09:36 2022 -0800

    chore: fixed "into" docker scripts

commit 19fde622c3
Author: Ken Snyder <ken@ken.net>
Date:   Thu Feb 3 12:38:36 2022 -0800

    docs: small updates to help with ENV variables and docker compose

    chore: made sure that push-caches CLI creates the indexes before pushing docs (so that Model's config is used)

commit 6d67bac1b1
Author: Ken Snyder <ken@ken.net>
Date:   Wed Feb 2 21:32:02 2022 -0800

    chore: moved to forked version of simple-markdown which doesn't pollute namespace

    chore: forked markdown-it-collapsible and then upgraded it ... now released as markdown-it-expandable

commit 6e4aa1a342
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 13:26:08 2022 -0800

    chore(docs): styling adjustment to inline code blocks

commit 6377eb0a08
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 12:20:26 2022 -0800

    chore(docs): 2nd round of de-linting once React type bleed-in was removed

commit 1742280b3f
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 12:00:18 2022 -0800

    chore(docs): de-linted Vue files to follow linting rules

commit b17b699135
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 10:39:01 2022 -0800

    chore: updated docs links in README for docs package

commit c80d5aba41
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 01:58:23 2022 -0800

    feature: calling a model's createIndex also configures the index per the Model's specification

commit e4ce4f813f
Author: Ken Snyder <ken@ken.net>
Date:   Tue Feb 1 00:16:38 2022 -0800

    docs: cleaned up playground and mildly improved ux

commit b9c2bea63d
Author: Ken Snyder <ken@ken.net>
Date:   Mon Jan 31 22:37:42 2022 -0800

    chore: de-lint

commit 2b43976401
Merge: db33076 3ca01df
Author: Ken Snyder <ken@ken.net>
Date:   Mon Jan 31 22:11:30 2022 -0800

    Merge branch 'feature/mappers-integrated-with-model' into develop

commit 3ca01dfa6b
Author: Ken Snyder <ken@ken.net>
Date:   Mon Jan 31 22:10:54 2022 -0800

    chore: added link tests

commit 1efd543861
Author: Ken Snyder <ken@ken.net>
Date:   Mon Jan 31 20:11:21 2022 -0800

    chore: all pipelines -- outside of Rust API -- are in place and playground starting to be useful in analyizing index characteristics

commit 0c771ff555
Author: Ken Snyder <ken@ken.net>
Date:   Fri Jan 28 16:37:30 2022 -0800

    chore: finished refreshRepos() functionality

commit 60cd7912c0
Author: Ken Snyder <ken@ken.net>
Date:   Fri Jan 28 16:00:00 2022 -0800

    chore: refreshProse CLI and tests complete

    chore: refreshRepos started

commit 75d97ed4da
Author: Ken Snyder <ken@ken.net>
Date:   Thu Jan 27 23:02:53 2022 -0800

    chore: better CLI message for refresh-prose when no changes are detected

commit 1d50595692
Author: Ken Snyder <ken@ken.net>
Date:   Thu Jan 27 22:56:57 2022 -0800

    chore: added "refresh-prose" CLI command to build AST and ProseModel cache

commit 39cf3f84e7
Author: Ken Snyder <ken@ken.net>
Date:   Wed Jan 26 16:47:59 2022 -0800

    chore: reduced number of mappers needed and removed the createMapper abstraction

commit 8b753b4cc8
Author: Ken Snyder <ken@ken.net>
Date:   Wed Jan 26 10:40:42 2022 -0800

    refactor: moved files into better directory structure, simplified TS parse, added MD parse

commit d95db76538
Author: Ken Snyder <ken@ken.net>
Date:   Tue Jan 25 06:30:24 2022 -0800

    chore: model config API accepts mappers but API still needs finishing

packages/docs/package.json

@@ -25,7 +25,6 @@
     "floating-vue": "^2.0.0-beta.6",
     "inferred-types": "^0.18.4",
     "markdown-it-expandable": "^1.0.2",
-    "meili-searchbar": "^2.1.0",
     "nprogress": "^0.2.0",
     "pinia": "^2.0.11",
     "prism-theme-vars": "^0.2.2",
@@ -37,7 +36,7 @@
   },
   "devDependencies": {
     "@antfu/eslint-config": "^0.16.1",
-    "@iconify/json": "^2.0.35",
+    "@iconify/json": "^2.0.36",
     "@intlify/vite-plugin-vue-i18n": "^3.2.2",
     "@types/markdown-it-link-attributes": "^3.0.1",
     "@types/nprogress": "^0.2.0",
@@ -48,26 +47,27 @@
     "critters": "^0.0.16",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.0",
-    "eslint": "^8.8.0",
+    "eslint": "^8.9.0",
     "eslint-plugin-cypress": "^2.12.1",
     "https-localhost": "^4.7.0",
     "jsdom": "^19.0.0",
     "markdown-it": "^12.3.2",
     "markdown-it-link-attributes": "^4.0.0",
     "markdown-it-prism": "^2.2.2",
+    "meili-searchbar": "^2.2.1",
     "typescript": "^4.5.5",
     "unplugin-auto-import": "^0.5.11",
     "unplugin-icons": "^0.13.1",
     "unplugin-vue-components": "^0.17.18",
-    "vite": "^2.8.0",
-    "vite-plugin-inspect": "^0.3.13",
+    "vite": "^2.8.1",
+    "vite-plugin-inspect": "^0.3.14",
     "vite-plugin-md": "^0.11.8",
     "vite-plugin-pages": "^0.20.2",
     "vite-plugin-pwa": "^0.11.13",
     "vite-plugin-vue-layouts": "^0.6.0",
     "vite-plugin-windicss": "^1.7.0",
     "vite-ssg": "^0.17.10",
-    "vitest": "^0.3.0",
+    "vitest": "^0.3.2",
     "vue-tsc": "^0.31.2"
   }
 }

packages/docs/pnpm-global/5/pnpm-lock.yaml

@@ -1,4 +1,8 @@
 lockfileVersion: 5.3
 
 specifiers:
+<<<<<<< HEAD:packages/docs/pnpm-global/5/pnpm-lock.yaml
   meili-searchbar: ^2.1.0
+=======
+  markdown-it-collapsible: ^1.0.0
+>>>>>>> aa60ea14d30f0a7d8de9c1528a5b632c470628f9:pnpm-global/5/pnpm-lock.yaml

packages/docs/src/App.vue

@@ -3,10 +3,8 @@
 // you can use this to manipulate the document head in any components,
 // they will be rendered correctly in the html results with vite-ssg
 useHead({
-  title: "Vitesse",
-  meta: [
-    { name: "description", content: "Opinionated Vite Starter Template" },
-  ],
+  title: "Tauri Search",
+  meta: [{ name: "description", content: "documentation for Tauri's search" }],
 });
 </script>
 

packages/docs/src/components/Footer.vue

@@ -1,8 +1,17 @@
 <script setup lang="ts">
 import { isDark, toggleDark } from "~/composables";
+import { SearchBar } from "meili-searchbar";
 
 const { t, availableLocales, locale } = useI18n();
 
+const config = {
+    hostUrl: "http://localhost:7700",
+    apiKey: undefined,
+    index: "consolidated",
+    primaryKey: "objectID",
+    debug: true,
+  };
+
 const toggleLocales = () => {
   // change to some real logic
   const locales = availableLocales;
@@ -123,6 +132,12 @@ const toggleLocales = () => {
         Switch Language for Heading Content
       </template>
     </v-tooltip>
+
+    <div class="px-4 text-gray-200/50 dark:text-gray-300/50 text-light ">
+      |
+    </div>
+
+    <search-bar :config="config" class="w-64 max-w-64" />
   </nav>
 </template>
 

packages/docs/src/components/SearchHit.vue

@@ -5,7 +5,9 @@ import { useSearch } from "~/modules/search";
 
 const props = defineProps({
   document: {
-    type: Object as PropType<GenericDoc>, required: true}
+    type: Object as PropType<GenericDoc>,
+    required: true,
+  },
 });
 
 const s = useSearch();
@@ -33,76 +35,122 @@ const details = () => {
   showDetails.value = !showDetails.value;
 };
 
-const searchable =s.indexSettings[doc.value._idx as any].searchableAttributes as string[];
-
+const searchable = s.indexSettings[doc.value._idx as any]
+  .searchableAttributes as string[];
 </script>
 
 <template>
   <div class="flex flex-col border-1 rounded px-2 py-1 border-gray-500">
-  <div class="flex flex-row space-x-1 -mx-2 -my-1 px-2 py-1 items-center place-content-center cursor-pointer hover:bg-gray-100/50 dark:hover:bg-gray-700/50" @click="details">
-    <div v-if="doc._idx === 'repo'" class="flex flex-row flex-grow space-x-1 items-center place-items-center">
+    <div
+      class="flex flex-row space-x-1 -mx-2 -my-1 px-2 py-1 items-center place-content-center cursor-pointer hover:bg-gray-100/50 dark:hover:bg-gray-700/50"
+      @click="details"
+    >
+      <div
+        v-if="doc._idx === 'repo'"
+        class="flex flex-row flex-grow space-x-1 items-center place-items-center"
+      >
         <mdi:github class="flex flex-shrink-0" />
         <div class="name font-semibold flex-shrink-0 pr-2">{{ doc.name }}</div>
-      <div class="description flex flex-grow font-light text-sm truncate text-gray-500">{{doc.description}}</div>
+        <div class="description flex flex-grow font-light text-sm truncate text-gray-500">
+          {{ doc.description }}
+        </div>
         <link-validation :url="(doc.url as string)" />
       </div>
 
-    <div v-if="doc._idx === 'api'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
-      <vscode-icons:file-type-typescript-official v-if="doc.language === 'typescript'" class="flex  flex-shrink-0" />
+      <div
+        v-if="doc._idx === 'api'"
+        class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+      >
+        <vscode-icons:file-type-typescript-official
+          v-if="doc.language === 'typescript'"
+          class="flex flex-shrink-0"
+        />
         <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
         <span class="flex flex-grow">{{ doc.name }}</span>
-      <div v-if="doc._idx === 'api'" class="flex text-xs font-medium px-1 py-0.5 bg-blue-500 dark:bg-blue-600 text-gray-50 rounded">
+        <div
+          v-if="doc._idx === 'api'"
+          class="flex text-xs font-medium px-1 py-0.5 bg-blue-500 dark:bg-blue-600 text-gray-50 rounded"
+        >
           {{ apiKind }}
         </div>
         <link-validation :url="(doc.url as string)" />
       </div>
 
-    <div v-if="doc._idx === 'prose'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
+      <div
+        v-if="doc._idx === 'prose'"
+        class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+      >
         <ant-design:file-markdown-outlined class="flex" />
         <div class="name flex flex-grow">{{ doc.title }}</div>
         <link-validation :url="(doc.url as string)" />
       </div>
 
       <div v-if="doc._idx === 'consolidated'" class="w-full">
-      <div v-if="doc.from === 'api'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
+        <div
+          v-if="doc.from === 'api'"
+          class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+        >
           <vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
           <vscode-icons:file-type-typescript-official v-else class="flex" />
 
           <div class="symbolName font-semibold">
-          <span class="font-light">{{doc.hierarchy_lvl1 && doc.hierarchy_lvl1 !== doc.hierarchy_lvl0  ? `${doc.hierarchy_lvl1}::` : ""}}</span>
-          {{doc.hierarchy_lvl0}}
+            <span class="font-light">
+              {{ doc.area !== doc.symbol ? `${doc.area}::` : "" }}
+            </span>
+            <span>{{ doc.symbol }}</span>
+          </div>
+          <div class="text-sm font-light flex flex-grow">
+            {{ doc.kind === "Namespace" ? "module" : doc.kind }}
           </div>
-        <div class="text-sm font-light flex flex-grow">{{doc.symbol}}</div>
 
           <link-validation :url="(doc.url as string)" />
         </div>
 
         <!-- PROSE -->
-      <div v-if="doc.from === 'prose'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
+        <div
+          v-if="doc.from === 'prose'"
+          class="flex flex-row flex-grow space-x-2 place-items-center items-center"
+        >
           <teenyicons:text-document-solid class="flex flex-shrink-0" />
           <div class="title font-semibold flex-shrink-0">{{ doc.hierarchy_lvl0 }}</div>
-        <div class="title font-light truncate text-gray-500 flex flex-grow">{{doc.hierarchy_lvl1 || doc.content}}</div>
+          <div class="title font-light truncate text-gray-500 flex flex-grow">
+            {{ doc.hierarchy_lvl1 || doc.content }}
+          </div>
           <link-validation :url="(doc.url as string)" />
         </div>
 
         <!-- REPOs -->
-      <div v-if="doc.from === 'repo'" class="flex flex-row space-x-2 place-items-start items-start">
+        <div
+          v-if="doc.from === 'repo'"
+          class="flex flex-row space-x-2 place-items-start items-start"
+        >
           <mdi:github class="flex flex-shrink-0 flex-grow-0" />
-        <div class="name font-semibold flex-shrink-0 flex-grow-0">{{doc.hierarchy_lvl0}}</div>
-        <div class="description truncate flex-shrink text-gray-500 font-light flex-grow">{{doc.hierarchy_lvl1}}</div>
+          <div class="name font-semibold flex-shrink-0 flex-grow-0">
+            {{ doc.hierarchy_lvl0 }}
+          </div>
+          <div
+            class="description truncate flex-shrink text-gray-500 font-light flex-grow"
+          >
+            {{ doc.hierarchy_lvl1 }}
+          </div>
           <link-validation :url="(doc.url as string)" />
         </div>
-      
       </div>
- 
-  
     </div>
     <div v-if="showDetails" class="items-start my-2">
       <div
-v-for="key in Object.keys(doc)" :key="key" class="flex items-start justify-start" :class="[
-        searchable?.includes(key) || searchable[0] === '*' || false ? 'text-gray-800 dark:text-gray-200' : 'text-gray-500',
-        String(doc[key]).length > 70 ? 'flex-col space-y-1' : 'flex-row space-x-2'
-      ].join(' ')">
+        v-for="key in Object.keys(doc)"
+        :key="key"
+        class="flex items-start justify-start"
+        :class="
+          [
+            searchable?.includes(key) || searchable[0] === '*' || false
+              ? 'text-gray-800 dark:text-gray-200'
+              : 'text-gray-500',
+            String(doc[key]).length > 70 ? 'flex-col space-y-1' : 'flex-row space-x-2',
+          ].join(' ')
+        "
+      >
         <div class="flex font-medium">{{ key }}:</div>
         <div class="flex font-light place-self-start">
           {{ doc[key] }}

packages/docs/vite.config.ts

@@ -127,8 +127,8 @@ export default defineConfig({
       registerType: "autoUpdate",
       includeAssets: ["favicon.svg", "robots.txt", "safari-pinned-tab.svg"],
       manifest: {
-        name: "Vitesse",
-        short_name: "Vitesse",
+        name: "Tauri Search",
+        short_name: "Search",
         theme_color: "#ffffff",
         icons: [
           {

packages/scraper/Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "scraper"
+version = "0.1.0"

packages/scraper/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "scraper"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

packages/scraper/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

packages/scraper/target/.rustc_info.json

@@ -0,0 +1 @@
+{"rustc_fingerprint":4190679088944615026,"outputs":{"2797684049618456168":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n","stderr":""},"17598535894874457435":{"success":true,"status":"","code":0,"stdout":"rustc 1.58.1 (db9d1b20b 2022-01-20)\nbinary: rustc\ncommit-hash: db9d1b20bba1968c1ec1fc49616d4742c1725b4b\ncommit-date: 2022-01-20\nhost: x86_64-apple-darwin\nrelease: 1.58.1\nLLVM version: 13.0.0\n","stderr":""},"931469667778813386":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/kenny/.rustup/toolchains/stable-x86_64-apple-darwin\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_feature=\"ssse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store=\"128\"\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_thread_local\ntarget_vendor=\"apple\"\nunix\n","stderr":""}},"successes":{}}

packages/scraper/target/debug/.fingerprint/scraper-301168c9c20867c7/bin-scraper

@@ -0,0 +1 @@
+590ad50a918fc1b3

packages/scraper/target/debug/.fingerprint/scraper-301168c9c20867c7/bin-scraper.json

@@ -0,0 +1 @@
+{"rustc":5208427512371702208,"features":"[]","target":11069728573315614410,"profile":17483045194147818835,"path":1684066648322511884,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/scraper-301168c9c20867c7/dep-bin-scraper"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

packages/scraper/target/debug/.fingerprint/scraper-301168c9c20867c7/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

packages/scraper/target/debug/.fingerprint/scraper-f75d9212d6ce1a93/invoked.timestamp

@@ -0,0 +1 @@
+This file has an mtime of when this was started.

packages/scraper/target/debug/.fingerprint/scraper-f75d9212d6ce1a93/test-bin-scraper

@@ -0,0 +1 @@
+960aa5c9601f201d

packages/scraper/target/debug/.fingerprint/scraper-f75d9212d6ce1a93/test-bin-scraper.json

@@ -0,0 +1 @@
+{"rustc":5208427512371702208,"features":"[]","target":11069728573315614410,"profile":11506243869495082934,"path":1684066648322511884,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/scraper-f75d9212d6ce1a93/dep-test-bin-scraper"}}],"rustflags":[],"metadata":7797948686568424061,"config":2202906307356721367,"compile_kind":0}

packages/scraper/target/debug/deps/scraper-301168c9c20867c7.d

@@ -0,0 +1,5 @@
+/Volumes/Coding/playground/t/tauri-search/packages/scraper/target/debug/deps/scraper-301168c9c20867c7.rmeta: src/main.rs
+
+/Volumes/Coding/playground/t/tauri-search/packages/scraper/target/debug/deps/scraper-301168c9c20867c7.d: src/main.rs
+
+src/main.rs:

packages/scraper/target/debug/deps/scraper-f75d9212d6ce1a93.d

@@ -0,0 +1,5 @@
+/Volumes/Coding/playground/t/tauri-search/packages/scraper/target/debug/deps/scraper-f75d9212d6ce1a93.rmeta: src/main.rs
+
+/Volumes/Coding/playground/t/tauri-search/packages/scraper/target/debug/deps/scraper-f75d9212d6ce1a93.d: src/main.rs
+
+src/main.rs:

packages/tauri-search/package.json

@@ -37,7 +37,8 @@
     "restart": "docker compose restart",
     "sitemap": "node bin/sitemap.cjs",
     "test": "vitest run",
-    "test:watch": "vitest watch --ui",
+    "test:watch": "vitest",
+    "test:ui": "vitest watch --ui --silent",
     "ts-ast": "node ./bin/ts-ast.cjs",
     "ts-ast-overview": "node ./bin/ts-ast-overview.cjs",
     "watch": "run-p watch:*",
@@ -52,19 +53,19 @@
     "gray-matter": "^4.0.3",
     "html-to-text": "^8.1.0",
     "inferred-types": "^0.18.4",
-    "native-dash": "^1.21.5",
+    "native-dash": "^1.22.0",
     "simple-markdown-2": "^0.7.5"
   },
   "devDependencies": {
     "@octokit/types": "^6.34.0",
     "@type-challenges/utils": "^0.1.1",
     "@types/markdown-it": "^12.2.3",
-    "@types/node": "^14.18.10",
-    "@typescript-eslint/eslint-plugin": "^5.11.0",
-    "@typescript-eslint/parser": "^5.11.0",
-    "@vitest/ui": "^0.3.0",
+    "@types/node": "^14.18.12",
+    "@typescript-eslint/eslint-plugin": "^5.12.0",
+    "@typescript-eslint/parser": "^5.12.0",
+    "@vitest/ui": "^0.3.5",
     "changeset": "^0.2.6",
-    "eslint": "^8.8.0",
+    "eslint": "^8.9.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-cypress": "^2.12.1",
     "eslint-plugin-import": "^2.25.4",
@@ -78,10 +79,10 @@
     "ts-node": "^10.5.0",
     "tsup": "^5.11.13",
     "typescript": "^4.5.5",
-    "vite": "^2.8.0",
+    "vite": "^2.8.2",
     "vite-plugin-dts": "^0.9.9",
-    "vite-plugin-inspect": "^0.3.13",
-    "vitest": "^0.3.0"
+    "vite-plugin-inspect": "^0.3.14",
+    "vitest": "^0.3.5"
   },
   "engines": {
     "node": ">=14",

packages/tauri-search/src/constants.ts

@@ -2,13 +2,12 @@ import { Stage } from "~/types";
 
 export const TAURI_BASE_URL = `https://tauri.studio`;
 export const TAURI_JS_DOCS_URL = `${TAURI_BASE_URL}/docs/api/js`;
-
 export const GITHUB_API_BASE = `https://api.github.com`;
-
 export const REPO_DOCS_CACHE = `src/generated/ast/repo/documents.json`;
 
 export const TS_DOCS_CACHE = `src/generated/ast/api/ts-documents.json`;
 export const TS_AST_CACHE = `src/generated/ast/api/ts-ast.json`;
+export const RUST_AST_FIXTURE = `test/fixtures/rust-ast.json`;
 export const RS_DOCS_CACHE = `src/generated/ast/api/rs-documents.json`;
 
 export const SERVERS: Record<Stage, { url: string; search_key: string }> = {

packages/tauri-search/src/mappers/ConsolidatedMapper.ts

@@ -1,4 +1,9 @@
-import { ModelMapper, isRepoDocument, isApiDocument, isProseDocument } from "~/types";
+import { ModelMapper, isApiDocument, isProseDocument } from "~/types";
+import {
+  ApiToConsolidated,
+  ProseToConsolidated,
+  RepoToConsolidated,
+} from "./consolidated";
 import { IApiModel, IConsolidatedModel, IProseModel, IRepoModel } from "~/models";
 
 export enum IndexRank {
@@ -10,79 +15,9 @@ export enum IndexRank {
 export const ConsolidatedMapper: ModelMapper<
   IApiModel | IRepoModel | IProseModel,
   IConsolidatedModel
-> = (i): IConsolidatedModel => ({
-  objectID: i.id,
-  anchor_l2: isProseDocument(i) ? i.headings || null : null,
-  anchor_l3: isProseDocument(i) ? i.subHeadings || null : null,
-  area: isProseDocument(i)
-    ? i.area
-    : isRepoDocument(i)
-    ? i.kind
-    : isApiDocument(i)
-    ? i.module
-    : "",
-  hierarchy_lvl0: isRepoDocument(i)
-    ? i.name
-    : isApiDocument(i)
-    ? i.name || null
-    : i.title,
-  hierarchy_lvl1: isRepoDocument(i)
-    ? i.topics?.join(" ") || null
-    : isApiDocument(i)
-    ? i.module || null
-    : i.tags?.join(" ") || null,
-  hierarchy_lvl2: isRepoDocument(i)
-    ? i.kind === "unknown"
-      ? null
-      : i.kind || null
-    : isApiDocument(i)
-    ? i.language
-    : i.headings?.join(" ") || null,
-  hierarchy_lvl3: isRepoDocument(i)
-    ? i.description || null
-    : isApiDocument(i)
-    ? i.module
-    : i.tags?.join(" ") || null,
-  hierarchy_lvl4: isRepoDocument(i)
-    ? i.language || null
-    : isApiDocument(i)
-    ? null
-    : i.section || null,
-  hierarchy_lvl5: isRepoDocument(i)
-    ? i.license || null
-    : isApiDocument(i)
-    ? null
-    : i.area || null,
-  hierarchy_lvl6: isRepoDocument(i)
-    ? String(i.stars) || null
-    : isApiDocument(i)
-    ? null
-    : i.code?.join(" ") || null,
-  from: isRepoDocument(i) ? "repo" : isApiDocument(i) ? "api" : "prose",
-  symbol: isApiDocument(i) ? i.kind : null,
-  language: isApiDocument(i)
-    ? i.language
-    : isRepoDocument(i)
-    ? i.language
-    : i.code?.pop() || null,
-  tags: isRepoDocument(i) ? i.topics || null : isApiDocument(i) ? null : i.tags || null,
-  content: isRepoDocument(i)
-    ? i.topics?.join(" ") || null
-    : isApiDocument(i)
-    ? i.declaration || null
-    : i.subHeadings?.join(" ") || null,
-
-  text: isApiDocument(i) ? i.comment || null : i.text || null,
-  rank: isRepoDocument(i)
-    ? IndexRank.repo
-    : isApiDocument(i)
-    ? IndexRank.api
-    : IndexRank.prose,
-  // hierarchy_radio_lvl0: null,
-  // hierarchy_radio_lvl1: null,
-  // hierarchy_radio_lvl2: null,
-  // hierarchy_radio_lvl3: null,
-  // hierarchy_radio_lvl4: null,
-  // hierarchy_radio_lvl5: null,
-  url: i.url,
-});
+> = (i) =>
+  isApiDocument(i)
+    ? ApiToConsolidated(i)
+    : isProseDocument(i)
+    ? ProseToConsolidated(i)
+    : RepoToConsolidated(i);

packages/tauri-search/src/mappers/consolidated/ApiToConsolidated.ts

@@ -0,0 +1,32 @@
+import { IConsolidatedModel, IApiModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const ApiToConsolidated: ModelMapper<IApiModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "api",
+  rank: IndexRank.repo,
+
+  sections: null,
+  sub_sections: null,
+
+  symbol: i.name,
+  kind: i.kind,
+  area: i.module,
+  language: i.language,
+
+  content: null,
+  tags: i.tags || [],
+  text: i.comment || "",
+
+  // compatibility props
+  hierarchy_lvl0: i.name,
+  hierarchy_lvl1: i.comment || null,
+  hierarchy_lvl2: i.declaration || null,
+  hierarchy_lvl3: null,
+  hierarchy_lvl4: null,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/ProseToConsolidated.ts

@@ -0,0 +1,31 @@
+import { IConsolidatedModel, IProseModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const ProseToConsolidated: ModelMapper<IProseModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "prose",
+  rank: IndexRank.prose,
+
+  sections: i.headings || [],
+  sub_sections: i.subHeadings || [],
+
+  symbol: null,
+  area: i.area || i.section || null,
+  language: i.code?.join(" ") || null,
+
+  content: null,
+  tags: i.tags || [],
+  text: i.text || "",
+
+  // compatibility props
+  hierarchy_lvl0: i.title,
+  hierarchy_lvl1: i.area || null,
+  hierarchy_lvl2: i.parentSection || null,
+  hierarchy_lvl3: i.headings?.join(" ") || null,
+  hierarchy_lvl4: i.text,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/RepoToConsolidated.ts

@@ -0,0 +1,32 @@
+import { IConsolidatedModel, IRepoModel } from "~/models";
+import { ModelMapper } from "~/types";
+import { IndexRank } from "../ConsolidatedMapper";
+
+export const RepoToConsolidated: ModelMapper<IRepoModel, IConsolidatedModel> = i => ({
+  objectID: i.id,
+  from: "repo",
+  rank: IndexRank.repo,
+
+  sections: null,
+  sub_sections: null,
+
+  symbol: null,
+  area: i.kind === "unknown" ? null : i.kind || null,
+  language: i.language,
+
+  content: null,
+  tags: i.topics || [],
+  text: i.text,
+
+
+  // compatibility props
+  hierarchy_lvl0: i.name,
+  hierarchy_lvl1: i.description,
+  hierarchy_lvl2: i.kind === "unknown" ? null : i.kind || null,
+  hierarchy_lvl3: null,
+  hierarchy_lvl4: null,
+  hierarchy_lvl5: null,
+  hierarchy_lvl6: null,
+
+  url: i.url
+});

packages/tauri-search/src/mappers/consolidated/index.ts

@@ -0,0 +1,3 @@
+export * from "./ApiToConsolidated";
+export * from "./ProseToConsolidated";
+export * from "./RepoToConsolidated";

packages/tauri-search/src/models/ConsolidatedModel.ts

@@ -11,14 +11,42 @@ export type IConsolidatedModel = Omit<
   | "hierarchy_radio_lvl4"
   | "hierarchy_radio_lvl5"
 > & {
-  anchor_l2: string[] | null;
-  anchor_l3: string[] | null;
-  area?: string;
+  /**
+   * top level sections in a document; in a Markdown doc this
+   * would typically be an H2 tag
+   */
+  sections: string[] | null;
+  /**
+   * a subsection of the document; in a Markdown doc this would
+   * typically be an H3 tag
+   */
+  sub_sections: string[] | null;
+
+  kind?: string;
+
+  area: string | null;
+  /**
+   * The source content/index which this document was derived from
+   */
   from: "prose" | "api" | "repo";
   rank: number;
+
+  /**
+   * most useful for API documents where it has the clearest mapping but
+   * we can sometimes report on this in other docs too
+   */
   symbol: string | null;
+  /**
+   * A great mechanism for content authors to add words that should
+   * be associated with the document in markdown (as frontmatter).
+   * In repo's we can also add topics of the repo into this property.
+   */
   tags: null | string[];
+  /**
+   * allows association of content to a particular programming language
+   */
   language: string | null;
+  /** the main body of text  */
   text: string | null;
 };
 
@@ -35,16 +63,19 @@ export const ConsolidatedModel = createModel<IConsolidatedModel>("consolidated",
     .filterable("from", "language", "symbol")
     .searchable(
       "area",
+      "symbol",
+      "tags",
       "hierarchy_lvl0",
       "hierarchy_lvl1",
-      "anchor_l2",
-      "symbol",
+      "sections",
       "rank",
-      "tags",
-      "content",
       "hierarchy_lvl2",
       "hierarchy_lvl3",
-      "anchor_l3"
+      "hierarchy_lvl4",
+      "hierarchy_lvl5",
+      "text",
+      "kind",
+      "sub_sections"
     )
     .rankingRules((r) =>
       r.words().typo().sort().attribute().proximity().ASC("rank").exactness()

packages/tauri-search/test/ast/parseRust.test.ts

@@ -1,5 +1,29 @@
+import { readFile } from "fs/promises";
+import { beforeAll } from "vitest";
 import { describe, expect, it } from "vitest";
+import { RUST_AST_FIXTURE } from "~/constants";
 
-describe("rustParser() - AST to List", () => {
-  it.todo("first test", () => {});
+export interface IRustAst {
+  root: string;
+  /** semver string */
+  crate_version: string;
+  includes_private: boolean;
+  index: any;
+  paths: any;
+  external_crates: any;
+  format_version: number;
+}
+
+describe("parseRustAst()", () => {
+  let ast: IRustAst;
+  beforeAll(async () => {
+    ast = JSON.parse(await readFile(RUST_AST_FIXTURE, "utf-8")) as IRustAst;
+  });
+  it("loaded fixture structure", () => {
+    expect(typeof ast.format_version).toEqual("number");
+
+    // crate_version is a semver string
+    expect(typeof ast.crate_version).toEqual("string");
+    expect(ast.crate_version.split(".")).toHaveLength(3);
+  });
 });

packages/tauri-search/test/ast/parseTypescript.test.ts

@@ -4,7 +4,7 @@ import { TypescriptKind } from "~/enums";
 import type { Expect, Equal } from "@type-challenges/utils";
 import { TsDocProject, TypescriptBlock, TypescriptSymbol } from "~/types";
 import { getRepoFile } from "~/utils/github/getRepoFile";
-import { getEnv } from "~/utils/getEnv";
+import { getEnv } from "~/utils/getEnv/esm/getEnv";
 
 let prj: TsDocProject;
 

pnpm-global/5/pnpm-lock.yaml

@@ -1,4 +1,12 @@
 lockfileVersion: 5.3
 
 specifiers:
+<<<<<<< HEAD
   markdown-it-collapsible: ^1.0.0
+=======
+<<<<<<< HEAD:packages/docs/pnpm-global/5/pnpm-lock.yaml
+  meili-searchbar: ^2.1.0
+=======
+  markdown-it-collapsible: ^1.0.0
+>>>>>>> aa60ea14d30f0a7d8de9c1528a5b632c470628f9:pnpm-global/5/pnpm-lock.yaml
+>>>>>>> aa60ea14d30f0a7d8de9c1528a5b632c470628f9

tauri-search.code-workspace

@@ -11,6 +11,10 @@
     {
       "name": "📚 Docs",
       "path": "packages/docs"
+    },
+    {
+      "name": "🦀 scraper",
+      "path": "packages/scraper"
     }
   ],
   "settings": {