mirror of
https://github.com/tauri-apps/tauri-search.git
synced 2026-02-04 02:41:20 +01:00
chore: all pipelines -- outside of Rust API -- are in place and playground starting to be useful in analyizing index characteristics
This commit is contained in:
Ken Snyder
16
package.json
16
package.json
@@ -9,9 +9,18 @@
|
||||
"start:tauri-search": "pnpm -C ./packages/tauri-search run watch",
|
||||
"start:docs": "pnpm -C ./packages/docs run watch",
|
||||
"build:cli": "pnpm -C ./packages/tauri-search run build:cli",
|
||||
"watch": "pnpm run watch --filter ./packages/* ",
|
||||
"test": "pnpm run test --filter ./packages/*",
|
||||
"build:npm": "pnpm -C ./packages/tauri-search run build:npm",
|
||||
"watch": "run-s watch:prep watch:rest",
|
||||
"watch:prep": "pnpm -C ./packages/tauri-search run build:npm",
|
||||
"watch:rest": "run-p watch:tauri-search watch:docs",
|
||||
"watch:tauri-search": "pnpm -C ./packages/tauri-search run watch",
|
||||
"watch:docs": "pnpm -C ./packages/docs run watch",
|
||||
"test": "pnpm -C ./packages/docs run test && pnpm -C ./packages/tauri-search run test",
|
||||
"test:watch": "pnpm -C ./packages/tauri-search run test:watch",
|
||||
"current-indexes": "pnpm -C ./packages/tauri-search run current-indexes",
|
||||
"create-indexes": "pnpm -C ./packages/tauri-search run create-indexes",
|
||||
"refresh-prose": "pnpm -C ./packages/tauri-search run refresh-prose",
|
||||
"push-prose": "pnpm -C ./packages/tauri-search run push-prose",
|
||||
"up": "docker compose up -d",
|
||||
"down": "docker compose down",
|
||||
"into:scraper": "docker exec -it scraper bash",
|
||||
@@ -33,5 +42,8 @@
|
||||
"docs": "pnpm -C ./packages/docs run dev",
|
||||
"npm": "pnpm -C ./packages/tauri-search run watch:npm",
|
||||
"npm:test": "pnpm -C ./packages/tauri-search run test"
|
||||
},
|
||||
"devDependencies": {
|
||||
"npm-run-all": "^4.1.5"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,3 +16,8 @@ home:
|
||||
search: Suche
|
||||
desc: Spielplatz für die Tauri-Suche
|
||||
whats-your-name: Suchen Sie nach etwas
|
||||
ask-for-search: Suchen Sie nach etwas
|
||||
nav:
|
||||
indexing: Indizierung
|
||||
toggle_langs: Sprachen ändern
|
||||
toggle_dark: Dunkelmodus umschalten
|
||||
|
||||
@@ -15,3 +15,9 @@ not-found: No se ha encontrado
|
||||
home:
|
||||
search: Búsqueda
|
||||
whats-your-name: buscar algo
|
||||
ask-for-search: buscar algo
|
||||
desc: Documentación y Patio Interactivo
|
||||
nav:
|
||||
indexing: Indexación
|
||||
toggle_dark: Alternar modo oscuro
|
||||
toggle_langs: Cambiar idiomas
|
||||
|
||||
@@ -16,3 +16,8 @@ home:
|
||||
desc: Aire de jeux pour Tauri Search
|
||||
search: Chercher
|
||||
whats-your-name: Rechercher quelque chose
|
||||
ask-for-search: Rechercher quelque chose
|
||||
nav:
|
||||
indexing: Indexage
|
||||
toggle_dark: Basculer en mode sombre
|
||||
toggle_langs: Changer de langue
|
||||
|
||||
@@ -5,39 +5,44 @@
|
||||
"start": "pnpm run dev",
|
||||
"build": "vite-ssg build",
|
||||
"dev": "vite --port 3333 --open",
|
||||
"lint": "eslint \"**/*.{vue,ts,js}\"",
|
||||
"lint": "run-s lint:*",
|
||||
"lint:eslint": "eslint \"**/*.{vue,ts,js}\"",
|
||||
"lint:tsc": "vue-tsc --noEmit",
|
||||
"preview": "vite preview",
|
||||
"preview-https": "serve dist",
|
||||
"test": "vitest run",
|
||||
"test:e2e": "cypress open",
|
||||
"test:unit": "vitest",
|
||||
"test:dev": "vitest dev",
|
||||
"typecheck": "vue-tsc --noEmit",
|
||||
"test:watch": "vitest watch",
|
||||
"watch": "pnpm run dev"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vueuse/core": "^7.5.4",
|
||||
"@vueuse/core": "^7.5.5",
|
||||
"@vueuse/head": "^0.7.5",
|
||||
"date-fns": "^2.28.0",
|
||||
"floating-vue": "^2.0.0-beta.3",
|
||||
"nprogress": "^0.2.0",
|
||||
"pinia": "^2.0.9",
|
||||
"pinia": "^2.0.11",
|
||||
"prism-theme-vars": "^0.2.2",
|
||||
"tauri-search": "workspace:*",
|
||||
"ts-morph": "^13.0.3",
|
||||
"vue": "^3.2.28",
|
||||
"vue": "^3.2.29",
|
||||
"vue-demi": "^0.12.1",
|
||||
"vue-i18n": "^9.1.9",
|
||||
"vue-router": "^4.0.12"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^0.16.0",
|
||||
"@antfu/eslint-config": "^0.16.1",
|
||||
"@iconify-json/ant-design": "^1.0.2",
|
||||
"@iconify-json/bx": "^1.0.3",
|
||||
"@iconify-json/carbon": "^1.0.14",
|
||||
"@iconify-json/cib": "^1.0.1",
|
||||
"@iconify-json/fluent": "^1.0.15",
|
||||
"@iconify-json/fluent": "^1.0.16",
|
||||
"@iconify-json/ic": "^1.0.8",
|
||||
"@iconify-json/mdi": "^1.0.12",
|
||||
"@iconify-json/iconoir": "^1.0.5",
|
||||
"@iconify-json/mdi": "^1.0.15",
|
||||
"@iconify-json/octicon": "^1.0.9",
|
||||
"@iconify-json/ph": "^1.0.4",
|
||||
"@iconify-json/tabler": "^1.0.12",
|
||||
"@iconify-json/teenyicons": "^1.0.1",
|
||||
@@ -46,13 +51,12 @@
|
||||
"@types/markdown-it-link-attributes": "^3.0.1",
|
||||
"@types/nprogress": "^0.2.0",
|
||||
"@vitejs/plugin-vue": "^2.1.0",
|
||||
"@vue/compiler-sfc": "^3.2.28",
|
||||
"@vue/server-renderer": "^3.2.28",
|
||||
"@vue/compiler-sfc": "^3.2.29",
|
||||
"@vue/server-renderer": "^3.2.29",
|
||||
"@vue/test-utils": "^2.0.0-rc.18",
|
||||
"critters": "^0.0.16",
|
||||
"cross-env": "^7.0.3",
|
||||
"cypress": "^9.3.1",
|
||||
"eslint": "^8.7.0",
|
||||
"eslint": "^8.8.0",
|
||||
"eslint-plugin-cypress": "^2.12.1",
|
||||
"https-localhost": "^4.7.0",
|
||||
"jsdom": "^19.0.0",
|
||||
@@ -62,17 +66,17 @@
|
||||
"typescript": "^4.5.5",
|
||||
"unplugin-auto-import": "^0.5.11",
|
||||
"unplugin-icons": "^0.13.0",
|
||||
"unplugin-vue-components": "^0.17.14",
|
||||
"unplugin-vue-components": "^0.17.15",
|
||||
"vite": "^2.7.13",
|
||||
"vite-plugin-inspect": "^0.3.13",
|
||||
"vite-plugin-md": "^0.11.7",
|
||||
"vite-plugin-pages": "^0.20.0",
|
||||
"vite-plugin-pages": "^0.20.1",
|
||||
"vite-plugin-pwa": "^0.11.13",
|
||||
"vite-plugin-vue-layouts": "^0.5.0",
|
||||
"vite-plugin-windicss": "^1.6.3",
|
||||
"vite-ssg": "^0.17.6",
|
||||
"vite-ssg": "^0.17.9",
|
||||
"vitepress": "^0.21.6",
|
||||
"vitest": "^0.1.27",
|
||||
"vitest": "^0.2.5",
|
||||
"vue-tsc": "^0.31.1"
|
||||
}
|
||||
}
|
||||
|
||||
1
packages/docs/src/auto-imports.d.ts
vendored
1
packages/docs/src/auto-imports.d.ts
vendored
@@ -183,6 +183,7 @@ declare global {
|
||||
const useSpeechSynthesis: typeof import('@vueuse/core')['useSpeechSynthesis']
|
||||
const useStorage: typeof import('@vueuse/core')['useStorage']
|
||||
const useStorageAsync: typeof import('@vueuse/core')['useStorageAsync']
|
||||
const useStyleTag: typeof import('@vueuse/core')['useStyleTag']
|
||||
const useSwipe: typeof import('@vueuse/core')['useSwipe']
|
||||
const useTemplateRefsList: typeof import('@vueuse/core')['useTemplateRefsList']
|
||||
const useTextSelection: typeof import('@vueuse/core')['useTextSelection']
|
||||
|
||||
16
packages/docs/src/components.d.ts
vendored
16
packages/docs/src/components.d.ts
vendored
@@ -4,31 +4,39 @@
|
||||
|
||||
declare module 'vue' {
|
||||
export interface GlobalComponents {
|
||||
'AntDesign:apiTwotone': typeof import('~icons/ant-design/api-twotone')['default']
|
||||
'AntDesign:fileMarkdownOutlined': typeof import('~icons/ant-design/file-markdown-outlined')['default']
|
||||
'Bx:bxCheckbox': typeof import('~icons/bx/bx-checkbox')['default']
|
||||
'Bx:bxCheckboxChecked': typeof import('~icons/bx/bx-checkbox-checked')['default']
|
||||
'Bx:bxCheckboxMinus': typeof import('~icons/bx/bx-checkbox-minus')['default']
|
||||
'Bx:bxSearchAlt': typeof import('~icons/bx/bx-search-alt')['default']
|
||||
'Carbon:document': typeof import('~icons/carbon/document')['default']
|
||||
'Carbon:errorFilled': typeof import('~icons/carbon/error-filled')['default']
|
||||
'Carbon:listDropdown': typeof import('~icons/carbon/list-dropdown')['default']
|
||||
'Carbon:nextFilled': typeof import('~icons/carbon/next-filled')['default']
|
||||
CarbonLanguage: typeof import('~icons/carbon/language')['default']
|
||||
CarbonMoon: typeof import('~icons/carbon/moon')['default']
|
||||
CarbonSun: typeof import('~icons/carbon/sun')['default']
|
||||
Checkbox: typeof import('./components/Checkbox.vue')['default']
|
||||
Counter: typeof import('./components/Counter.vue')['default']
|
||||
CurrentIndex: typeof import('./components/CurrentIndex.vue')['default']
|
||||
'Fluent:databaseSearch24Regular': typeof import('~icons/fluent/database-search24-regular')['default']
|
||||
Footer: typeof import('./components/Footer.vue')['default']
|
||||
'Iconoir:download': typeof import('~icons/iconoir/download')['default']
|
||||
'Mdi:folderHome': typeof import('~icons/mdi/folder-home')['default']
|
||||
'Mdi:github': typeof import('~icons/mdi/github')['default']
|
||||
'Mdi:languageRust': typeof import('~icons/mdi/language-rust')['default']
|
||||
'Mdi:languageTypescript': typeof import('~icons/mdi/language-typescript')['default']
|
||||
MissingIndex: typeof import('./components/MissingIndex.vue')['default']
|
||||
'Octicon:trash16': typeof import('~icons/octicon/trash16')['default']
|
||||
'Ph:info': typeof import('~icons/ph/info')['default']
|
||||
'Ph:linkLight': typeof import('~icons/ph/link-light')['default']
|
||||
README: typeof import('./components/README.md')['default']
|
||||
SearchActions: typeof import('./components/SearchActions.vue')['default']
|
||||
SearchHit: typeof import('./components/SearchHit.vue')['default']
|
||||
SearchIndexes: typeof import('./components/SearchIndexes.vue')['default']
|
||||
SearchResults: typeof import('./components/SearchResults.vue')['default']
|
||||
SearchStats: typeof import('./components/SearchStats.vue')['default']
|
||||
SimpleCard: typeof import('./components/SimpleCard.vue')['default']
|
||||
'Tabler:databaseImport': typeof import('~icons/tabler/database-import')['default']
|
||||
'Teenyicons:dockerOutline': typeof import('~icons/teenyicons/docker-outline')['default']
|
||||
'Teenyicons:textDocumentSolid': typeof import('~icons/teenyicons/text-document-solid')['default']
|
||||
'VscodeIcons:fileTypeRust': typeof import('~icons/vscode-icons/file-type-rust')['default']
|
||||
'VscodeIcons:fileTypeTypescriptOfficial': typeof import('~icons/vscode-icons/file-type-typescript-official')['default']
|
||||
}
|
||||
|
||||
19
packages/docs/src/components/Checkbox.vue
Normal file
19
packages/docs/src/components/Checkbox.vue
Normal file
@@ -0,0 +1,19 @@
|
||||
<script setup lang="ts">import { PropType } from "vue";
|
||||
|
||||
const props = defineProps({
|
||||
value: {type: Boolean as PropType<boolean | undefined>, default: undefined, required: true}
|
||||
});
|
||||
|
||||
const toggle = () => {
|
||||
|
||||
};
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="checkbox text-xl mr-1">
|
||||
<bx:bx-checkbox-minus v-if="props.value === undefined" class="flex cursor-default text-gray-500/50" />
|
||||
<bx:bx-checkbox v-if="props.value=== false" class="flex cursor-pointer text-gray-500/75" @click="toggle" />
|
||||
<bx:bx-checkbox-checked v-if="props.value===true" class="flex cursor-pointer text-gray-800/95 dark:text-gray-100/90" @click="toggle" />
|
||||
|
||||
</div>
|
||||
</template>
|
||||
112
packages/docs/src/components/CurrentIndex.vue
Normal file
112
packages/docs/src/components/CurrentIndex.vue
Normal file
@@ -0,0 +1,112 @@
|
||||
<script setup lang="ts">
|
||||
/* eslint-disable no-console */
|
||||
import { useSearch } from "~/modules/search";
|
||||
import { format } from "date-fns";
|
||||
import { PropType } from "vue";
|
||||
import { MeiliSearchInterface } from "~/types/meilisearch";
|
||||
import { ApiModel } from "tauri-search";
|
||||
const s = useSearch();
|
||||
const props = defineProps({
|
||||
idx: {type: Object as PropType<MeiliSearchInterface>, required: true},
|
||||
unknown: {type: Boolean, default: false, required: false}
|
||||
});
|
||||
const indexName = computed(() => props.idx.name as "api" | "repo" | "prose");
|
||||
|
||||
const remove = async () => {
|
||||
// just using ApiModel at random, as we can state which index to remove
|
||||
await ApiModel.query.deleteIndex(indexName.value);
|
||||
};
|
||||
|
||||
const pushCache = async() => {
|
||||
// console.group(`downloading cached docs for "${indexName.value}"`);
|
||||
// const docs = await getCache(indexName.value);
|
||||
// console.info(`${docs.length} docs downloaded, now calling Meilisearch API to add each doc`);
|
||||
// const tasks = await pushDocs(docs);
|
||||
// console.info(`all documents have been sent to the MeiliSearch API where they will be processed in a task queu`);
|
||||
// console.log(`The tasks in queue are:`, tasks);
|
||||
// console.groupEnd();
|
||||
};
|
||||
|
||||
const usedInSearch = computed(() => s.indexIsUsed(indexName.value));
|
||||
|
||||
// async function pushCache() {
|
||||
// const docs = await
|
||||
// }
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div
|
||||
class="index flex flex-row w-full py-2 px-3 border-1 rounded border-gray-500 mt-4 cursor-default items-center justify-center"
|
||||
:class="props.unknown ? 'unknown' : ''"
|
||||
>
|
||||
<checkbox :value="usedInSearch" @click="s.toggleUseOfIndex(indexName)" />
|
||||
<span class="index-name flex flex-grow font-medium">{{indexName}}</span>
|
||||
<div class="flex flex-row items-center">
|
||||
<!-- Right Content -->
|
||||
<v-tooltip v-if="s.indexInfo(indexName)?.numberOfDocuments === 0 && !props.unknown" class="cursor-pointer text-sm flex flex-grow-0 mr-3 text-gray-500 hover:text-green-800 dark:hover:text-green-500">
|
||||
<iconoir:download class="" @click="pushCache" />
|
||||
<template #popper>Push documents in cache into<br/>local MeiliSearch server</template>
|
||||
</v-tooltip>
|
||||
<v-tooltip>
|
||||
<octicon:trash-16
|
||||
class="cursor-pointer text-sm flex flex-grow-0 mr-3 text-gray-500 hover:text-red-800 dark:hover:text-red-500"
|
||||
@click="remove"
|
||||
/>
|
||||
<template #popper>remove "{{indexName}}" index<span v-if="s.indexInfo(indexName)?.numberOfDocuments || 0 > 0">; including all documents.</span></template>
|
||||
</v-tooltip>
|
||||
<span class="index-count flex flex-grow-0 font-light items-baseline ">
|
||||
<span class="flex">{{s.indexInfo(indexName)?.numberOfDocuments}}</span>
|
||||
|
||||
<span class="flex italic font-light text-xs">docs</span>
|
||||
</span>
|
||||
<v-tooltip placement="right" >
|
||||
<ph:info
|
||||
class=" flex flex-grow-0 ml-2 dark:text-gray-500/75 hover:text-opacity-100 "
|
||||
/>
|
||||
<template #popper>
|
||||
<div class="w-56 text-sm">
|
||||
<div class="font-bold text-lg">
|
||||
{{indexName}}
|
||||
|
||||
<span class="font-light">index</span>
|
||||
</div>
|
||||
<table width="100%">
|
||||
<tr>
|
||||
<td class="font-normal">created:</td>
|
||||
<td class="font-light" align="right">{{format(new Date(props.idx.createdAt), "do MMM, yyyy")}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="font-normal">updated:</td>
|
||||
<td class="font-light justify-end" align="right" >{{format(new Date(props.idx.updatedAt), "do MMM, yyyy")}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>stop words</td>
|
||||
<td>[]</td>
|
||||
</tr>
|
||||
<tr v-if="s.indexInfo(indexName)?.numberOfDocuments || 0 > 0" class="text-xs">
|
||||
<td>Prop<br/>Distribution</td>
|
||||
<td>
|
||||
<table width="100%">
|
||||
<tr v-for="k in Object.keys(s.$state.stats?.indexes[indexName].fieldDistribution|| {}) " :key="k">
|
||||
<td>{{ k }}</td>
|
||||
<td>{{ s.$state.stats?.indexes[indexName].fieldDistribution[k]}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
</table>
|
||||
</div>
|
||||
</template>
|
||||
</v-tooltip>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</template>
|
||||
|
||||
<style lang="css" scoped>
|
||||
.unknown {
|
||||
@apply border-orange-500/50
|
||||
}
|
||||
</style>
|
||||
41
packages/docs/src/components/MissingIndex.vue
Normal file
41
packages/docs/src/components/MissingIndex.vue
Normal file
@@ -0,0 +1,41 @@
|
||||
<script setup lang="ts">
|
||||
import { ApiModel, ConsolidatedModel, ProseModel, RepoModel } from "tauri-search";
|
||||
import { PropType } from "vue";
|
||||
import { useSearch } from "~/modules/search";
|
||||
const s = useSearch();
|
||||
const props = defineProps({
|
||||
idx: {type: String as PropType<"api" | "prose" | "repo" | "consolidated">, required: true}
|
||||
});
|
||||
const models = {
|
||||
api: ApiModel,
|
||||
prose: ProseModel,
|
||||
repo: RepoModel,
|
||||
consolidated: ConsolidatedModel
|
||||
};
|
||||
|
||||
async function addIndex() {
|
||||
const model = models[props.idx];
|
||||
await model.query.createIndex();
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<v-tooltip class="w-full" placement="right" :delay="8000">
|
||||
<div
|
||||
class="index flex flex-row w-full py-2 px-3 border-1 rounded border-dashed border-gray-300 dark:border-gray-700 mt-4 cursor-default"
|
||||
>
|
||||
<span class="index-name flex flex-grow font-medium">{{idx}}</span>
|
||||
<span class="index-count flex flex-grow-0 font-light cursor-pointer items-baseline">
|
||||
<v-tooltip :delay="750" placement="right">
|
||||
<button class="text-xs italic px-2 py-1 bg-gray-300/75 hover:bg-green-300/75 dark:hover:bg-green-700/75 dark:bg-gray-700/75 rounded-md" @click="addIndex">add index</button>
|
||||
<template #popper>will create an empty index for "{{idx}}"</template>
|
||||
</v-tooltip>
|
||||
</span>
|
||||
|
||||
</div>
|
||||
<template #popper>
|
||||
The <span class="tag">{{idx}}</span> index is defined as a MODEL but doesn't yet exist in MeiliSearch as an index.
|
||||
</template>
|
||||
</v-tooltip>
|
||||
</template>
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useSearch } from "~/modules/search";
|
||||
const s = useSearch();
|
||||
const indexPossibilites = ["repo", "prose", "TS API", "RS API"];
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex flex-col rounded-md bg-gray-100/25 dark:bg-gray-900/25">
|
||||
<div class="flex flex-col space-y-2 rounded-t-md overflow-hidden">
|
||||
|
||||
<div class="font-bold text-lg px-4 py-3 bg-gray-900/10 dark:bg-gray-100/10">
|
||||
ACTIONS
|
||||
</div>
|
||||
|
||||
<div class="p-4 flex flex-col space-y-2">
|
||||
|
||||
<div v-for="idx in indexPossibilites" :key="idx" class=" rounded cursor-pointer p-1 flex">
|
||||
<div class="actions flex flex-row space-x-1 items-center text-gray-100 dark:text-gray-700 transition-colors duration-200">
|
||||
<carbon:next-filled class="flex hover:text-green-500" />
|
||||
<carbon:error-filled class="flex hover:text-orange-500" />
|
||||
</div>
|
||||
|
||||
<div class="flex flex-row ml-2">
|
||||
<span class="font-medium">{{idx}} </span>
|
||||
<span class="font-light">pipeline</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style lang="css" scoped>
|
||||
.btn {
|
||||
@apply text-xs ;
|
||||
}
|
||||
</style>
|
||||
@@ -33,18 +33,53 @@ const details = () => {
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex flex-col border-1 rounded px-2 py-1 border-gray-500 hover:bg-gray-100 dark:hover:bg-gray-700 ">
|
||||
<div class="flex flex-col border-1 rounded px-2 py-1 border-gray-500 hover:bg-gray-100/50 dark:hover:bg-gray-700/50 ">
|
||||
<div class="flex flex-row space-x-1 items-center place-content-center cursor-pointer" @click="details">
|
||||
<mdi:github v-if="doc._idx === 'repo'" class="flex flex-shrink-0" />
|
||||
<vscode-icons:file-type-typescript-official v-if="doc._idx === 'api' && doc.language === 'typescript'" class="flex flex-shrink-0" />
|
||||
<vscode-icons:file-type-rust v-if="doc._idx === 'api' && doc.language === 'rust'" class="flex" />
|
||||
<ant-design:file-markdown-outlined v-if="doc._idx === 'prose'" class="flex" />
|
||||
<div class="flex flex-shrink-0">{{doc.name}}</div>
|
||||
|
||||
<div class="flex flex-grow ml-1 font-light text-xs italic truncate">
|
||||
{{doc.description}}
|
||||
<div v-if="doc._idx === 'repo'" class="flex flex-row flex-grow space-x-1 items-center place-items-center">
|
||||
<mdi:github class="flex flex-shrink-0" />
|
||||
<div class="name font-semibold flex-shrink-0 pr-2">{{doc.name}}</div>
|
||||
<div class="description flex flex-grow font-light text-sm truncate text-gray-500">{{doc.description}}</div>
|
||||
</div>
|
||||
|
||||
<div v-if="doc._idx === 'api'">
|
||||
<vscode-icons:file-type-typescript-official v-if="doc.language === 'typescript'" class="flex flex-shrink-0" />
|
||||
<vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
|
||||
<span class="flex">{{ doc.name }}</span>
|
||||
</div>
|
||||
|
||||
<div v-if="doc._idx === 'prose'">
|
||||
<ant-design:file-markdown-outlined class="flex" />
|
||||
|
||||
</div>
|
||||
|
||||
<div v-if="doc._idx === 'consolidated'" class="w-full">
|
||||
<div v-if="doc.from === 'api'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
|
||||
<vscode-icons:file-type-rust v-if="doc.language === 'rust'" class="flex" />
|
||||
<vscode-icons:file-type-typescript-official v-else class="flex" />
|
||||
|
||||
<div class="symbolName font-semibold">{{doc.lvl0}}</div>
|
||||
<div class="text-sm font-light">{{doc.symbol}}</div>
|
||||
</div>
|
||||
|
||||
<!-- PROSE -->
|
||||
<div v-if="doc.from === 'prose'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
|
||||
<teenyicons:text-document-solid class="flex" />
|
||||
<div class="title font-semibold flex-shrink-0">{{doc.lvl0}}</div>
|
||||
<div class="title font-light truncate text-gray-500">{{doc.lvl1}}</div>
|
||||
</div>
|
||||
|
||||
<!-- REPOs -->
|
||||
<div v-if="doc.from === 'repo'" class="flex flex-row flex-grow space-x-2 place-items-center items-center">
|
||||
<mdi:github class="flex flex-shrink-0" />
|
||||
<div class="name font-semibold flex-shrink-0">{{doc.lvl0}}</div>
|
||||
<div class="description truncate flex-shrink text-gray-500 font-light">{{doc.lvl1}}</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div v-if="doc._idx === 'api'" class="flex text-xs font-medium px-1 py-0.5 bg-blue-500 dark:bg-blue-600 text-gray-50 rounded">
|
||||
{{apiKind}}
|
||||
</div>
|
||||
|
||||
50
packages/docs/src/components/SearchIndexes.vue
Normal file
50
packages/docs/src/components/SearchIndexes.vue
Normal file
@@ -0,0 +1,50 @@
|
||||
<script setup lang="ts">
|
||||
import { useSearch } from "~/modules/search";
|
||||
const search = useSearch();
|
||||
|
||||
/**
|
||||
* this allows all expected indexes to be shown and in a sensible order
|
||||
*/
|
||||
const knownIndexes = ["api", "prose", "repo", "consolidated"];
|
||||
/** the name of the indexes current known by MeiliSearch */
|
||||
const currentIndexes = computed(() => search.$state.indexes.map(i => i.name));
|
||||
|
||||
/**
|
||||
* we do want to see ANY index that exists though too
|
||||
*/
|
||||
const unknownIndexes = computed(() => {
|
||||
return search.$state.indexes.filter(i => !knownIndexes.includes(i.name));
|
||||
});
|
||||
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="search-indexes flex flex-col flex-grow rounded-md bg-gray-100/25 dark:bg-gray-800/25">
|
||||
|
||||
<div class="rounded-t-md overflow-hidden">
|
||||
<div class="font-bold text-lg rounded-t-md bg-gray-900/10 dark:bg-gray-100/10 py-3 px-4">
|
||||
INDEXES
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="search.$state.health" class="text-xs mt-2" >
|
||||
<div class="text-green-500">MeiliSearch Service Healthy!</div>
|
||||
<div class="db-size">db size: {{search.dbSize}}</div>
|
||||
</div>
|
||||
<span v-else class="text-red-500 ">
|
||||
Meilisearch is not available locally!
|
||||
</span>
|
||||
|
||||
<div class="index-list p-4 w-full flex flex-col ">
|
||||
<template v-for="idx in knownIndexes" :key="idx">
|
||||
<current-index v-if="currentIndexes.includes(idx)" :idx="search.$state.indexes.find(i => i.name === idx)"/>
|
||||
<missing-index v-else :idx="idx" />
|
||||
</template>
|
||||
<template v-for="idx in unknownIndexes" :key="idx">
|
||||
<current-index :idx="idx" :unknown="true" />
|
||||
</template>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,18 +1,26 @@
|
||||
<script setup lang="ts">
|
||||
import { useSearch } from "~/modules/search";
|
||||
const props = defineProps({
|
||||
query: {type: String, default: ""}
|
||||
});
|
||||
import { useSearch } from "~/modules/search";
|
||||
const s = useSearch();
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="search-stats flex flex-col rounded-md h-full">
|
||||
<div class="search-stats flex flex-col rounded-md h-full min-h-128">
|
||||
|
||||
<div class="rounded-t-md overflow-hidden">
|
||||
<div class="font-bold text-lg rounded-t-md bg-gray-900/10 dark:bg-gray-100/10 py-3 px-4">
|
||||
<div class="grid grid-cols-3 rounded-t-md bg-gray-900/10 dark:bg-gray-100/10 py-3 px-4 ">
|
||||
<div class="block text-sm font-light place-self-start self-center">
|
||||
{{s.$state.searchUsing.length}} <span class="italic">index(s) searched</span>
|
||||
</div>
|
||||
<div class="block font-bold text-lg place-self-center self-center">
|
||||
Results
|
||||
</div>
|
||||
<div class="block text-sm font-light place-self-end self-center">
|
||||
{{s.$state.searchResults.length}} <span class="italic">docs found</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="s.searchResults.length >0 && s.searchStatus ==='ready'" class="p-4 mb-2 space-y-2">
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useSearch } from "~/modules/search";
|
||||
const s = useSearch();
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="search-stats flex flex-col rounded-md bg-gray-100/25 dark:bg-gray-800/25">
|
||||
|
||||
<div class="rounded-t-md overflow-hidden">
|
||||
<div class="font-bold text-lg rounded-t-md bg-gray-900/10 dark:bg-gray-100/10 py-3 px-4">
|
||||
INDEXES
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="s.$state.health" class="text-xs mt-2" >
|
||||
<div class="text-green-500">MeiliSearch Service Healthy!</div>
|
||||
<div class="db-size">db size: {{s.dbSize}}</div>
|
||||
</div>
|
||||
<span v-else class="text-red-500 ">
|
||||
Meilisearch is not available locally!
|
||||
</span>
|
||||
<div class="p-4">
|
||||
<div
|
||||
v-for="idx in s.$state.indexes"
|
||||
:key="idx.uid"
|
||||
class="index flex flex-row w-full py-2 border-1 rounded border-gray-500 p-4 mt-4"
|
||||
>
|
||||
<span class="index-name flex flex-grow font-medium">{{idx.name}}</span>
|
||||
<v-tooltip >
|
||||
<span class="index-name font-light cursor-pointer">
|
||||
{{s.indexInfo(idx.name)?.numberOfDocuments}}
|
||||
</span>
|
||||
<template #popper>
|
||||
<span class="dark:text-gray-300">
|
||||
# of Docs
|
||||
</span>
|
||||
</template>
|
||||
</v-tooltip>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
const { t } = useI18n()
|
||||
const { t } = useI18n();
|
||||
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<main class="text-center text-gray-700 dark:text-gray-200 flex items-center justify-center flex-col ">
|
||||
<main class="text-center text-gray-700 dark:text-gray-200 flex flex-col items-center justify-center flex-col min-h-full ">
|
||||
<!-- HEADER -->
|
||||
<div class="flex flex-col justify-center bg-blue-900 text-gray-100 dark:bg-blue-800/50 w-full space-y-4 align-middle items-center justify-center py-2">
|
||||
<div class="flex flex-col ">
|
||||
@@ -18,8 +19,8 @@ const { t } = useI18n()
|
||||
<Footer />
|
||||
</div>
|
||||
|
||||
<!-- MAIN CONTENt -->
|
||||
<div class="flex max-w-4xl h-auto min-h-full dark:bg-black/25 p-4 rounded-lg w-4xl">
|
||||
<!-- MAIN CONTENT -->
|
||||
<div class="flex flex-grow max-w-6xl h-auto min-h-full dark:bg-black/25 p-4 rounded-lg w-6xl">
|
||||
<router-view />
|
||||
</div>
|
||||
</main>
|
||||
|
||||
@@ -1,22 +1,23 @@
|
||||
// register vue composition api globally
|
||||
import { ViteSSG } from 'vite-ssg'
|
||||
import generatedRoutes from 'virtual:generated-pages'
|
||||
import { setupLayouts } from 'virtual:generated-layouts'
|
||||
import FloatingVue from 'floating-vue'
|
||||
import 'floating-vue/dist/style.css'
|
||||
import App from './App.vue'
|
||||
import { ViteSSG } from "vite-ssg";
|
||||
import generatedRoutes from "virtual:generated-pages";
|
||||
import { setupLayouts } from "virtual:generated-layouts";
|
||||
import FloatingVue from "floating-vue";
|
||||
|
||||
import "floating-vue/dist/style.css";
|
||||
import App from "./App.vue";
|
||||
|
||||
// windicss layers
|
||||
import 'virtual:windi-base.css'
|
||||
import 'virtual:windi-components.css'
|
||||
import "virtual:windi-base.css";
|
||||
import "virtual:windi-components.css";
|
||||
// your custom styles here
|
||||
import './styles/main.css'
|
||||
import "./styles/main.css";
|
||||
// windicss utilities should be the last style import
|
||||
import 'virtual:windi-utilities.css'
|
||||
import "virtual:windi-utilities.css";
|
||||
// windicss devtools support (dev only)
|
||||
import 'virtual:windi-devtools'
|
||||
import "virtual:windi-devtools";
|
||||
|
||||
const routes = setupLayouts(generatedRoutes)
|
||||
const routes = setupLayouts(generatedRoutes);
|
||||
|
||||
// https://github.com/antfu/vite-ssg
|
||||
export const createApp = ViteSSG(
|
||||
@@ -24,7 +25,9 @@ export const createApp = ViteSSG(
|
||||
{ routes, base: import.meta.env.BASE_URL },
|
||||
(ctx) => {
|
||||
// install all modules under `modules/`
|
||||
Object.values(import.meta.globEager('./modules/*.ts')).map(i => i.install?.(ctx))
|
||||
ctx.app.use(FloatingVue)
|
||||
Object.values(import.meta.globEager("./modules/*.ts")).map(i => i.install?.(ctx));
|
||||
ctx.app.use(FloatingVue);
|
||||
},
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
import { acceptHMRUpdate, defineStore } from "pinia";
|
||||
import type { UserModule } from "~/types";
|
||||
import { MeiliSearchHealth, MeiliSearchIndex, MeiliSearchInterface, MeiliSearchResponse, MeiliSearchStats } from "~/types/meilisearch";
|
||||
import {ApiModel, ProseModel } from "tauri-search";
|
||||
|
||||
export function createIndexes() {
|
||||
console.group("Establishing known search indexes");
|
||||
|
||||
ApiModel.query.createIndex().then(() => console.log(`ApiModel index created`));
|
||||
ProseModel.query.createIndex().then(() => console.log(`ApiModel index created`));
|
||||
ApiModel.query.createIndex().then(() => console.log(`ApiModel index created`));
|
||||
}
|
||||
|
||||
//#region STORE
|
||||
export interface SearchState {
|
||||
@@ -12,6 +21,9 @@ export interface SearchState {
|
||||
/** the indexes which are defined on MeiliSearch */
|
||||
indexes: MeiliSearchInterface[];
|
||||
|
||||
/** indexes to use when searching */
|
||||
searchUsing: string[];
|
||||
|
||||
/** database stats for MeiliSearch */
|
||||
stats?: MeiliSearchStats;
|
||||
|
||||
@@ -24,19 +36,18 @@ export const useSearch = defineStore("search", ({
|
||||
state: () => ({
|
||||
health: "initializing",
|
||||
indexes: [],
|
||||
searchUsing: ["consolidated"],
|
||||
stats: undefined,
|
||||
searchStatus: "not-ready",
|
||||
searchResults: [],
|
||||
}) as SearchState,
|
||||
actions: {
|
||||
async search(text: string, indexes?: string[]) {
|
||||
async search(text: string) {
|
||||
if(text.trim()==="") {
|
||||
this.$state.searchResults = [];
|
||||
return;
|
||||
}
|
||||
if(!indexes) {
|
||||
indexes = this.$state.indexes.map(i => i.uid);
|
||||
}
|
||||
const indexes = this.$state.searchUsing;
|
||||
console.group(`searching for: "${text}"`);
|
||||
console.info(`search on ${indexes.length} indexes`, indexes);
|
||||
console.time("search");
|
||||
@@ -71,6 +82,13 @@ export const useSearch = defineStore("search", ({
|
||||
|
||||
return results;
|
||||
},
|
||||
toggleUseOfIndex(idx: string) {
|
||||
if(this.$state.searchUsing.includes(idx)) {
|
||||
this.$state.searchUsing = this.$state.searchUsing.filter(i => i !== idx);
|
||||
} else {
|
||||
this.$state.searchUsing = [...this.$state.searchUsing, idx];
|
||||
}
|
||||
},
|
||||
statsUpdate(s: MeiliSearchStats) {
|
||||
this.$state.stats = s;
|
||||
},
|
||||
@@ -78,10 +96,17 @@ export const useSearch = defineStore("search", ({
|
||||
this.$state.health = h;
|
||||
},
|
||||
indexUpdate(idx: MeiliSearchInterface[]) {
|
||||
const current = this.$state.indexes.map(i => i.name);
|
||||
const newList = idx.map(i => i.name);
|
||||
if(current.length === newList.length && newList.every(i => current.includes(i))) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.$state.indexes = idx;
|
||||
}
|
||||
},
|
||||
getters: {
|
||||
indexIsUsed: (state) => (idx: string) => state.searchUsing.includes(idx),
|
||||
dbSize: (state) => state.stats?.databaseSize || 0,
|
||||
indexInfo: (state) => (idx: string): MeiliSearchIndex | undefined => state.stats?.indexes[idx]
|
||||
}
|
||||
@@ -118,8 +143,11 @@ export const install: UserModule = ({ isClient }) => {
|
||||
s.healthUpdate(h);
|
||||
s.$state.searchStatus = !h ? "not-ready": "ready";
|
||||
|
||||
if(h === true && !isActive) resume();
|
||||
if(h === false && isActive) pause();
|
||||
if(h === true && !isActive.value) {
|
||||
createIndexes();
|
||||
resume();
|
||||
};
|
||||
if(h === false && isActive.value) pause();
|
||||
}, 1000, {immediate: true});
|
||||
|
||||
}
|
||||
@@ -157,10 +185,14 @@ function api(base: string = "http://localhost:7700") {
|
||||
}
|
||||
|
||||
async function health(): Promise<boolean> {
|
||||
try {
|
||||
const response = await (
|
||||
await fetch(api().health)
|
||||
).json() as MeiliSearchHealth;
|
||||
return response.status === "available";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function indexes() {
|
||||
|
||||
@@ -24,7 +24,7 @@ onStartTyping(() => {
|
||||
<template>
|
||||
<div class="py-3 w-full h-full">
|
||||
<h1 class="text-xl mb-4">
|
||||
Meilisearch Playground
|
||||
MeiliSearch Playground
|
||||
</h1>
|
||||
|
||||
<input
|
||||
@@ -44,8 +44,7 @@ onStartTyping(() => {
|
||||
|
||||
<div class="results-area grid grid-cols-3 gap-4 h-full mt-8 mx-4">
|
||||
<div class="flex flex-col col-span-1 space-y-4">
|
||||
<search-actions />
|
||||
<search-stats class="mt-4" />
|
||||
<search-indexes class="" />
|
||||
</div>
|
||||
|
||||
<div class="col-span-2 rounded-md bg-gray-100/25 dark:bg-gray-900/25">
|
||||
|
||||
@@ -23,5 +23,5 @@
|
||||
}
|
||||
},
|
||||
"include": ["src", "test", "vite.config.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
"exclude": ["dist", "node_modules",]
|
||||
}
|
||||
|
||||
@@ -6,21 +6,35 @@
|
||||
"license": "MIT",
|
||||
"author": "Ken Snyder<ken@ken.net>",
|
||||
"scripts": {
|
||||
"push-cache": "node bin/push-cache.js",
|
||||
"clean": "rimraf dist/* bin/*",
|
||||
"lint": "eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
|
||||
"prune": "docker system prune",
|
||||
"sitemap": "node bin/sitemap.js",
|
||||
"current-indexes": "node bin/current-indexes.js",
|
||||
"create-indexes": "node bin/create-indexes.js",
|
||||
"refresh-prose": "node bin/refresh-prose.js",
|
||||
"push-prose": "node bin/push-prose.js",
|
||||
"refresh-repos": "node bin/refresh-repos.js",
|
||||
"push-repos": "node bin/push-repos.js",
|
||||
"refresh-typescript": "node bin/refresh-typescript.js",
|
||||
"push-typescript": "node bin/push-typescript.js",
|
||||
"push-consolidated": "node bin/push-consolidated.js",
|
||||
"restart": "docker compose restart",
|
||||
"build:cli": "tsup src/cli/*.ts --format=esm,cjs --clean --sourcemap -d bin",
|
||||
"build:npm": "tsup src/index.ts --dts --format=esm,cjs --sourcemap --clean -d dist ",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest watch",
|
||||
"ts-ast": "node ./bin/ts-ast.js",
|
||||
"ts-ast-overview": "node ./bin/ts-ast-overview.js",
|
||||
"watch": "run-p watch:*",
|
||||
"watch:cli": "tsup src/cli/*.ts --format=esm,cjs --clean --sourcemap -d bin --watch",
|
||||
"watch:npm": "vite build --watch"
|
||||
"watch:cli": "tsup src/cli/*.ts --format=esm,cjs --sourcemap -d bin --watch",
|
||||
"watch:npm": "tsup src/index.ts --dts --format=esm,cjs --sourcemap -d dist --watch"
|
||||
},
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.mjs",
|
||||
"types": "dist/index.d.ts",
|
||||
"bin": {},
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.10",
|
||||
"dotenv": "^14.3.2",
|
||||
@@ -33,22 +47,19 @@
|
||||
"xxhash-wasm": "^1.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@jest/types": "^27.4.2",
|
||||
"@octokit/types": "^6.34.0",
|
||||
"@type-challenges/utils": "^0.1.1",
|
||||
"@types/jest": "^27.4.0",
|
||||
"@types/markdown-it": "^12.2.3",
|
||||
"@types/node": "^14.18.9",
|
||||
"@typescript-eslint/eslint-plugin": "^5.10.0",
|
||||
"@typescript-eslint/parser": "^5.10.0",
|
||||
"@vitest/ui": "^0.2.3",
|
||||
"@typescript-eslint/eslint-plugin": "^5.10.2",
|
||||
"@typescript-eslint/parser": "^5.10.2",
|
||||
"@vitest/ui": "^0.2.5",
|
||||
"axios": "^0.25.0",
|
||||
"changeset": "^0.2.6",
|
||||
"eslint": "^8.7.0",
|
||||
"eslint": "^8.8.0",
|
||||
"eslint-config-prettier": "^8.3.0",
|
||||
"eslint-plugin-cypress": "^2.12.1",
|
||||
"eslint-plugin-import": "^2.25.4",
|
||||
"eslint-plugin-jest": "^25.7.0",
|
||||
"eslint-plugin-prettier": "^4.0.0",
|
||||
"eslint-plugin-promise": "^6.0.0",
|
||||
"fx": "^20.0.2",
|
||||
@@ -57,12 +68,12 @@
|
||||
"prettier": "^2.5.1",
|
||||
"rimraf": "^3.0.2",
|
||||
"ts-node": "^10.4.0",
|
||||
"tsup": "^5.11.11",
|
||||
"typescript": "^4.6.0-dev.20220124",
|
||||
"tsup": "^5.11.13",
|
||||
"typescript": "^4.6.0-dev.20220131",
|
||||
"vite": "^2.7.13",
|
||||
"vite-plugin-dts": "^0.9.9",
|
||||
"vite-plugin-inspect": "^0.3.13",
|
||||
"vitest": "^0.2.3"
|
||||
"vitest": "^0.2.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14",
|
||||
|
||||
23
packages/tauri-search/src/cli/create-indexes.ts
Normal file
23
packages/tauri-search/src/cli/create-indexes.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
/* eslint-disable no-console */
|
||||
import { createIndexes } from "~/pipelines/createIndexes";
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const { skipping, created } = await createIndexes();
|
||||
if (skipping.length > 0) {
|
||||
console.log(
|
||||
`- the following indexes -- ${skipping.join(
|
||||
", "
|
||||
)} -- were already setup so skipping`
|
||||
);
|
||||
}
|
||||
if (created.length === 0) {
|
||||
console.log(`- it appears there were no other indexes to start`);
|
||||
} else {
|
||||
console.log(`- created indexes for: ${created.join(", ")}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error((err as Error).message);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
17
packages/tauri-search/src/cli/current-indexes.ts
Normal file
17
packages/tauri-search/src/cli/current-indexes.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
/* eslint-disable no-console */
|
||||
import { ApiModel } from "..";
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const result = await ApiModel.query.currentIndexes();
|
||||
console.log(`MeiliSearch currently has the following indexes:\n`);
|
||||
for (const r of result) {
|
||||
console.log(` > ${r.name} - created at ${r.createdAt}`);
|
||||
}
|
||||
console.log();
|
||||
} catch (err) {
|
||||
console.log(`Failed to get the list of indexes on MeiliSearch`);
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
@@ -1,5 +0,0 @@
|
||||
import { getOrgRepoList } from "~/utils/github/getOrgRepoList";
|
||||
|
||||
(async () => {
|
||||
await getOrgRepoList();
|
||||
})();
|
||||
88
packages/tauri-search/src/cli/push-cache.ts
Normal file
88
packages/tauri-search/src/cli/push-cache.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { REPO_DOCS_CACHE, TS_DOCS_CACHE } from "~/constants";
|
||||
import { ApiModel, ConsolidatedModel, ProseModel, RepoModel } from "~/models";
|
||||
import {
|
||||
proseDocsCacheFile,
|
||||
pushProseDocs,
|
||||
pushRepoDocs,
|
||||
pushTypescriptDocs,
|
||||
refreshProse,
|
||||
refreshRepos,
|
||||
refreshTypescript,
|
||||
} from "~/pipelines";
|
||||
import { pushConsolidatedDocs } from "~/pipelines/pushConsolidatedDocs";
|
||||
import { communicateTaskStatus } from "~/utils/communicateTaskStatus";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
|
||||
(async () => {
|
||||
console.log(`- Pushing ALL document caches into local MeiliSearch server`);
|
||||
|
||||
const { repo, branch } = getEnv();
|
||||
if (!existsSync(proseDocsCacheFile(repo, branch))) {
|
||||
await refreshProse(repo, branch);
|
||||
}
|
||||
|
||||
console.log(`- Pushing "prose" documents to MeiliSearch`);
|
||||
const proseTasks = await pushProseDocs(repo, branch);
|
||||
console.log(
|
||||
`- all ${proseTasks.length} documents were pushed via API; monitoring task status ...`
|
||||
);
|
||||
|
||||
await communicateTaskStatus(ProseModel, proseTasks, { timeout: 30000 });
|
||||
|
||||
console.log(`- Pushing Repo document cache into MeiliSearch`);
|
||||
|
||||
if (!existsSync(REPO_DOCS_CACHE)) {
|
||||
console.log("- No cache for Repo documents found, so refreshing cache first");
|
||||
await refreshRepos();
|
||||
}
|
||||
const { docs, errors, tasks: repoTasks } = await pushRepoDocs();
|
||||
console.log();
|
||||
if (errors.length > 0) {
|
||||
console.log(
|
||||
`- Completed pushing Repo docs to MeiliSearch but ${errors.length} of ${
|
||||
docs.length
|
||||
} encountered errors:\n\t${errors.map((e) => e.name).join(", ")}`
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(
|
||||
`- Completed pushing all ${docs.length} Repo docs to MeiliSearch; monitoring queue status`
|
||||
);
|
||||
|
||||
await communicateTaskStatus(RepoModel, repoTasks);
|
||||
|
||||
if (!existsSync(TS_DOCS_CACHE)) {
|
||||
console.log(`- The Typescript documents cache wasn't found; creating first`);
|
||||
await refreshTypescript(repo, branch);
|
||||
}
|
||||
|
||||
console.log(`- Starting update process for Typescript API documents`);
|
||||
const { errors, tasks } = await pushTypescriptDocs();
|
||||
console.log();
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log(
|
||||
`- Completed pushing Typescript docs to MeiliSearch but ${errors.length} of ${
|
||||
tasks.length
|
||||
} encountered errors and were not processed:\n\t${errors
|
||||
.map((e) => e.name)
|
||||
.join(", ")}`
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(
|
||||
`- Completed pushing all Typescript docs [${tasks.length}] to MeiliSearch. Now monitoring task progress ...`
|
||||
);
|
||||
communicateTaskStatus(ApiModel, tasks, { timeout: 45000 });
|
||||
}
|
||||
|
||||
const { tasks: consolidatedTasks } = await pushConsolidatedDocs(repo, branch);
|
||||
console.log();
|
||||
console.log(
|
||||
`- all consolidated documents [${tasks.length}] have been pushed to MeiliSearch queue`
|
||||
);
|
||||
|
||||
communicateTaskStatus(ConsolidatedModel, consolidatedTasks, { timeout: 75000 });
|
||||
}
|
||||
})();
|
||||
37
packages/tauri-search/src/cli/push-consolidated.ts
Normal file
37
packages/tauri-search/src/cli/push-consolidated.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { REPO_DOCS_CACHE, TS_DOCS_CACHE } from "~/constants";
|
||||
import {
|
||||
proseDocsCacheFile,
|
||||
refreshProse,
|
||||
refreshRepos,
|
||||
refreshTypescript,
|
||||
} from "~/pipelines";
|
||||
import { pushConsolidatedDocs } from "~/pipelines/pushConsolidatedDocs";
|
||||
import { communicateTaskStatus } from "~/utils/communicateTaskStatus";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
import { ConsolidatedModel } from "..";
|
||||
|
||||
(async () => {
|
||||
console.log(`- pushing all models into consolidated index`);
|
||||
const { repo, branch } = getEnv();
|
||||
|
||||
if (!existsSync(TS_DOCS_CACHE)) {
|
||||
console.log(`- The Typescript documents cache wasn't found; creating first`);
|
||||
await refreshTypescript(repo, branch);
|
||||
}
|
||||
if (!existsSync(REPO_DOCS_CACHE)) {
|
||||
console.log("- No cache for Repo documents found, so refreshing cache first");
|
||||
await refreshRepos();
|
||||
}
|
||||
if (!existsSync(proseDocsCacheFile(repo, branch))) {
|
||||
await refreshProse(repo, branch);
|
||||
}
|
||||
|
||||
const { tasks } = await pushConsolidatedDocs(repo, branch);
|
||||
console.log();
|
||||
console.log(
|
||||
`- all consolidated documents [${tasks.length}] have been pushed to MeiliSearch queue`
|
||||
);
|
||||
|
||||
communicateTaskStatus(ConsolidatedModel, tasks, { timeout: 75000 });
|
||||
})();
|
||||
20
packages/tauri-search/src/cli/push-prose.ts
Normal file
20
packages/tauri-search/src/cli/push-prose.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { existsSync } from "fs";
|
||||
import { pushProseDocs } from "~/pipelines/pushProseDocs";
|
||||
import { proseDocsCacheFile, refreshProse } from "~/pipelines/refreshProse";
|
||||
import { communicateTaskStatus } from "~/utils/communicateTaskStatus";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
import { ProseModel } from "..";
|
||||
|
||||
(async () => {
|
||||
const { repo, branch } = getEnv();
|
||||
if (!existsSync(proseDocsCacheFile(repo, branch))) {
|
||||
await refreshProse(repo, branch);
|
||||
}
|
||||
console.log(`- Pushing "prose" documents to MeiliSearch`);
|
||||
const tasks = await pushProseDocs(repo, branch);
|
||||
console.log(
|
||||
`- all ${tasks.length} documents were pushed via API; monitoring task status ...`
|
||||
);
|
||||
|
||||
await communicateTaskStatus(ProseModel, tasks, { timeout: 30000 });
|
||||
})();
|
||||
30
packages/tauri-search/src/cli/push-repos.ts
Normal file
30
packages/tauri-search/src/cli/push-repos.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { existsSync } from "fs";
|
||||
import { REPO_DOCS_CACHE } from "~/constants";
|
||||
import { pushRepoDocs, refreshRepos } from "~/pipelines";
|
||||
import { communicateTaskStatus } from "~/utils/communicateTaskStatus";
|
||||
import { RepoModel } from "..";
|
||||
|
||||
(async () => {
|
||||
console.log(`- Pushing Repo document cache into MeiliSearch`);
|
||||
|
||||
if (!existsSync(REPO_DOCS_CACHE)) {
|
||||
console.log("- No cache for Repo documents found, so refreshing cache first");
|
||||
await refreshRepos();
|
||||
}
|
||||
const { docs, errors, tasks } = await pushRepoDocs();
|
||||
console.log();
|
||||
if (errors.length > 0) {
|
||||
console.log(
|
||||
`- Completed pushing Repo docs to MeiliSearch but ${errors.length} of ${
|
||||
docs.length
|
||||
} encountered errors:\n\t${errors.map((e) => e.name).join(", ")}`
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(
|
||||
`- Completed pushing all ${docs.length} Repo docs to MeiliSearch; monitoring queue status`
|
||||
);
|
||||
|
||||
await communicateTaskStatus(RepoModel, tasks);
|
||||
}
|
||||
})();
|
||||
35
packages/tauri-search/src/cli/push-typescript.ts
Normal file
35
packages/tauri-search/src/cli/push-typescript.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { TS_DOCS_CACHE } from "~/constants";
|
||||
import { pushTypescriptDocs } from "~/pipelines/pushTypescriptDocs";
|
||||
import { refreshTypescript } from "~/pipelines/refreshTypescript";
|
||||
import { communicateTaskStatus } from "~/utils/communicateTaskStatus";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
import { ApiModel } from "..";
|
||||
|
||||
(async () => {
|
||||
const { repo, branch } = getEnv();
|
||||
if (!existsSync(TS_DOCS_CACHE)) {
|
||||
console.log(`- The Typescript documents cache wasn't found; creating first`);
|
||||
await refreshTypescript(repo, branch);
|
||||
}
|
||||
|
||||
console.log(`- Starting update process for Typescript API documents`);
|
||||
const { errors, tasks } = await pushTypescriptDocs();
|
||||
console.log();
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.log(
|
||||
`- Completed pushing Typescript docs to MeiliSearch but ${errors.length} of ${
|
||||
tasks.length
|
||||
} encountered errors and were not processed:\n\t${errors
|
||||
.map((e) => e.name)
|
||||
.join(", ")}`
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log(
|
||||
`- Completed pushing all Typescript docs [${tasks.length}] to MeiliSearch. Now monitoring task progress ...`
|
||||
);
|
||||
communicateTaskStatus(ApiModel, tasks, { timeout: 45000 });
|
||||
}
|
||||
})();
|
||||
@@ -1,5 +1,5 @@
|
||||
import { refreshProse } from "~/pipelines/refreshProse";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
import { refreshProse } from "~/utils/refreshProse";
|
||||
|
||||
(async () => {
|
||||
const { repo, branch, force } = getEnv();
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { refreshRepos } from "~/utils/refreshRepos";
|
||||
import { refreshRepos } from "~/pipelines/refreshRepos";
|
||||
|
||||
(async () => {
|
||||
await refreshRepos();
|
||||
console.log(`- Refreshing Repo info to cache`);
|
||||
const tasks = await refreshRepos();
|
||||
})();
|
||||
|
||||
13
packages/tauri-search/src/cli/refresh-typescript.ts
Normal file
13
packages/tauri-search/src/cli/refresh-typescript.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { TS_AST_CACHE, TS_DOCS_CACHE } from "~/constants";
|
||||
import { refreshTypescript } from "~/pipelines/refreshTypescript";
|
||||
import { getEnv } from "~/utils/getEnv";
|
||||
|
||||
(async () => {
|
||||
const { repo, branch } = getEnv();
|
||||
console.log(`- refreshing Typescript ASTs and Docs cache`);
|
||||
const docs = await refreshTypescript(repo, branch);
|
||||
console.log(`- completed caching of ${docs.length} TS API documents:`);
|
||||
console.log(` - AST Cache: ${TS_AST_CACHE}`);
|
||||
console.log(` - Doc Cache: ${TS_DOCS_CACHE}`);
|
||||
console.log();
|
||||
})();
|
||||
@@ -1,5 +0,0 @@
|
||||
import { typescriptPipeline } from "~/pipelines/typescriptPipeline";
|
||||
|
||||
(async () => {
|
||||
await typescriptPipeline();
|
||||
})();
|
||||
@@ -2,8 +2,13 @@ export const TAURI_BASE_URL = `https://tauri.studio`;
|
||||
export const TAURI_JS_DOCS_URL = `${TAURI_BASE_URL}/docs/api/js`;
|
||||
|
||||
export const GITHUB_API_BASE = `https://api.github.com`;
|
||||
|
||||
export const REPO_DOCS_CACHE = `src/generated/ast/repo/documents.json`;
|
||||
|
||||
export const TS_DOCS_CACHE = `src/generated/ast/api/ts-documents.json`;
|
||||
export const TS_AST_CACHE = `src/generated/ast/api/ts-ast.json`;
|
||||
export const RS_DOCS_CACHE = `src/generated/ast/api/rs-documents.json`;
|
||||
|
||||
export const REPOS: `${string}/${string}`[] = [
|
||||
"tauri-apps/tauri",
|
||||
"tauri-apps/wry",
|
||||
|
||||
1
packages/tauri-search/src/generated/ast/api/ts-ast.json
Normal file
1
packages/tauri-search/src/generated/ast/api/ts-ast.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,2 +1,3 @@
|
||||
export * from "./types";
|
||||
export * from "./models";
|
||||
// export * from "./pipelines";
|
||||
|
||||
@@ -1,4 +1,46 @@
|
||||
import { ModelMapper } from "../types";
|
||||
import { IApiModel, IRepoModel } from "..";
|
||||
import { ModelMapper, isRepoDocument, isApiDocument } from "~/types";
|
||||
import { IApiModel, IConsolidatedModel, IProseModel, IRepoModel } from "~/models";
|
||||
|
||||
export const ConsolidatedMapper: ModelMapper<IApiModel | IRepoModel, any> = (i) => ({});
|
||||
export const ConsolidatedMapper: ModelMapper<
|
||||
IApiModel | IRepoModel | IProseModel,
|
||||
IConsolidatedModel
|
||||
> = (i): IConsolidatedModel => ({
|
||||
id: i.id,
|
||||
lvl0: isRepoDocument(i) ? i.name : isApiDocument(i) ? i.name : i.title,
|
||||
lvl1: isRepoDocument(i)
|
||||
? i.description || null
|
||||
: isApiDocument(i)
|
||||
? i.module
|
||||
: i.tags?.join(" ") || null,
|
||||
lvl2: isRepoDocument(i)
|
||||
? i.kind || null
|
||||
: isApiDocument(i)
|
||||
? i.comment || null
|
||||
: i.sections?.join(" ") || null,
|
||||
lvl3: isRepoDocument(i)
|
||||
? i.topics?.join(" ") || null
|
||||
: isApiDocument(i)
|
||||
? i.language || null
|
||||
: i.subSections?.join(" ") || null,
|
||||
lvl4: isRepoDocument(i)
|
||||
? i.language || null
|
||||
: isApiDocument(i)
|
||||
? null
|
||||
: i.category || null,
|
||||
lvl5: isRepoDocument(i)
|
||||
? i.license || null
|
||||
: isApiDocument(i)
|
||||
? null
|
||||
: i.code?.join(" ") || null,
|
||||
lvl6: isRepoDocument(i) ? String(i.stars) || null : isApiDocument(i) ? null : null,
|
||||
from: isRepoDocument(i) ? "repo" : isApiDocument(i) ? "api" : "prose",
|
||||
symbol: isApiDocument(i) ? i.kind : null,
|
||||
language: isApiDocument(i)
|
||||
? i.language
|
||||
: isRepoDocument(i)
|
||||
? i.language
|
||||
: i.code?.pop() || null,
|
||||
|
||||
content: isRepoDocument(i) ? i.text : isApiDocument(i) ? null : i.text,
|
||||
url: i.url,
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { IRepoModel } from "~/models/RepoModel";
|
||||
import { url } from "~/types/aliases";
|
||||
import { GithubRepoResp, ModelMapper } from "~/types";
|
||||
import { sanitizeDocId } from "~/utils/sanitizeDocId";
|
||||
|
||||
/**
|
||||
* Maps Github Repo's API response to the appropriate document response for a Repo
|
||||
@@ -9,7 +10,7 @@ export const GithubMapper: ModelMapper<
|
||||
GithubRepoResp & { text: string | undefined },
|
||||
IRepoModel
|
||||
> = (i) => ({
|
||||
id: `github_${i.full_name.replace(/[\/-]/g, "_")}`,
|
||||
id: sanitizeDocId(`github_${i.full_name}`),
|
||||
name: i.name,
|
||||
description: i.description,
|
||||
kind: i.name.includes("plugin")
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import { TAURI_BASE_URL } from "~/constants";
|
||||
import { IProseModel } from "~/models/ProseModel";
|
||||
import { MarkdownAst } from "~/types/markdown";
|
||||
import { ModelMapper } from "../types";
|
||||
import { MarkdownAst, ModelMapper } from "~/types";
|
||||
import { sanitizeDocId } from "~/utils/sanitizeDocId";
|
||||
|
||||
/**
|
||||
* Map markdown AST to the appropriate document structure
|
||||
*/
|
||||
export const ProseMapper: ModelMapper<MarkdownAst, IProseModel> = (i) => ({
|
||||
id: `prose_${i.filepath.replace("/", "_")}_${i.filename}`,
|
||||
id: sanitizeDocId(`prose_${i.filepath}_${i.filename}`),
|
||||
title: i.frontmatter.title || i.h1.shift() || "UNKNOWN",
|
||||
tags: i.frontmatter.tags as string[],
|
||||
category: i.frontmatter.section as string,
|
||||
|
||||
@@ -2,6 +2,7 @@ import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import { IApiModel } from "~/models/ApiModel";
|
||||
import { ModelMapper, TypescriptSymbol } from "~/types";
|
||||
import { sanitizeDocId } from "~/utils/sanitizeDocId";
|
||||
|
||||
function symbolToUrl(module: string, kind: TypescriptKind, symbol: string) {
|
||||
switch (kind) {
|
||||
@@ -41,7 +42,7 @@ function symbolToDeclaration(i: TypescriptSymbol) {
|
||||
* Maps a Typescript symbol definition to an API document modeled as `IApiModel`
|
||||
*/
|
||||
export const TypescriptMapper: ModelMapper<TypescriptSymbol, IApiModel> = (i) => ({
|
||||
id: `ts_${i.module}_${i.kind}_${i.name}`,
|
||||
id: sanitizeDocId(`ts_${i.module}_${i.kind}_${i.name}`),
|
||||
name: i.name,
|
||||
kind: i.kind,
|
||||
module: i.module,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import { en } from "~/stop-words";
|
||||
import { createModel } from "~/utils/createModel";
|
||||
import { TsComment } from "..";
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import { TsComment } from "~/types";
|
||||
|
||||
export interface IApiModel {
|
||||
id: string;
|
||||
@@ -39,5 +39,5 @@ export interface IApiModel {
|
||||
export const ApiModel = createModel<IApiModel>("api", (c) =>
|
||||
c //
|
||||
.stopWords(en)
|
||||
.filterable("language", "kind", "module", "tags")
|
||||
.sortable("language", "kind", "module", "tags")
|
||||
);
|
||||
|
||||
13
packages/tauri-search/src/models/ConsolidatedModel.ts
Normal file
13
packages/tauri-search/src/models/ConsolidatedModel.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { en } from "~/stop-words";
|
||||
import { IScrapeSelectorTargets } from "~/types";
|
||||
import { createModel } from "~/utils/createModel";
|
||||
|
||||
export type IConsolidatedModel = IScrapeSelectorTargets & {
|
||||
from: "prose" | "api" | "repo";
|
||||
symbol: string | null;
|
||||
language: string | null;
|
||||
};
|
||||
|
||||
export const ConsolidatedModel = createModel<IConsolidatedModel>("consolidated", (c) =>
|
||||
c.stopWords(en)
|
||||
);
|
||||
@@ -18,7 +18,6 @@ export interface IRepoModel {
|
||||
topics?: string[];
|
||||
isTemplate?: boolean;
|
||||
|
||||
// latestVersion: string;
|
||||
lastUpdated: datetime;
|
||||
createdAt: datetime;
|
||||
|
||||
@@ -28,4 +27,6 @@ export interface IRepoModel {
|
||||
url: url;
|
||||
}
|
||||
|
||||
export const RepoModel = createModel<IRepoModel>("repo");
|
||||
export const RepoModel = createModel<IRepoModel>("repo", (c) =>
|
||||
c.searchable("name", "description", "topics")
|
||||
);
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
export * from "./RepoModel";
|
||||
export * from "./ApiModel";
|
||||
export * from "./ProseModel";
|
||||
export * from "./ConsolidatedModel";
|
||||
|
||||
3
packages/tauri-search/src/npm/README.md
Normal file
3
packages/tauri-search/src/npm/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# NPM directory
|
||||
|
||||
Holds files which are exported for use by consumers of the `tauri-search` dependency (of which the docs repo is a member of this probably small group)
|
||||
21
packages/tauri-search/src/npm/getCache.ts
Normal file
21
packages/tauri-search/src/npm/getCache.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { IApiModel, IProseModel, IRepoModel } from "~/models";
|
||||
|
||||
/**
|
||||
* Gets the document cache for a given index
|
||||
*/
|
||||
export async function getCache(
|
||||
index: "api" | "prose" | "repo",
|
||||
options: { repo?: string; branch?: string } = {}
|
||||
): Promise<(IProseModel | IRepoModel | IApiModel)[]> {
|
||||
switch (index) {
|
||||
case "prose":
|
||||
return (await import(`~/generated/ast/prose/tauri_dev/documents.json`))
|
||||
.default as IProseModel[];
|
||||
case "repo":
|
||||
return (await import("~/generated/ast/repo/documents.json"))
|
||||
.default as IRepoModel[];
|
||||
case "api":
|
||||
return (await import("~/generated/ast/api/ts-documents.json"))
|
||||
.default as IApiModel[];
|
||||
}
|
||||
}
|
||||
3
packages/tauri-search/src/npm/index.ts
Normal file
3
packages/tauri-search/src/npm/index.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
// export * from "./getCache";
|
||||
// export * from "./pushDocs";
|
||||
export const foobar = 42;
|
||||
37
packages/tauri-search/src/npm/pushDocs.ts
Normal file
37
packages/tauri-search/src/npm/pushDocs.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import {
|
||||
ApiModel,
|
||||
IApiModel,
|
||||
IProseModel,
|
||||
IRepoModel,
|
||||
IConsolidatedModel,
|
||||
ProseModel,
|
||||
RepoModel,
|
||||
ConsolidatedModel,
|
||||
} from "~/models";
|
||||
import {
|
||||
isApiDocument,
|
||||
isConsolidatedDocument,
|
||||
isProseDocument,
|
||||
isRepoDocument,
|
||||
MsTaskStatus,
|
||||
} from "~/types";
|
||||
|
||||
export async function pushDocs(
|
||||
docs: (IApiModel | IProseModel | IRepoModel | IConsolidatedModel)[],
|
||||
options: { repo?: string; branch?: string } = {}
|
||||
) {
|
||||
const t: Promise<MsTaskStatus>[] = [];
|
||||
for (const doc of docs) {
|
||||
if (isProseDocument(doc)) {
|
||||
t.push(ProseModel.query.addOrReplaceDocuments(doc));
|
||||
} else if (isRepoDocument(doc)) {
|
||||
t.push(RepoModel.query.addOrReplaceDocuments(doc));
|
||||
} else if (isApiDocument(doc)) {
|
||||
t.push(ApiModel.query.addOrReplaceDocuments(doc));
|
||||
} else if (isConsolidatedDocument(doc)) {
|
||||
t.push(ConsolidatedModel.query.addOrReplaceDocuments(doc));
|
||||
}
|
||||
}
|
||||
const tasks = await Promise.all(t);
|
||||
return tasks;
|
||||
}
|
||||
41
packages/tauri-search/src/pipelines/createIndexes.ts
Normal file
41
packages/tauri-search/src/pipelines/createIndexes.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { ProseModel, ApiModel, RepoModel } from "~/models";
|
||||
import { MsSettingsUpdate } from "..";
|
||||
|
||||
const models = {
|
||||
api: ApiModel,
|
||||
repo: RepoModel,
|
||||
prose: ProseModel,
|
||||
// consolidated: ConsolidatedModel,
|
||||
};
|
||||
|
||||
/**
|
||||
* Will add -- and configure -- all known indexes to MeiliSearch which aren't already
|
||||
* present in server.
|
||||
*/
|
||||
export async function createIndexes() {
|
||||
const skipping = (await ProseModel.query.currentIndexes()).map((i) => i.name);
|
||||
const created: string[] = [];
|
||||
for (const key of Object.keys(models)) {
|
||||
const model = models[key as keyof typeof models];
|
||||
if (!skipping.includes(model.name)) {
|
||||
created.push(key);
|
||||
// create the index
|
||||
console.log(await model.query.createIndex());
|
||||
// then update settings
|
||||
const indexSettings: MsSettingsUpdate<any> = {
|
||||
...(model.index.displayed ? { displayedAttributes: model.index.displayed } : {}),
|
||||
...(model.index.searchable
|
||||
? { searchableAttributes: model.index.searchable }
|
||||
: {}),
|
||||
...(model.index.filterable
|
||||
? { filterableAttributes: model.index.filterable }
|
||||
: {}),
|
||||
...(model.index.sortable ? { sortableAttributes: model.index.sortable } : {}),
|
||||
...(model.index.rules ? { rankingRules: model.index.rules } : {}),
|
||||
};
|
||||
await model.query.updateIndexSettings(indexSettings);
|
||||
}
|
||||
}
|
||||
|
||||
return { skipping, created };
|
||||
}
|
||||
7
packages/tauri-search/src/pipelines/index.ts
Normal file
7
packages/tauri-search/src/pipelines/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export * from "./createIndexes";
|
||||
export * from "./pushProseDocs";
|
||||
export * from "./pushRepoDocs";
|
||||
export * from "./pushTypescriptDocs";
|
||||
export * from "./refreshProse";
|
||||
export * from "./refreshRepos";
|
||||
export * from "./refreshTypescript";
|
||||
41
packages/tauri-search/src/pipelines/pushConsolidatedDocs.ts
Normal file
41
packages/tauri-search/src/pipelines/pushConsolidatedDocs.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { readFile } from "fs/promises";
|
||||
import { REPO_DOCS_CACHE, TS_DOCS_CACHE } from "~/constants";
|
||||
import { ConsolidatedMapper, IConsolidatedModel } from "~/mappers/ConsolidatedMapper";
|
||||
import {
|
||||
ConsolidatedModel,
|
||||
IApiModel,
|
||||
IMonitoredTask,
|
||||
IProseModel,
|
||||
IRepoModel,
|
||||
} from "..";
|
||||
import { proseDocsCacheFile } from "./refreshProse";
|
||||
|
||||
export async function pushConsolidatedDocs(repo: string, branch: string) {
|
||||
// gather documents
|
||||
const ts: IConsolidatedModel[] = (
|
||||
JSON.parse(await readFile(TS_DOCS_CACHE, "utf-8")) as IApiModel[]
|
||||
).map((c) => ConsolidatedMapper(c));
|
||||
// TODO: add in Rust API docs
|
||||
const prose: IConsolidatedModel[] = (
|
||||
JSON.parse(await readFile(proseDocsCacheFile(repo, branch), "utf-8")) as IProseModel[]
|
||||
).map((i) => ConsolidatedMapper(i));
|
||||
const repos: IConsolidatedModel[] = (
|
||||
JSON.parse(await readFile(REPO_DOCS_CACHE, "utf-8")) as IRepoModel[]
|
||||
).map((i) => ConsolidatedMapper(i));
|
||||
|
||||
// push into MeiliSearch task queue
|
||||
const errors: IConsolidatedModel[] = [];
|
||||
const tasks: IMonitoredTask[] = [];
|
||||
const docs = [...ts, ...prose, ...repos];
|
||||
for (const doc of docs) {
|
||||
const res = await ConsolidatedModel.query.addOrReplaceDocuments(doc);
|
||||
if (res.status !== "enqueued") {
|
||||
process.stdout.write("x");
|
||||
errors.push(doc);
|
||||
} else {
|
||||
process.stdout.write(".");
|
||||
tasks.push({ docId: doc.docId, taskId: res.uid });
|
||||
}
|
||||
}
|
||||
return { docs, tasks, errors };
|
||||
}
|
||||
23
packages/tauri-search/src/pipelines/pushProseDocs.ts
Normal file
23
packages/tauri-search/src/pipelines/pushProseDocs.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { readFile } from "fs/promises";
|
||||
import { IProseModel, ProseModel } from "~/models/ProseModel";
|
||||
import { IMonitoredTask } from "..";
|
||||
import { proseDocsCacheFile } from "./refreshProse";
|
||||
|
||||
/**
|
||||
* Pushes the cached prose documents into the MeiliSearch "prose" index
|
||||
*/
|
||||
export async function pushProseDocs(repo: string, branch: string) {
|
||||
const filename = proseDocsCacheFile(repo, branch);
|
||||
const cache = JSON.parse(await readFile(filename, "utf-8")) as IProseModel[];
|
||||
const tasks: IMonitoredTask[] = [];
|
||||
|
||||
for (const doc of cache) {
|
||||
tasks.push(
|
||||
await ProseModel.query
|
||||
.addOrReplaceDocuments(doc)
|
||||
.then((i) => ({ docId: doc.id, taskId: i.uid }))
|
||||
);
|
||||
}
|
||||
|
||||
return tasks;
|
||||
}
|
||||
27
packages/tauri-search/src/pipelines/pushRepoDocs.ts
Normal file
27
packages/tauri-search/src/pipelines/pushRepoDocs.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import { readFile } from "fs/promises";
|
||||
import { REPO_DOCS_CACHE } from "~/constants";
|
||||
import { IRepoModel, RepoModel } from "~/models";
|
||||
import { IMonitoredTask } from "..";
|
||||
|
||||
/**
|
||||
* Pushes the cached REPO documents into the MeiliSearch "repo" index
|
||||
*/
|
||||
export async function pushRepoDocs() {
|
||||
const docs = JSON.parse(await readFile(REPO_DOCS_CACHE, "utf-8")) as IRepoModel[];
|
||||
const errors: IRepoModel[] = [];
|
||||
const tasks: IMonitoredTask[] = [];
|
||||
|
||||
process.stdout.write(" ");
|
||||
for (const doc of docs) {
|
||||
const res = await RepoModel.query.addOrReplaceDocuments(doc);
|
||||
if (res.status !== "enqueued") {
|
||||
process.stdout.write("x");
|
||||
errors.push(doc);
|
||||
} else {
|
||||
process.stdout.write(".");
|
||||
tasks.push({ docId: doc.id, taskId: res.uid });
|
||||
}
|
||||
}
|
||||
|
||||
return { docs, errors, tasks };
|
||||
}
|
||||
28
packages/tauri-search/src/pipelines/pushTypescriptDocs.ts
Normal file
28
packages/tauri-search/src/pipelines/pushTypescriptDocs.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { readFile } from "fs/promises";
|
||||
import { TS_DOCS_CACHE } from "~/constants";
|
||||
import { ApiModel, IApiModel } from "~/models";
|
||||
import { IMonitoredTask } from "~/types";
|
||||
|
||||
/**
|
||||
* Iterates over each Typescript module and all of the
|
||||
* modules symbols and uses the `addOrUpdate` call to ensure
|
||||
* the index is fully up-to-date.
|
||||
*/
|
||||
export async function pushTypescriptDocs() {
|
||||
const docs = JSON.parse(await readFile(TS_DOCS_CACHE, "utf-8")) as IApiModel[];
|
||||
const errors: IApiModel[] = [];
|
||||
const tasks: IMonitoredTask[] = [];
|
||||
|
||||
for (const doc of docs) {
|
||||
const res = await ApiModel.query.addOrReplaceDocuments(doc);
|
||||
if (res.status !== "enqueued") {
|
||||
process.stdout.write("x");
|
||||
errors.push(doc);
|
||||
} else {
|
||||
process.stdout.write(".");
|
||||
tasks.push({ docId: doc.id, taskId: res.uid });
|
||||
}
|
||||
}
|
||||
|
||||
return { docs, tasks, errors };
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import path, { join } from "node:path";
|
||||
import { parseMarkdown } from "~/ast/parseMarkdown";
|
||||
import { ProseMapper } from "~/mappers";
|
||||
import { IProseModel } from "~/models/ProseModel";
|
||||
import { flattenSitemap, sitemapDictionary } from "./convertSitemap";
|
||||
import { buildDocsSitemap, IDocsSitemap } from "./github/buildDocsSitemap";
|
||||
import { flattenSitemap, sitemapDictionary } from "~/utils/convertSitemap";
|
||||
import { buildDocsSitemap, IDocsSitemap } from "~/utils/github/buildDocsSitemap";
|
||||
|
||||
/* eslint-disable no-console */
|
||||
export interface IRefreshProseOptions {
|
||||
@@ -26,12 +26,12 @@ async function write(file: string, data: string) {
|
||||
await writeFile(file, data, "utf-8");
|
||||
}
|
||||
|
||||
function documentsCacheFile(file: string, repo: string, branch: string) {
|
||||
export function proseDocsCacheFile(repo: string, branch: string) {
|
||||
const dir = `src/generated/ast/prose/${repo}_${branch}`;
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
return join(dir, file);
|
||||
return join(dir, "documents.json");
|
||||
}
|
||||
|
||||
async function cacheMarkdownAst(file: string, url: string, repo: string, branch: string) {
|
||||
@@ -65,6 +65,7 @@ export async function refreshProse(
|
||||
const newSitemap = await buildDocsSitemap({ repo, ref: branch });
|
||||
const flatmap = flattenSitemap(newSitemap);
|
||||
const documents: IProseModel[] = [];
|
||||
const unchangedDocuments: IProseModel[] = [];
|
||||
const unchanged: string[] = [];
|
||||
const changed: string[] = [];
|
||||
for (const file of flatmap) {
|
||||
@@ -80,6 +81,12 @@ export async function refreshProse(
|
||||
await cacheMarkdownAst(file.filepath, file.download_url, repo, branch)
|
||||
)
|
||||
);
|
||||
} else {
|
||||
unchangedDocuments.push(
|
||||
ProseMapper(
|
||||
await cacheMarkdownAst(file.filepath, file.download_url, repo, branch)
|
||||
)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
changed.push(file.filepath);
|
||||
@@ -93,21 +100,21 @@ export async function refreshProse(
|
||||
}
|
||||
if (changed.length === 0 && !options.force) {
|
||||
console.log(`- all AST cache files remain valid; nothing new written to cache`);
|
||||
if (!existsSync(proseDocsCacheFile(repo, branch))) {
|
||||
console.log(
|
||||
`- while AST files exist, the documents cache was missing and will be refreshed`
|
||||
);
|
||||
await writeFile(
|
||||
proseDocsCacheFile(repo, branch),
|
||||
JSON.stringify(unchangedDocuments)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`- finished writing markdown AST files [ ${changed.length} changed, ${unchanged.length} unchanged]`
|
||||
);
|
||||
await write(
|
||||
documentsCacheFile("documents.json", repo, branch),
|
||||
JSON.stringify(documents)
|
||||
);
|
||||
console.log(
|
||||
`- wrote Meilisearch documents to "${documentsCacheFile(
|
||||
"documents.json",
|
||||
repo,
|
||||
branch
|
||||
)}"`
|
||||
);
|
||||
await write(proseDocsCacheFile(repo, branch), JSON.stringify(documents));
|
||||
console.log(`- wrote Meilisearch documents to "${proseDocsCacheFile(repo, branch)}"`);
|
||||
}
|
||||
|
||||
if (currentSitemap) {
|
||||
@@ -1,10 +1,10 @@
|
||||
/* eslint-disable no-console */
|
||||
import { getRepo } from "./github/getRepo";
|
||||
import { getRepoReadme } from "./github/getRepoReadme";
|
||||
import { getRepo } from "~/utils/github/getRepo";
|
||||
import { getRepoReadme } from "~/utils/github/getRepoReadme";
|
||||
import { REPOS, REPO_DOCS_CACHE } from "~/constants";
|
||||
import { GithubMapper } from "~/mappers";
|
||||
import { GithubRepoResp } from "~/types";
|
||||
import { IRepoModel } from "..";
|
||||
import { IRepoModel } from "~/models";
|
||||
import { writeFile } from "fs/promises";
|
||||
|
||||
/**
|
||||
@@ -19,7 +19,6 @@ export async function refreshRepos() {
|
||||
const resp = getRepo(repo);
|
||||
repoPromise.push(resp);
|
||||
readmePromise.push(getRepoReadme(repo).then((i) => [repo, i]));
|
||||
// waitFor.push(model.query.addOrReplaceDoc(GithubMapper(resp)))
|
||||
}
|
||||
const readmes = (await Promise.all(readmePromise)).reduce((acc, tuple) => {
|
||||
return { ...acc, [tuple[0]]: tuple[1] };
|
||||
25
packages/tauri-search/src/pipelines/refreshTypescript.ts
Normal file
25
packages/tauri-search/src/pipelines/refreshTypescript.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { config } from "dotenv";
|
||||
import { writeFile } from "fs/promises";
|
||||
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
|
||||
import { TS_AST_CACHE, TS_DOCS_CACHE } from "~/constants";
|
||||
import { TypescriptMapper } from "~/mappers";
|
||||
import { IApiModel } from "..";
|
||||
|
||||
export async function refreshTypescript(repo: string, branch: string) {
|
||||
const prod = { repo, branch, filepath: "ts-api.json" };
|
||||
config();
|
||||
|
||||
const ast =
|
||||
process.env.NODE_ENV === "production"
|
||||
? await parseTypescriptAst(prod)
|
||||
: await parseTypescriptAst();
|
||||
|
||||
await writeFile(TS_AST_CACHE, JSON.stringify(ast));
|
||||
let docs: IApiModel[] = [];
|
||||
for (const i of ast.symbols) {
|
||||
docs.push(TypescriptMapper(i));
|
||||
}
|
||||
await writeFile(TS_DOCS_CACHE, JSON.stringify(docs));
|
||||
|
||||
return docs;
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
// import { ApiModel } from "~/models";
|
||||
// import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
|
||||
// import { TypescriptMapper } from "~/mappers/TypescriptMapper";
|
||||
// import { MsAddOrReplace } from "~/types";
|
||||
|
||||
/**
|
||||
* Will iterate over each Typescript _module_ and all of the
|
||||
* modules symbols and use the `addOrUpdate` call to ensure
|
||||
* the index is fully up-to-date.
|
||||
*/
|
||||
export async function typescriptPipeline() {
|
||||
// get AST from output of TSDoc's JSON option
|
||||
// const ast = await parseTypescriptAst();
|
||||
// const model = ApiModel;
|
||||
// const waitFor: Promise<MsAddOrReplace>[] = [];
|
||||
// for (const sym of ast.symbols) {
|
||||
// waitFor.push(TypescriptMapper())
|
||||
// // map AST model to the Document Model and add/update in MeiliSearch
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsModule.map(mod)));
|
||||
// // functions
|
||||
// for (const fn of mod.functions) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsFunction.map(fn)));
|
||||
// }
|
||||
// // classes
|
||||
// for (const c of mod.classes) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsClass.map(c)));
|
||||
// }
|
||||
// // interfaces
|
||||
// for (const i of mod.interfaces) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsInterface.map(i)));
|
||||
// }
|
||||
// // variables
|
||||
// for (const v of mod.variables) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsVariable.map(v)));
|
||||
// }
|
||||
// // enumerations
|
||||
// for (const e of mod.enums) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsEnumeration.map(e)));
|
||||
// }
|
||||
// // type-aliases
|
||||
// for (const ta of mod.typeAliases) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsTypeAlias.map(ta)));
|
||||
// }
|
||||
// // references
|
||||
// for (const ref of mod.references) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsReference.map(ref)));
|
||||
// }
|
||||
// }
|
||||
// const results = await Promise.all(waitFor);
|
||||
// console.log(results.map((i) => `${i?.indexUid}: ${i?.status}`));
|
||||
}
|
||||
@@ -4,6 +4,9 @@ export * from "./apis";
|
||||
export * from "./github";
|
||||
export * from "./type-guards";
|
||||
export * from "./mapping";
|
||||
export * from "./scraper";
|
||||
export * from "./markdown";
|
||||
export * from "./model";
|
||||
export * from "./meiliseach";
|
||||
export * from "./tasks";
|
||||
export * from "./utility";
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { AxiosRequestConfig } from "axios";
|
||||
import { ApiOptions } from "~/utils/MeiliSearchApi";
|
||||
import { RankingRule } from ".";
|
||||
|
||||
export interface MsIndexStatusResponse {
|
||||
@@ -19,6 +21,12 @@ export interface MsTaskStatus {
|
||||
indexUid: string;
|
||||
status: string;
|
||||
type: string;
|
||||
error?: {
|
||||
message: string;
|
||||
link?: string;
|
||||
type: string;
|
||||
code: string;
|
||||
};
|
||||
enqueuedAt: string;
|
||||
}
|
||||
|
||||
@@ -49,6 +57,32 @@ export interface MsSettingsResponse<T extends {}> {
|
||||
distinctAttribute: null | (keyof T)[] | ["*"];
|
||||
}
|
||||
|
||||
export interface MsSettingsUpdate<T extends {}> {
|
||||
/** List of associated words treated similarly. A word associated to an array of word as synonyms. */
|
||||
synonyms?: Record<string, string[]>;
|
||||
/** List of words ignored when present in search queries. */
|
||||
stopWords?: string[];
|
||||
/**
|
||||
* List of ranking rules sorted by order of importance. The order is customizable.
|
||||
*
|
||||
* The default is: words, typo, proximity, attribute, sort, exactness
|
||||
*/
|
||||
rankingRules?: RankingRule[];
|
||||
/** Search returns documents with distinct (different) values of the given field. */
|
||||
distinctAttribute?: null | keyof T;
|
||||
/** Fields in which to search for matching query words sorted by order of importance. */
|
||||
searchableAttributes?: (keyof T)[];
|
||||
/** Fields displayed in the returned documents. */
|
||||
displayedAttributes?: (keyof T)[];
|
||||
/**
|
||||
* Attributes to use for facetting and filtering. See
|
||||
* [Filtering and Faceted Search](https://docs.meilisearch.com/reference/features/filtering_and_faceted_search.html).
|
||||
*/
|
||||
filterableAttributes?: (keyof T)[];
|
||||
/** List of attributes to sort on at search. */
|
||||
sortableAttributes?: (keyof T)[];
|
||||
}
|
||||
|
||||
export interface MeiliSearchHealth {
|
||||
/** the current health status; is "available" when healthy */
|
||||
status: string;
|
||||
@@ -106,8 +140,7 @@ export type WithIndex<T extends MeiliSearchResponse> = Omit<T, "hits"> & {
|
||||
}[number];
|
||||
};
|
||||
|
||||
export interface MsIndexTasks {
|
||||
results: {
|
||||
export interface MsIndexTask {
|
||||
uid: number;
|
||||
indexUid: string;
|
||||
status: string;
|
||||
@@ -116,11 +149,20 @@ export interface MsIndexTasks {
|
||||
receiveDocuments: number;
|
||||
indexedDocuments: number;
|
||||
};
|
||||
error?: {
|
||||
message: string;
|
||||
link?: string;
|
||||
type: string;
|
||||
code: string;
|
||||
};
|
||||
duration: string;
|
||||
enqueuedAt: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface MsAllTasks {
|
||||
results: MsIndexTask[];
|
||||
}
|
||||
|
||||
export interface MsVersion {
|
||||
@@ -147,3 +189,54 @@ export interface MsKey {
|
||||
/** todo: check that this can be a number, it CAN be a null value */
|
||||
expiresAt: number | null;
|
||||
}
|
||||
|
||||
export interface MeiliSearchQueryApi<TDoc extends {}> {
|
||||
/**
|
||||
* Creates an index for the given model.
|
||||
*
|
||||
* Note: the primary key will be set to whatever is in your Model
|
||||
* which will be `id` unless stated otherwise.
|
||||
*/
|
||||
createIndex: () => Promise<MsTaskStatus>;
|
||||
getIndexTasks: () => Promise<MsAllTasks>;
|
||||
getDocument: (docId: string) => Promise<TDoc>;
|
||||
deleteDocument: (docId: string) => Promise<MsTaskStatus>;
|
||||
getDocuments: (o?: AxiosRequestConfig) => Promise<TDoc[]>;
|
||||
deleteAllDocuments: () => Promise<MsTaskStatus>;
|
||||
/**
|
||||
* Delete's an index on MeiliSeach (including all docs).
|
||||
*
|
||||
* If no index name is
|
||||
* given then it will delete the index associated with this model but you can
|
||||
* override that to whatever you like.
|
||||
*/
|
||||
deleteIndex: (idx?: string) => Promise<MsTaskStatus>;
|
||||
addOrReplaceDocuments: (doc: TDoc, o?: ApiOptions) => Promise<MsAddOrReplace>;
|
||||
addOrUpdateDocuments: (doc: TDoc, o?: ApiOptions) => Promise<MsAddOrReplace>;
|
||||
|
||||
search: (text: string) => Promise<MeiliSearchResponse>;
|
||||
|
||||
getAllIndexSettings: () => Promise<MsSettingsResponse<TDoc>>;
|
||||
updateIndexSettings: (settings: MsSettingsUpdate<TDoc>) => Promise<MsTaskStatus>;
|
||||
|
||||
resetIndexSettings: () => Promise<MsTaskStatus>;
|
||||
updateRankingRules: () => Promise<MsTaskStatus>;
|
||||
updateDistinctAttribute: () => Promise<MsTaskStatus>;
|
||||
updateSearchableAttributes: () => Promise<MsTaskStatus>;
|
||||
updateSortableAttributes: () => Promise<MsTaskStatus>;
|
||||
updateDisplayedAttributes: () => Promise<MsTaskStatus>;
|
||||
updateSynonyms: () => Promise<MsTaskStatus>;
|
||||
updateStopWords: () => Promise<MsTaskStatus>;
|
||||
|
||||
// cross-index
|
||||
|
||||
stats: () => Promise<MeiliSearchStats>;
|
||||
health: () => Promise<MeiliSearchHealth>;
|
||||
/** all of the indexes which currently exist in MeiliSearch */
|
||||
currentIndexes: () => Promise<MsIndexStatusResponse[]>;
|
||||
version: () => Promise<MsVersion>;
|
||||
getKeys: () => Promise<MsKey[]>;
|
||||
getTask: (id: number) => Promise<MsTaskStatus>;
|
||||
createKey: (key: MsKey) => Promise<MsTaskStatus>;
|
||||
deleteKey: (key: string) => Promise<MsTaskStatus>;
|
||||
}
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import { RankingRulesApi } from "~/types/apis";
|
||||
import { IndexSynonyms, RankingRule, RankingRulesApi } from "~/types/apis";
|
||||
import { MeiliSearchApi } from "~/utils/MeiliSearchApi";
|
||||
import { MeiliSearchQueryApi } from ".";
|
||||
|
||||
export type MeiliApi = ReturnType<typeof MeiliSearchApi>;
|
||||
|
||||
export type Wildcard<T> = (keyof T)[] | ["*"];
|
||||
|
||||
export type IndexApi<TDoc, TExclude extends string = never> = Omit<
|
||||
{
|
||||
searchable: (...props: (keyof TDoc)[]) => IndexApi<TDoc, TExclude | "searchable">;
|
||||
displayed: (...props: (keyof TDoc)[]) => IndexApi<TDoc, TExclude | "displayed">;
|
||||
distinct: (...props: (keyof TDoc)[]) => IndexApi<TDoc, TExclude | "distinct">;
|
||||
filterable: (...props: (keyof TDoc)[]) => IndexApi<TDoc, TExclude | "filterable">;
|
||||
sortable: (...props: (keyof TDoc)[]) => IndexApi<TDoc, TExclude | "sortable">;
|
||||
searchable: (...props: Wildcard<TDoc>) => IndexApi<TDoc, TExclude | "searchable">;
|
||||
displayed: (...props: Wildcard<TDoc>) => IndexApi<TDoc, TExclude | "displayed">;
|
||||
distinct: (...props: Wildcard<TDoc>) => IndexApi<TDoc, TExclude | "distinct">;
|
||||
filterable: (...props: Wildcard<TDoc>) => IndexApi<TDoc, TExclude | "filterable">;
|
||||
sortable: (...props: Wildcard<TDoc>) => IndexApi<TDoc, TExclude | "sortable">;
|
||||
/**
|
||||
* Because your website might provide content with structured English sentences, we
|
||||
* recommend adding stop words. Indeed, the search-engine would not be "spoiled" by
|
||||
@@ -37,3 +40,26 @@ export type IndexApi<TDoc, TExclude extends string = never> = Omit<
|
||||
},
|
||||
TExclude
|
||||
>;
|
||||
|
||||
/**
|
||||
* The interface/API surface which a **Model** exposes
|
||||
*/
|
||||
export type ISearchModel<TDoc extends {}> = {
|
||||
name: string;
|
||||
type: TDoc;
|
||||
index: {
|
||||
pk: string;
|
||||
rules?: RankingRule[];
|
||||
displayed?: Wildcard<TDoc>;
|
||||
searchable?: Wildcard<TDoc>;
|
||||
filterable?: Wildcard<TDoc>;
|
||||
distinct?: Wildcard<TDoc>;
|
||||
sortable?: Wildcard<TDoc>;
|
||||
stopWords?: string[];
|
||||
synonyms?: IndexSynonyms;
|
||||
};
|
||||
query: MeiliSearchQueryApi<TDoc>;
|
||||
toString(): string;
|
||||
};
|
||||
|
||||
export type ISearchConfig<TDoc extends {}> = Omit<ISearchModel<TDoc>, "query">;
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
export type ScrapeSelector =
|
||||
/** the simple representation is just to put a selector in as a string **/
|
||||
| string
|
||||
| null
|
||||
/** but there is an object notation for greater control */
|
||||
| {
|
||||
selector: string;
|
||||
@@ -18,7 +19,8 @@ export type ScrapeSelector =
|
||||
default_value?: string;
|
||||
};
|
||||
|
||||
export type ScrapeSelectorTargets = {
|
||||
export type IScrapeSelectorTargets = {
|
||||
id: string;
|
||||
lvl0: ScrapeSelector;
|
||||
lvl1: ScrapeSelector;
|
||||
lvl2: ScrapeSelector;
|
||||
@@ -27,7 +29,9 @@ export type ScrapeSelectorTargets = {
|
||||
lvl5: ScrapeSelector;
|
||||
lvl6: ScrapeSelector;
|
||||
/** the main body of text */
|
||||
text: ScrapeSelector;
|
||||
content: ScrapeSelector;
|
||||
|
||||
url: string;
|
||||
};
|
||||
|
||||
export type ScrapeUrls =
|
||||
@@ -67,7 +71,7 @@ export interface MeiliSearchConfig {
|
||||
* different pages (`default` is the fallback). To use this format you must also add
|
||||
* `selectors_key` props to the start_urls
|
||||
*/
|
||||
selectors: ScrapeSelectorTargets | Record<string, ScrapeSelectorTargets>;
|
||||
selectors: IScrapeSelectorTargets | Record<string, IScrapeSelectorTargets>;
|
||||
/**
|
||||
* This expects an array of CSS selectors. Any element matching one of those selectors
|
||||
* will be removed from the page before any data is extracted from it.
|
||||
|
||||
59
packages/tauri-search/src/types/tasks.ts
Normal file
59
packages/tauri-search/src/types/tasks.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { MsTaskStatus } from "./meiliseach";
|
||||
|
||||
export interface IMonitoredTask {
|
||||
/** the document's primary key */
|
||||
docId: string;
|
||||
/** the task ID assigned by Meilisearch */
|
||||
taskId: number;
|
||||
status?: string;
|
||||
error?: MsTaskStatus["error"];
|
||||
}
|
||||
|
||||
export enum TaskStatus {
|
||||
working,
|
||||
success,
|
||||
partialSuccess,
|
||||
failure,
|
||||
unknown,
|
||||
timeout,
|
||||
}
|
||||
|
||||
export interface IMonitoredTaskStatusWorking {
|
||||
status: TaskStatus.working;
|
||||
/** those documents which have successfully completed the task */
|
||||
successful: string[];
|
||||
/**
|
||||
* the document ID's -- along with a message -- which failed at the given task
|
||||
*/
|
||||
failed: { docId: string; message: string; link?: string }[];
|
||||
/** the document ID's which are still waiting to reach a final state */
|
||||
incomplete: IMonitoredTask[];
|
||||
/** document's which left the "enqueued" state but ended in an unknown/unexpected state */
|
||||
unknown: IMonitoredTask[];
|
||||
delta: { successful: string[]; failed: string[] };
|
||||
}
|
||||
|
||||
export interface IMonitoredTaskStatusComplete {
|
||||
status:
|
||||
| TaskStatus.success
|
||||
| TaskStatus.failure
|
||||
| TaskStatus.partialSuccess
|
||||
| TaskStatus.unknown
|
||||
| TaskStatus.timeout;
|
||||
/** those documents which have successfully completed the task */
|
||||
successful: string[];
|
||||
|
||||
failed: { docId: string; message: string; link?: string }[];
|
||||
incomplete: IMonitoredTask[];
|
||||
/** document's which left the "enqueued" state but ended in an unknown/unexpected state */
|
||||
unknown: IMonitoredTask[];
|
||||
}
|
||||
|
||||
export type IMonitoredTaskStatus =
|
||||
| IMonitoredTaskStatusWorking
|
||||
| IMonitoredTaskStatusComplete;
|
||||
|
||||
export interface ITaskStatusOptions {
|
||||
timeout?: number;
|
||||
callback?: (s: IMonitoredTaskStatus) => void;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Type } from "~/enums";
|
||||
import { IApiModel, IConsolidatedModel, IProseModel, IRepoModel } from "~/models";
|
||||
import { DocumentProperty, NumericLiteral, StringLiteral } from "./apis";
|
||||
|
||||
export function isStringLiteral(prop: DocumentProperty): prop is StringLiteral {
|
||||
@@ -8,3 +9,31 @@ export function isStringLiteral(prop: DocumentProperty): prop is StringLiteral {
|
||||
export function isNumericLiteral(prop: DocumentProperty): prop is NumericLiteral {
|
||||
return prop.type === Type.StringLiteral;
|
||||
}
|
||||
|
||||
export function isProseDocument(doc: unknown): doc is IProseModel {
|
||||
return (
|
||||
typeof doc === "object" &&
|
||||
"id" in (doc as Object) &&
|
||||
(doc as any)?.id.startsWith("prose_")
|
||||
);
|
||||
}
|
||||
|
||||
export function isApiDocument(doc: unknown): doc is IApiModel {
|
||||
return (
|
||||
typeof doc === "object" &&
|
||||
"id" in (doc as Object) &&
|
||||
((doc as any)?.id.startsWith("ts_") || (doc as any)?.id.startsWith("rs_"))
|
||||
);
|
||||
}
|
||||
|
||||
export function isRepoDocument(doc: unknown): doc is IRepoModel {
|
||||
return (
|
||||
typeof doc === "object" &&
|
||||
"id" in (doc as Object) &&
|
||||
(doc as any)?.id.startsWith("github_")
|
||||
);
|
||||
}
|
||||
|
||||
export function isConsolidatedDocument(doc: unknown): doc is IConsolidatedModel {
|
||||
return typeof doc === "object" && "lvl0" in (doc as Object);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import axios, { AxiosRequestConfig } from "axios";
|
||||
import {
|
||||
MeiliSearchHealth,
|
||||
MeiliSearchIndex,
|
||||
MeiliSearchResponse,
|
||||
MeiliSearchStats,
|
||||
MsVersion,
|
||||
MsAddOrReplace,
|
||||
MsIndexTasks,
|
||||
MsSettingsResponse,
|
||||
MsTaskStatus,
|
||||
MsKey,
|
||||
MsIndexStatusResponse,
|
||||
MsSettingsUpdate,
|
||||
MeiliSearchQueryApi,
|
||||
ISearchConfig,
|
||||
MsAllTasks,
|
||||
} from "~/types";
|
||||
|
||||
export interface MeiliSearchOptions {
|
||||
@@ -20,102 +23,149 @@ export type PagingOptions = {
|
||||
offset?: number;
|
||||
};
|
||||
|
||||
export type ApiOptions = Omit<RequestInit, "method">;
|
||||
export type ApiOptions = Omit<AxiosRequestConfig, "method">;
|
||||
|
||||
export function MeiliSearchApi<TDoc extends {}>(
|
||||
idx: string,
|
||||
model: ISearchConfig<TDoc>,
|
||||
options: MeiliSearchOptions = {}
|
||||
) {
|
||||
const baseUrl = options.url || "http://localhost:7700";
|
||||
const fetch = axios.create({
|
||||
baseURL: "baseURL",
|
||||
timeout: 1000,
|
||||
headers: {},
|
||||
});
|
||||
const baseURL = options.url || "http://localhost:7700";
|
||||
const idx = model.name;
|
||||
|
||||
const call = async <T>(
|
||||
method: "get" | "post" | "put" | "delete",
|
||||
url: string,
|
||||
options: AxiosRequestConfig = {}
|
||||
): Promise<T> => {
|
||||
const res = await fetch[method]<T>(url, options);
|
||||
return res.data;
|
||||
const fullUrl = `${baseURL}/${url.startsWith("/") ? url.slice(1) : url}`;
|
||||
|
||||
const res = await axios({
|
||||
method,
|
||||
url: fullUrl,
|
||||
...{
|
||||
...options,
|
||||
headers: {
|
||||
"Content-Type": options.data ? "application/json" : "application/text",
|
||||
},
|
||||
},
|
||||
}).catch((err) => {
|
||||
if (axios.isAxiosError(err)) {
|
||||
// request was made but server responded with non-200 response
|
||||
if (err.response) {
|
||||
switch (err.response.status) {
|
||||
case 401:
|
||||
throw new Error(
|
||||
`Unauthorized to use MeiliSearch endpoint [${method.toUpperCase()} /${url}]!`
|
||||
);
|
||||
case 404:
|
||||
throw new Error(
|
||||
`Couldn't find MeiliSearch endpoint [${method.toUpperCase()} /${url}]!`
|
||||
);
|
||||
|
||||
case 415:
|
||||
throw new Error(
|
||||
`\nUnsupported media type received by MeiliSearch endpoint [${method.toUpperCase()} /${url}]!\n - message: ${
|
||||
err.response.data?.message
|
||||
}\n - headers: ${JSON.stringify(err.response.headers, null, 2)}\n\n`
|
||||
);
|
||||
|
||||
case 429:
|
||||
throw new Error(
|
||||
`You are being rate limited on the MeiliSearch API [${method.toUpperCase()} /${url}]`
|
||||
);
|
||||
|
||||
default:
|
||||
throw new Error(
|
||||
`Unexpected Error [${
|
||||
err.response.status
|
||||
}] returned from MeiliSearch server [${method.toUpperCase()} /${url}]: [${
|
||||
err.response.status
|
||||
}] ${err.response.statusText}`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
});
|
||||
return res?.data;
|
||||
};
|
||||
|
||||
const get = <T>(url: string, options: AxiosRequestConfig = {}) => {
|
||||
return call("get", `${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`, options);
|
||||
return call<T>("get", url, options);
|
||||
};
|
||||
const put = <T>(
|
||||
const put = <T, U extends any = string>(
|
||||
url: string,
|
||||
data?: AxiosRequestConfig["data"],
|
||||
data?: string | U,
|
||||
options: AxiosRequestConfig = {}
|
||||
) => {
|
||||
return call<T>("put", `${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`, {
|
||||
return call<T>("put", url, {
|
||||
...options,
|
||||
data,
|
||||
});
|
||||
};
|
||||
const post = async <T>(
|
||||
const post = async <T, U extends any = string>(
|
||||
url: string,
|
||||
data?: AxiosRequestConfig["data"],
|
||||
data?: string | U,
|
||||
options: AxiosRequestConfig = {}
|
||||
): Promise<T> => {
|
||||
return call<T>("post", `${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`, {
|
||||
return call<T>("post", url, {
|
||||
...options,
|
||||
data,
|
||||
});
|
||||
};
|
||||
const del = <T>(url: string, options: AxiosRequestConfig = {}) => {
|
||||
return call<T>(
|
||||
"delete",
|
||||
`${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`,
|
||||
options
|
||||
);
|
||||
return call<T>("delete", url, options);
|
||||
};
|
||||
|
||||
const endpoints = {
|
||||
const endpoints: MeiliSearchQueryApi<TDoc> = {
|
||||
// per index
|
||||
getIndexTasks: get<MsIndexTasks>(`indexes/${idx}/tasks`),
|
||||
createIndex: () =>
|
||||
post<MsTaskStatus, { uid: string; primaryKey: string }>(`indexes`, {
|
||||
uid: model.name,
|
||||
primaryKey: model.index.pk,
|
||||
}),
|
||||
getIndexTasks: () => get<MsAllTasks>(`indexes/${idx}/tasks`),
|
||||
getDocument: (docId: string) => get<TDoc>(`indexes/${idx}/documents/${docId}`),
|
||||
deleteIndex: (indexName?: string) => del<MsTaskStatus>(`indexes/${indexName || idx}`),
|
||||
deleteDocument: (docId: string) =>
|
||||
del<MsTaskStatus>(`indexes/${idx}/documents/${docId}`),
|
||||
getDocuments: (o: AxiosRequestConfig = {}) => get<TDoc[]>(`indexes/${idx}/documents`, o),
|
||||
deleteAllDocuments: del<MsTaskStatus>(`indexes/${idx}/documents`),
|
||||
getDocuments: (o: AxiosRequestConfig = {}) =>
|
||||
get<TDoc[]>(`indexes/${idx}/documents`, o),
|
||||
deleteAllDocuments: () => del<MsTaskStatus>(`indexes/${idx}/documents`),
|
||||
addOrReplaceDocuments: (doc: TDoc, o: ApiOptions = {}) =>
|
||||
post<MsAddOrReplace>(`indexes/${idx}/documents`, JSON.stringify(doc), o),
|
||||
addOrUpdateDocuments: (doc: TDoc, o: ApiOptions = {}) =>
|
||||
put<MsAddOrReplace>(`indexes/${idx}/documents`, JSON.stringify(doc), o),
|
||||
search: (text: string) => get<MeiliSearchResponse>(`indexes/${idx}/search?q=${text}`),
|
||||
getAllIndexSettings: get<MsSettingsResponse<TDoc>>(`indexes/${idx}/settings`),
|
||||
updateIndexSettings: (settings: MsSettingsResponse<TDoc>) =>
|
||||
post<MsTaskStatus>(`indexes/${idx}/settings`, JSON.stringify(settings)),
|
||||
resetIndexSettings: del<MsTaskStatus>(`indexes/${idx}/settings`),
|
||||
updateRankingRules: post<MsTaskStatus>(`indexes/${idx}/settings/ranking-rules`),
|
||||
updateDistinctAttribute: post<MsTaskStatus>(
|
||||
`indexes/${idx}/settings/distinct-attribute`
|
||||
),
|
||||
updateSearchableAttributes: post<MsTaskStatus>(
|
||||
`indexes/${idx}/settings/searchable-attributes`
|
||||
),
|
||||
updateSortableAttributes: post<MsTaskStatus>(
|
||||
`indexes/${idx}/settings/sortable-attributes`
|
||||
),
|
||||
updateDisplayedAttributes: post<MsTaskStatus>(
|
||||
`indexes/${idx}/settings/displayed-attributes`
|
||||
),
|
||||
updateSynonyms: post<MsTaskStatus>(`indexes/${idx}/settings/synonyms`),
|
||||
updateStopWords: post<MsTaskStatus>(`indexes/${idx}/settings/stop-words`),
|
||||
getAllIndexSettings: () => get<MsSettingsResponse<TDoc>>(`indexes/${idx}/settings`),
|
||||
updateIndexSettings: (settings: MsSettingsUpdate<TDoc>) =>
|
||||
post<MsTaskStatus, MsSettingsUpdate<TDoc>>(`indexes/${idx}/settings`, settings),
|
||||
resetIndexSettings: () => del<MsTaskStatus>(`indexes/${idx}/settings`),
|
||||
updateRankingRules: () => post<MsTaskStatus>(`indexes/${idx}/settings/ranking-rules`),
|
||||
updateDistinctAttribute: () =>
|
||||
post<MsTaskStatus>(`indexes/${idx}/settings/distinct-attribute`),
|
||||
updateSearchableAttributes: () =>
|
||||
post<MsTaskStatus>(`indexes/${idx}/settings/searchable-attributes`),
|
||||
updateSortableAttributes: () =>
|
||||
post<MsTaskStatus>(`indexes/${idx}/settings/sortable-attributes`),
|
||||
updateDisplayedAttributes: () =>
|
||||
post<MsTaskStatus>(`indexes/${idx}/settings/displayed-attributes`),
|
||||
updateSynonyms: () => post<MsTaskStatus>(`indexes/${idx}/settings/synonyms`),
|
||||
updateStopWords: () => post<MsTaskStatus>(`indexes/${idx}/settings/stop-words`),
|
||||
|
||||
// cross-index
|
||||
|
||||
stats: get<MeiliSearchStats>(`stats`),
|
||||
health: get<MeiliSearchHealth>(`health`),
|
||||
indexes: get<MeiliSearchIndex>(`indexes`),
|
||||
version: get<MsVersion>(`version`),
|
||||
getKeys: get<MsKey[]>(`keys`),
|
||||
stats: () => get<MeiliSearchStats>(`stats`),
|
||||
health: () => get<MeiliSearchHealth>(`health`),
|
||||
/** all of the indexes which currently exist in MeiliSearch */
|
||||
currentIndexes: () => get<MsIndexStatusResponse[]>("indexes"),
|
||||
getTask: (id: number) => get<MsTaskStatus>(`tasks/${id}`),
|
||||
version: () => get<MsVersion>(`version`),
|
||||
getKeys: () => get<MsKey[]>(`keys`),
|
||||
createKey: (key: MsKey) => post<MsTaskStatus>(`keys`, JSON.stringify(key)),
|
||||
deleteKey: (key: string) => del<MsTaskStatus>(`keys/${key}`),
|
||||
};
|
||||
|
||||
return { get, put, post, delete: del, endpoints };
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
71
packages/tauri-search/src/utils/communicateTaskStatus.ts
Normal file
71
packages/tauri-search/src/utils/communicateTaskStatus.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { IMonitoredTask, ISearchModel, ITaskStatusOptions, TaskStatus } from "~/types";
|
||||
import { monitorTasks } from "./monitorTasks";
|
||||
|
||||
export async function communicateTaskStatus(
|
||||
model: ISearchModel<any>,
|
||||
tasks: IMonitoredTask[],
|
||||
options: ITaskStatusOptions = {}
|
||||
) {
|
||||
const final = await monitorTasks(model, tasks, options);
|
||||
console.log();
|
||||
|
||||
switch (final.status) {
|
||||
case TaskStatus.unknown:
|
||||
console.log(
|
||||
`- there were ${final.unknown.length} documents which ended in an unknown/unexpected state`
|
||||
);
|
||||
console.log(
|
||||
` - ${final.unknown.map((i) => `${i.docId}[${i.status}, ${i.taskId}]`)}`
|
||||
);
|
||||
console.log(`- there were also ${final.successful.length} successful ingestions`);
|
||||
if (final.failed.length > 0) {
|
||||
console.log(`- but also ${final.failed.length} failed ingestions`);
|
||||
}
|
||||
process.exit(1);
|
||||
|
||||
case TaskStatus.success:
|
||||
console.log(`- all documents successfully ingested by MeiliSearch`);
|
||||
break;
|
||||
|
||||
case TaskStatus.partialSuccess:
|
||||
console.log(
|
||||
`- ${final.successful.length} documents were ingested successfully but ${final.failed.length} failed. Failures detailed below:`
|
||||
);
|
||||
console.log(
|
||||
final.failed
|
||||
.map((f) => `${f.docId}: ${f.link || "<no help URL>"}\t${f.message}`)
|
||||
.join("\n")
|
||||
);
|
||||
process.exit(1);
|
||||
|
||||
case TaskStatus.timeout:
|
||||
console.log(`- The full set of documents was not completed in the TIMEOUT window.`);
|
||||
console.log(`- ${final.successful.length} documents were ingested successfully`);
|
||||
if (final.failed.length > 0) {
|
||||
console.log(
|
||||
`-e ${final.failed.length} documents failed to ingest: ${final.failed
|
||||
.map((f) => `${f.docId}[${f.link}]`)
|
||||
.join(", ")}`
|
||||
);
|
||||
}
|
||||
if (final?.unknown?.length > 0) {
|
||||
console.log(
|
||||
`- there were ${final.unknown.length} documents which ended in an unknown/unexpected state`
|
||||
);
|
||||
console.log(
|
||||
` - ${final.unknown.map((i) => `${i.docId}[${i.status}, ${i.taskId}]`)}`
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
`- ${
|
||||
final.incomplete.length
|
||||
} documents did not complete before the timeout:\n\t${final.incomplete
|
||||
.map((i) => `${i.docId}[${i.taskId}]`)
|
||||
.join(", ")}`
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -1,24 +1,13 @@
|
||||
import { IndexApi, IndexSynonyms, RankingRule, RankingRulesApi } from "~/types";
|
||||
import {
|
||||
IndexApi,
|
||||
ISearchConfig,
|
||||
ISearchModel,
|
||||
RankingRule,
|
||||
RankingRulesApi,
|
||||
} from "~/types";
|
||||
import { MeiliSearchApi } from "./MeiliSearchApi";
|
||||
import { rankingRules } from "./model-api/rankingRules";
|
||||
|
||||
export type ISearchModel<T extends {}> = {
|
||||
name: string;
|
||||
type: T;
|
||||
index: {
|
||||
rules?: RankingRule[];
|
||||
displayed?: (keyof T)[];
|
||||
searchable?: (keyof T)[];
|
||||
filterable?: (keyof T)[];
|
||||
distinct?: (keyof T)[];
|
||||
sortable?: (keyof T)[];
|
||||
stopWords?: string[];
|
||||
synonyms?: IndexSynonyms;
|
||||
};
|
||||
query: ReturnType<typeof MeiliSearchApi>;
|
||||
toString(): string;
|
||||
};
|
||||
|
||||
export type PartialModel<T extends {}> = Omit<Partial<ISearchModel<T>>, "index"> & {
|
||||
index: Partial<ISearchModel<T>["index"]>;
|
||||
};
|
||||
@@ -78,32 +67,34 @@ const modelConfigApi = <TDoc extends {}>(update: (s: PartialModel<TDoc>) => void
|
||||
return api();
|
||||
};
|
||||
|
||||
export const createModel = <T extends Record<string, any>>(
|
||||
export const createModel = <TDoc extends Record<string, any>>(
|
||||
/** the MeiliSearch index name which this model is servicing */
|
||||
name: string,
|
||||
cb?: (api: IndexApi<T>) => void,
|
||||
cb?: (api: IndexApi<TDoc>) => void,
|
||||
url: string = "http://localhost:7700"
|
||||
) => {
|
||||
const state: ISearchModel<T> = {
|
||||
const state: ISearchConfig<TDoc> = {
|
||||
name,
|
||||
type: {} as unknown as T,
|
||||
index: {},
|
||||
query: MeiliSearchApi(name, { url }),
|
||||
type: {} as unknown as TDoc,
|
||||
index: {
|
||||
pk: "id",
|
||||
},
|
||||
};
|
||||
const updateState = (s: PartialModel<T>) => {
|
||||
const updateState = (s: PartialModel<TDoc>) => {
|
||||
if (s.index) {
|
||||
state.index = { ...state.index, ...s.index };
|
||||
}
|
||||
};
|
||||
|
||||
if (cb) {
|
||||
cb(modelConfigApi<T>(updateState));
|
||||
cb(modelConfigApi<TDoc>(updateState));
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
query: MeiliSearchApi<TDoc>(state, { url }),
|
||||
toString() {
|
||||
return `Model(${name})`;
|
||||
return `Model(${name}[${state.index.pk}])`;
|
||||
},
|
||||
} as ISearchModel<T>;
|
||||
} as ISearchModel<TDoc>;
|
||||
};
|
||||
|
||||
@@ -1,11 +1,33 @@
|
||||
import axios from "axios";
|
||||
import { readFile } from "fs/promises";
|
||||
import { getRepoDefaultBranch } from "./github/getRepoDefaultBranch";
|
||||
import { getRepoFile } from "./github/getRepoFile";
|
||||
|
||||
export type IGetContentFallback =
|
||||
| { file: string; url?: undefined }
|
||||
| { url: string; file?: undefined };
|
||||
export type FileSource = { file: string; url?: undefined; repo?: undefined };
|
||||
export type UrlSource = { url: string; file?: undefined; repo?: undefined };
|
||||
export type RepoSource = {
|
||||
url?: undefined;
|
||||
file?: undefined;
|
||||
repo: `${string}/${string}`;
|
||||
filepath: string;
|
||||
branch?: string;
|
||||
};
|
||||
|
||||
export type IGetContentFallback = FileSource | UrlSource | RepoSource;
|
||||
export type IGetContentOptions = IGetContentFallback | {};
|
||||
|
||||
export function isRepoSource(source: IGetContentFallback): source is RepoSource {
|
||||
return "repo" in source && "branch" in source;
|
||||
}
|
||||
|
||||
export function isFileSource(source: IGetContentFallback): source is FileSource {
|
||||
return "file" in source;
|
||||
}
|
||||
|
||||
export function isUrlSource(source: IGetContentFallback): source is UrlSource {
|
||||
return "url" in source;
|
||||
}
|
||||
|
||||
/**
|
||||
* **getContent**
|
||||
*
|
||||
@@ -22,10 +44,16 @@ export const getContent =
|
||||
const config =
|
||||
Object.keys(options).length > 0 ? (options as IGetContentFallback) : fallback;
|
||||
let content: string;
|
||||
if (config?.file) {
|
||||
if (isFileSource(config)) {
|
||||
content = await readFile(config.file, { encoding: "utf-8" });
|
||||
} else if (config?.url) {
|
||||
} else if (isUrlSource(config)) {
|
||||
content = (await axios.get(config.url)).data;
|
||||
} else if (isRepoSource(config)) {
|
||||
content = (await getRepoFile(
|
||||
config.repo,
|
||||
config.filepath,
|
||||
config.branch
|
||||
)) as string;
|
||||
} else {
|
||||
throw new Error(`No content reference passed in!`);
|
||||
}
|
||||
|
||||
19
packages/tauri-search/src/utils/getUrl.ts
Normal file
19
packages/tauri-search/src/utils/getUrl.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import axios, { AxiosRequestConfig } from "axios";
|
||||
|
||||
export async function getUrl<T = unknown>(url: string, options: AxiosRequestConfig = {}) {
|
||||
const res = await axios.get<T>(url, options).catch((err) => {
|
||||
if (axios.isAxiosError(err) && err.response) {
|
||||
throw new Error(
|
||||
`\nProblems requesting the URL[${err.response.status}]: ${url}\n - message: ${err.response.data?.message}`
|
||||
);
|
||||
} else {
|
||||
throw new Error(
|
||||
`\nProblems requesting the URL [${err?.status || " "}]: ${url}\n - message: ${
|
||||
err.message
|
||||
}`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return res.data;
|
||||
}
|
||||
@@ -3,21 +3,25 @@ import { GITHUB_API_BASE } from "~/constants";
|
||||
import { GithubRepoResp } from "~/types";
|
||||
import { getEnv } from "../getEnv";
|
||||
|
||||
export async function getRepo(ownerRepo: `${string}/${string}`): Promise<GithubRepoResp> {
|
||||
const url = `${GITHUB_API_BASE}/repos/${ownerRepo}`;
|
||||
export async function getRepo(repo: `${string}/${string}`): Promise<GithubRepoResp> {
|
||||
const url = `${GITHUB_API_BASE}/repos/${repo}`;
|
||||
const { github_token, github_user } = getEnv();
|
||||
const res = await axios.get<GithubRepoResp>(url, {
|
||||
const res = await axios
|
||||
.get<GithubRepoResp>(url, {
|
||||
httpAgent: "Tauri Search",
|
||||
...(github_token && github_user
|
||||
? { auth: { username: github_user, password: github_token } }
|
||||
: {}),
|
||||
})
|
||||
.catch((err) => {
|
||||
if (axios.isAxiosError(err) && err.response) {
|
||||
throw new Error(
|
||||
`\nProblem calling Github API [repos, ${err.response.status}, ${url}]\n - message: ${err.response.data?.message}`
|
||||
);
|
||||
} else {
|
||||
throw new Error(`\nProblem calling Github API [repos, ${url}]: ${err.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
if (res.status < 300) {
|
||||
return res.data as GithubRepoResp;
|
||||
} else {
|
||||
throw new Error(
|
||||
`Problem getting Github repo at: ${url}. Error [${res.status}]: ${res.statusText}`
|
||||
);
|
||||
}
|
||||
return res.data;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import { getRepo } from "./getRepo";
|
||||
|
||||
export async function getRepoDefaultBranch(repo: `${string}/${string}`) {
|
||||
const r = await getRepo(repo);
|
||||
return r.default_branch;
|
||||
}
|
||||
21
packages/tauri-search/src/utils/github/getRepoFile.ts
Normal file
21
packages/tauri-search/src/utils/github/getRepoFile.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { getUrl } from "../getUrl";
|
||||
import { getRepoDefaultBranch } from "./getRepoDefaultBranch";
|
||||
|
||||
/**
|
||||
* Gets the raw content from a file in a github repo
|
||||
*
|
||||
* @param repo repo name in form of `owner/name`
|
||||
* @param filepath path from root of repo to the file (including file extension)
|
||||
* @param branch optionally specify a branch; will default to repo's default branch if not specified
|
||||
*/
|
||||
export async function getRepoFile(
|
||||
repo: `${string}/${string}`,
|
||||
filepath: string,
|
||||
branch?: string
|
||||
) {
|
||||
if (!branch) {
|
||||
branch = await getRepoDefaultBranch(repo);
|
||||
}
|
||||
const url = `https://raw.githubusercontent.com/${repo}/${branch}/${filepath}`;
|
||||
return getUrl(url);
|
||||
}
|
||||
149
packages/tauri-search/src/utils/monitorTasks.ts
Normal file
149
packages/tauri-search/src/utils/monitorTasks.ts
Normal file
@@ -0,0 +1,149 @@
|
||||
import {
|
||||
IMonitoredTask,
|
||||
IMonitoredTaskStatus,
|
||||
IMonitoredTaskStatusComplete,
|
||||
IMonitoredTaskStatusWorking,
|
||||
ISearchModel,
|
||||
ITaskStatusOptions,
|
||||
TaskStatus,
|
||||
} from "~/types";
|
||||
import { wait } from "./wait";
|
||||
|
||||
function defaultCallback(status: IMonitoredTaskStatus) {
|
||||
if (areWorkingTasks(status)) {
|
||||
for (const _s of status.delta.successful) {
|
||||
process.stdout.write(".");
|
||||
}
|
||||
for (const _f of status.delta.failed) {
|
||||
process.stdout.write("x");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function areCompletedTasks(
|
||||
status: IMonitoredTaskStatus
|
||||
): status is IMonitoredTaskStatusComplete {
|
||||
return status.status !== TaskStatus.working;
|
||||
}
|
||||
|
||||
export function areWorkingTasks(
|
||||
status: IMonitoredTaskStatus
|
||||
): status is IMonitoredTaskStatusWorking {
|
||||
return status.status === TaskStatus.working;
|
||||
}
|
||||
|
||||
async function getStatus(
|
||||
model: ISearchModel<any>,
|
||||
tasks: IMonitoredTask[],
|
||||
prior?: IMonitoredTaskStatus
|
||||
): Promise<IMonitoredTaskStatus> {
|
||||
if (prior && areCompletedTasks(prior)) {
|
||||
throw new Error(`Completed tasks sent to getStatus()!`);
|
||||
}
|
||||
const waitFor: Promise<IMonitoredTask>[] = [];
|
||||
|
||||
for (const t of tasks) {
|
||||
waitFor.push(
|
||||
model.query.getTask(t.taskId).then((task) => ({
|
||||
taskId: t.taskId,
|
||||
docId: t.docId,
|
||||
status: task?.status,
|
||||
error: task?.error,
|
||||
}))
|
||||
);
|
||||
}
|
||||
const results = await Promise.all(waitFor);
|
||||
const successful = results.filter((r) => r.status === "succeeded").map((i) => i.docId);
|
||||
const failed = results
|
||||
.filter((r) => r.status === "failed")
|
||||
.map((i) => ({
|
||||
docId: i.docId,
|
||||
message: i?.error?.message || "unknown",
|
||||
link: i?.error?.link,
|
||||
}));
|
||||
const incomplete = results.filter((r) =>
|
||||
["enqueued", "processing"].includes(r.status || "__")
|
||||
);
|
||||
const unknown = results.filter(
|
||||
(r) => !["enqueued", "processing", "succeeded", "failed"].includes(r.status || "___")
|
||||
);
|
||||
const status =
|
||||
incomplete.length > 0
|
||||
? TaskStatus.working
|
||||
: unknown.length > 0
|
||||
? TaskStatus.unknown
|
||||
: failed.length === 0
|
||||
? TaskStatus.success
|
||||
: results.length === 0
|
||||
? TaskStatus.failure
|
||||
: TaskStatus.partialSuccess;
|
||||
|
||||
return (
|
||||
status === TaskStatus.working
|
||||
? ({
|
||||
status,
|
||||
successful: [...successful, ...(prior?.successful || [])],
|
||||
failed: [...failed, ...(prior?.failed || [])],
|
||||
incomplete,
|
||||
unknown,
|
||||
delta: {
|
||||
successful: prior
|
||||
? successful.filter((i) => !prior.successful.includes(i))
|
||||
: successful,
|
||||
failed: prior
|
||||
? failed
|
||||
.map((i) => i.docId)
|
||||
.filter((i) => !prior.failed.map((f) => f.docId).includes(i))
|
||||
: failed,
|
||||
},
|
||||
} as IMonitoredTaskStatusWorking)
|
||||
: ({
|
||||
status,
|
||||
successful,
|
||||
failed,
|
||||
unknown,
|
||||
incomplete,
|
||||
} as IMonitoredTaskStatusComplete)
|
||||
) as IMonitoredTaskStatus;
|
||||
}
|
||||
|
||||
export const monitorTasks = async (
|
||||
model: ISearchModel<any>,
|
||||
tasks: IMonitoredTask[] | IMonitoredTask,
|
||||
options: ITaskStatusOptions
|
||||
): Promise<IMonitoredTaskStatusComplete> => {
|
||||
const DEFAULT_TIMEOUT = 60000;
|
||||
const INITIAL_DELAY = 500;
|
||||
const POLLING_INTERVAL = 250;
|
||||
const [TIMEOUT, statusCallback] = [
|
||||
options.timeout || DEFAULT_TIMEOUT,
|
||||
options.callback || defaultCallback,
|
||||
];
|
||||
|
||||
const isArray = Array.isArray(tasks);
|
||||
const t = !isArray
|
||||
? ([tasks] as unknown as IMonitoredTask[])
|
||||
: (tasks as unknown as IMonitoredTask[]);
|
||||
|
||||
let status = await getStatus(model, t);
|
||||
const start = Date.now();
|
||||
await wait(INITIAL_DELAY);
|
||||
|
||||
while (Date.now() - start < TIMEOUT && areWorkingTasks(status)) {
|
||||
status = await getStatus(model, status.incomplete, status);
|
||||
if (statusCallback) {
|
||||
statusCallback(status);
|
||||
}
|
||||
await wait(POLLING_INTERVAL);
|
||||
}
|
||||
|
||||
return areCompletedTasks(status)
|
||||
? status
|
||||
: {
|
||||
status: TaskStatus.timeout,
|
||||
successful: status.successful,
|
||||
failed: status.failed,
|
||||
unknown: status.unknown,
|
||||
incomplete: status.incomplete,
|
||||
};
|
||||
};
|
||||
3
packages/tauri-search/src/utils/sanitizeDocId.ts
Normal file
3
packages/tauri-search/src/utils/sanitizeDocId.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export function sanitizeDocId(id: string) {
|
||||
return id.replace(/[^a-zA-Z]/g, "_");
|
||||
}
|
||||
5
packages/tauri-search/src/utils/wait.ts
Normal file
5
packages/tauri-search/src/utils/wait.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export async function wait(time: number) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => resolve(time), time);
|
||||
});
|
||||
}
|
||||
@@ -1,8 +1,17 @@
|
||||
import { existsSync } from "fs";
|
||||
import { rm } from "fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { refreshRepos } from "~/utils/refreshRepos";
|
||||
import { REPO_DOCS_CACHE } from "~/constants";
|
||||
import { refreshRepos } from "~/pipelines/refreshRepos";
|
||||
|
||||
describe("refreshRepos()", () => {
|
||||
it("running refreshRepos() generates valid documents.json file", async () => {
|
||||
// turn on to test but otherwise it consumes a lot of Github calls
|
||||
it.skip("running refreshRepos() generates valid documents.json file", async () => {
|
||||
try {
|
||||
await rm(REPO_DOCS_CACHE);
|
||||
} catch {}
|
||||
|
||||
await refreshRepos();
|
||||
expect(existsSync(REPO_DOCS_CACHE)).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"noUnusedLocals": true,
|
||||
"strictNullChecks": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"types": [],
|
||||
"types": ["vite/client"],
|
||||
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
|
||||
@@ -20,8 +20,8 @@ export default defineConfig({
|
||||
entry: path.resolve(__dirname, "src/index.ts"),
|
||||
formats: ["es", "cjs"],
|
||||
},
|
||||
// watch: {},
|
||||
},
|
||||
|
||||
test: {
|
||||
include: ["test/**{test,spec}.{js,mjs,cjs,ts,mts}"],
|
||||
exclude: ["test/e2e/**"],
|
||||
@@ -35,5 +35,5 @@ export default defineConfig({
|
||||
// inline: ["@vue/test-utils", "@vue", "@vueuse", "vue-demi"],
|
||||
},
|
||||
},
|
||||
plugins: [dts(), inspect()],
|
||||
plugins: [dts({ logDiagnostics: true }), inspect()],
|
||||
} as UserConfig);
|
||||
|
||||
1854
pnpm-lock.yaml
generated
1854
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,7 @@
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"monorepoWorkspace.includeRoot": true
|
||||
"monorepoWorkspace.includeRoot": true,
|
||||
"i18n-ally.keystyle": "nested"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"noUnusedLocals": true,
|
||||
"strictNullChecks": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"types": [],
|
||||
"types": ["vite/client"],
|
||||
|
||||
"baseUrl": "."
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user