From 1efd5438619de2ad3d7169f0242b49c70d988e88 Mon Sep 17 00:00:00 2001 From: Ken Snyder Date: Mon, 31 Jan 2022 20:11:21 -0800 Subject: [PATCH] chore: all pipelines -- outside of Rust API -- are in place and playground starting to be useful in analyizing index characteristics --- package.json | 16 +- packages/docs/locales/de.yml | 5 + packages/docs/locales/es.yml | 6 + packages/docs/locales/fr.yml | 5 + packages/docs/package.json | 36 +- packages/docs/src/auto-imports.d.ts | 1 + packages/docs/src/components.d.ts | 16 +- packages/docs/src/components/Checkbox.vue | 19 + packages/docs/src/components/CurrentIndex.vue | 112 + packages/docs/src/components/MissingIndex.vue | 41 + .../docs/src/components/SearchActions.vue | 38 - packages/docs/src/components/SearchHit.vue | 53 +- .../docs/src/components/SearchIndexes.vue | 50 + .../docs/src/components/SearchResults.vue | 16 +- packages/docs/src/components/SearchStats.vue | 42 - packages/docs/src/layouts/default.vue | 9 +- packages/docs/src/main.ts | 33 +- packages/docs/src/modules/search.ts | 52 +- packages/docs/src/pages/search.vue | 5 +- packages/docs/tsconfig.json | 2 +- packages/tauri-search/package.json | 35 +- .../tauri-search/src/cli/create-indexes.ts | 23 + .../tauri-search/src/cli/current-indexes.ts | 17 + .../tauri-search/src/cli/githubRepoList.ts | 5 - packages/tauri-search/src/cli/push-cache.ts | 88 + .../tauri-search/src/cli/push-consolidated.ts | 37 + packages/tauri-search/src/cli/push-prose.ts | 20 + packages/tauri-search/src/cli/push-repos.ts | 30 + .../tauri-search/src/cli/push-typescript.ts | 35 + .../tauri-search/src/cli/refresh-prose.ts | 2 +- .../tauri-search/src/cli/refresh-repos.ts | 5 +- .../src/cli/refresh-typescript.ts | 13 + packages/tauri-search/src/cli/ts-pipeline.ts | 5 - packages/tauri-search/src/constants.ts | 5 + .../src/generated/ast/api/ts-ast.json | 1 + .../src/generated/ast/api/ts-documents.json | 1 + .../ast/prose/tauri_dev/documents.json | 2 +- .../src/generated/ast/repo/documents.json | 2 +- packages/tauri-search/src/index.ts | 1 + .../src/mappers/ConsolidatedMapper.ts | 48 +- .../tauri-search/src/mappers/GithubMapper.ts | 3 +- .../tauri-search/src/mappers/ProseMapper.ts | 6 +- .../src/mappers/TypescriptMapper.ts | 3 +- packages/tauri-search/src/models/ApiModel.ts | 6 +- .../src/models/ConsolidatedModel.ts | 13 + packages/tauri-search/src/models/RepoModel.ts | 5 +- packages/tauri-search/src/models/index.ts | 2 + packages/tauri-search/src/npm/README.md | 3 + packages/tauri-search/src/npm/getCache.ts | 21 + packages/tauri-search/src/npm/index.ts | 3 + packages/tauri-search/src/npm/pushDocs.ts | 37 + .../src/pipelines/createIndexes.ts | 41 + packages/tauri-search/src/pipelines/index.ts | 7 + .../src/pipelines/prosePipeline.ts | 0 .../src/pipelines/pushConsolidatedDocs.ts | 41 + .../src/pipelines/pushProseDocs.ts | 23 + .../src/pipelines/pushRepoDocs.ts | 27 + .../src/pipelines/pushTypescriptDocs.ts | 28 + .../src/{utils => pipelines}/refreshProse.ts | 37 +- .../src/{utils => pipelines}/refreshRepos.ts | 7 +- .../src/pipelines/refreshTypescript.ts | 25 + .../src/pipelines/typescriptPipeline.ts | 51 - packages/tauri-search/src/types/index.ts | 3 + packages/tauri-search/src/types/meiliseach.ts | 121 +- packages/tauri-search/src/types/model.ts | 38 +- packages/tauri-search/src/types/scraper.ts | 10 +- packages/tauri-search/src/types/tasks.ts | 59 + .../tauri-search/src/types/type-guards.ts | 29 + .../tauri-search/src/utils/MeiliSearchApi.ts | 156 +- .../src/utils/communicateTaskStatus.ts | 71 + .../tauri-search/src/utils/createModel.ts | 47 +- packages/tauri-search/src/utils/getContent.ts | 38 +- packages/tauri-search/src/utils/getUrl.ts | 19 + .../tauri-search/src/utils/github/getRepo.ts | 34 +- .../src/utils/github/getRepoDefaultBranch.ts | 6 + .../src/utils/github/getRepoFile.ts | 21 + .../tauri-search/src/utils/monitorTasks.ts | 149 ++ .../tauri-search/src/utils/sanitizeDocId.ts | 3 + packages/tauri-search/src/utils/wait.ts | 5 + .../test/caching/refreshRepos.test.ts | 13 +- packages/tauri-search/tsconfig.json | 2 +- packages/tauri-search/vite.config.ts | 4 +- pnpm-lock.yaml | 1854 +++++------------ tauri-search.code-workspace | 3 +- tsconfig.json | 2 +- 85 files changed, 2314 insertions(+), 1694 deletions(-) create mode 100644 packages/docs/src/components/Checkbox.vue create mode 100644 packages/docs/src/components/CurrentIndex.vue create mode 100644 packages/docs/src/components/MissingIndex.vue delete mode 100644 packages/docs/src/components/SearchActions.vue create mode 100644 packages/docs/src/components/SearchIndexes.vue delete mode 100644 packages/docs/src/components/SearchStats.vue create mode 100644 packages/tauri-search/src/cli/create-indexes.ts create mode 100644 packages/tauri-search/src/cli/current-indexes.ts delete mode 100644 packages/tauri-search/src/cli/githubRepoList.ts create mode 100644 packages/tauri-search/src/cli/push-cache.ts create mode 100644 packages/tauri-search/src/cli/push-consolidated.ts create mode 100644 packages/tauri-search/src/cli/push-prose.ts create mode 100644 packages/tauri-search/src/cli/push-repos.ts create mode 100644 packages/tauri-search/src/cli/push-typescript.ts create mode 100644 packages/tauri-search/src/cli/refresh-typescript.ts delete mode 100644 packages/tauri-search/src/cli/ts-pipeline.ts create mode 100644 packages/tauri-search/src/generated/ast/api/ts-ast.json create mode 100644 packages/tauri-search/src/generated/ast/api/ts-documents.json create mode 100644 packages/tauri-search/src/models/ConsolidatedModel.ts create mode 100644 packages/tauri-search/src/npm/README.md create mode 100644 packages/tauri-search/src/npm/getCache.ts create mode 100644 packages/tauri-search/src/npm/index.ts create mode 100644 packages/tauri-search/src/npm/pushDocs.ts create mode 100644 packages/tauri-search/src/pipelines/createIndexes.ts create mode 100644 packages/tauri-search/src/pipelines/index.ts delete mode 100644 packages/tauri-search/src/pipelines/prosePipeline.ts create mode 100644 packages/tauri-search/src/pipelines/pushConsolidatedDocs.ts create mode 100644 packages/tauri-search/src/pipelines/pushProseDocs.ts create mode 100644 packages/tauri-search/src/pipelines/pushRepoDocs.ts create mode 100644 packages/tauri-search/src/pipelines/pushTypescriptDocs.ts rename packages/tauri-search/src/{utils => pipelines}/refreshProse.ts (79%) rename packages/tauri-search/src/{utils => pipelines}/refreshRepos.ts (86%) create mode 100644 packages/tauri-search/src/pipelines/refreshTypescript.ts delete mode 100644 packages/tauri-search/src/pipelines/typescriptPipeline.ts create mode 100644 packages/tauri-search/src/types/tasks.ts create mode 100644 packages/tauri-search/src/utils/communicateTaskStatus.ts create mode 100644 packages/tauri-search/src/utils/getUrl.ts create mode 100644 packages/tauri-search/src/utils/github/getRepoDefaultBranch.ts create mode 100644 packages/tauri-search/src/utils/github/getRepoFile.ts create mode 100644 packages/tauri-search/src/utils/monitorTasks.ts create mode 100644 packages/tauri-search/src/utils/sanitizeDocId.ts create mode 100644 packages/tauri-search/src/utils/wait.ts diff --git a/package.json b/package.json index 8a7a940..b8bbcd5 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,18 @@ "start:tauri-search": "pnpm -C ./packages/tauri-search run watch", "start:docs": "pnpm -C ./packages/docs run watch", "build:cli": "pnpm -C ./packages/tauri-search run build:cli", - "watch": "pnpm run watch --filter ./packages/* ", - "test": "pnpm run test --filter ./packages/*", + "build:npm": "pnpm -C ./packages/tauri-search run build:npm", + "watch": "run-s watch:prep watch:rest", + "watch:prep": "pnpm -C ./packages/tauri-search run build:npm", + "watch:rest": "run-p watch:tauri-search watch:docs", + "watch:tauri-search": "pnpm -C ./packages/tauri-search run watch", + "watch:docs": "pnpm -C ./packages/docs run watch", + "test": "pnpm -C ./packages/docs run test && pnpm -C ./packages/tauri-search run test", + "test:watch": "pnpm -C ./packages/tauri-search run test:watch", + "current-indexes": "pnpm -C ./packages/tauri-search run current-indexes", + "create-indexes": "pnpm -C ./packages/tauri-search run create-indexes", "refresh-prose": "pnpm -C ./packages/tauri-search run refresh-prose", + "push-prose": "pnpm -C ./packages/tauri-search run push-prose", "up": "docker compose up -d", "down": "docker compose down", "into:scraper": "docker exec -it scraper bash", @@ -33,5 +42,8 @@ "docs": "pnpm -C ./packages/docs run dev", "npm": "pnpm -C ./packages/tauri-search run watch:npm", "npm:test": "pnpm -C ./packages/tauri-search run test" + }, + "devDependencies": { + "npm-run-all": "^4.1.5" } } diff --git a/packages/docs/locales/de.yml b/packages/docs/locales/de.yml index 8eca06b..e3dde10 100644 --- a/packages/docs/locales/de.yml +++ b/packages/docs/locales/de.yml @@ -16,3 +16,8 @@ home: search: Suche desc: Spielplatz für die Tauri-Suche whats-your-name: Suchen Sie nach etwas + ask-for-search: Suchen Sie nach etwas +nav: + indexing: Indizierung + toggle_langs: Sprachen ändern + toggle_dark: Dunkelmodus umschalten diff --git a/packages/docs/locales/es.yml b/packages/docs/locales/es.yml index 19f637c..02bff3c 100644 --- a/packages/docs/locales/es.yml +++ b/packages/docs/locales/es.yml @@ -15,3 +15,9 @@ not-found: No se ha encontrado home: search: Búsqueda whats-your-name: buscar algo + ask-for-search: buscar algo + desc: Documentación y Patio Interactivo +nav: + indexing: Indexación + toggle_dark: Alternar modo oscuro + toggle_langs: Cambiar idiomas diff --git a/packages/docs/locales/fr.yml b/packages/docs/locales/fr.yml index 870c80b..a4a373d 100644 --- a/packages/docs/locales/fr.yml +++ b/packages/docs/locales/fr.yml @@ -16,3 +16,8 @@ home: desc: Aire de jeux pour Tauri Search search: Chercher whats-your-name: Rechercher quelque chose + ask-for-search: Rechercher quelque chose +nav: + indexing: Indexage + toggle_dark: Basculer en mode sombre + toggle_langs: Changer de langue diff --git a/packages/docs/package.json b/packages/docs/package.json index 23383c4..3e5a635 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -5,39 +5,44 @@ "start": "pnpm run dev", "build": "vite-ssg build", "dev": "vite --port 3333 --open", - "lint": "eslint \"**/*.{vue,ts,js}\"", + "lint": "run-s lint:*", + "lint:eslint": "eslint \"**/*.{vue,ts,js}\"", + "lint:tsc": "vue-tsc --noEmit", "preview": "vite preview", "preview-https": "serve dist", "test": "vitest run", "test:e2e": "cypress open", "test:unit": "vitest", "test:dev": "vitest dev", - "typecheck": "vue-tsc --noEmit", + "test:watch": "vitest watch", "watch": "pnpm run dev" }, "dependencies": { - "@vueuse/core": "^7.5.4", + "@vueuse/core": "^7.5.5", "@vueuse/head": "^0.7.5", + "date-fns": "^2.28.0", "floating-vue": "^2.0.0-beta.3", "nprogress": "^0.2.0", - "pinia": "^2.0.9", + "pinia": "^2.0.11", "prism-theme-vars": "^0.2.2", "tauri-search": "workspace:*", "ts-morph": "^13.0.3", - "vue": "^3.2.28", + "vue": "^3.2.29", "vue-demi": "^0.12.1", "vue-i18n": "^9.1.9", "vue-router": "^4.0.12" }, "devDependencies": { - "@antfu/eslint-config": "^0.16.0", + "@antfu/eslint-config": "^0.16.1", "@iconify-json/ant-design": "^1.0.2", "@iconify-json/bx": "^1.0.3", "@iconify-json/carbon": "^1.0.14", "@iconify-json/cib": "^1.0.1", - "@iconify-json/fluent": "^1.0.15", + "@iconify-json/fluent": "^1.0.16", "@iconify-json/ic": "^1.0.8", - "@iconify-json/mdi": "^1.0.12", + "@iconify-json/iconoir": "^1.0.5", + "@iconify-json/mdi": "^1.0.15", + "@iconify-json/octicon": "^1.0.9", "@iconify-json/ph": "^1.0.4", "@iconify-json/tabler": "^1.0.12", "@iconify-json/teenyicons": "^1.0.1", @@ -46,13 +51,12 @@ "@types/markdown-it-link-attributes": "^3.0.1", "@types/nprogress": "^0.2.0", "@vitejs/plugin-vue": "^2.1.0", - "@vue/compiler-sfc": "^3.2.28", - "@vue/server-renderer": "^3.2.28", + "@vue/compiler-sfc": "^3.2.29", + "@vue/server-renderer": "^3.2.29", "@vue/test-utils": "^2.0.0-rc.18", "critters": "^0.0.16", "cross-env": "^7.0.3", - "cypress": "^9.3.1", - "eslint": "^8.7.0", + "eslint": "^8.8.0", "eslint-plugin-cypress": "^2.12.1", "https-localhost": "^4.7.0", "jsdom": "^19.0.0", @@ -62,17 +66,17 @@ "typescript": "^4.5.5", "unplugin-auto-import": "^0.5.11", "unplugin-icons": "^0.13.0", - "unplugin-vue-components": "^0.17.14", + "unplugin-vue-components": "^0.17.15", "vite": "^2.7.13", "vite-plugin-inspect": "^0.3.13", "vite-plugin-md": "^0.11.7", - "vite-plugin-pages": "^0.20.0", + "vite-plugin-pages": "^0.20.1", "vite-plugin-pwa": "^0.11.13", "vite-plugin-vue-layouts": "^0.5.0", "vite-plugin-windicss": "^1.6.3", - "vite-ssg": "^0.17.6", + "vite-ssg": "^0.17.9", "vitepress": "^0.21.6", - "vitest": "^0.1.27", + "vitest": "^0.2.5", "vue-tsc": "^0.31.1" } } diff --git a/packages/docs/src/auto-imports.d.ts b/packages/docs/src/auto-imports.d.ts index deef757..d2b4ccc 100644 --- a/packages/docs/src/auto-imports.d.ts +++ b/packages/docs/src/auto-imports.d.ts @@ -183,6 +183,7 @@ declare global { const useSpeechSynthesis: typeof import('@vueuse/core')['useSpeechSynthesis'] const useStorage: typeof import('@vueuse/core')['useStorage'] const useStorageAsync: typeof import('@vueuse/core')['useStorageAsync'] + const useStyleTag: typeof import('@vueuse/core')['useStyleTag'] const useSwipe: typeof import('@vueuse/core')['useSwipe'] const useTemplateRefsList: typeof import('@vueuse/core')['useTemplateRefsList'] const useTextSelection: typeof import('@vueuse/core')['useTextSelection'] diff --git a/packages/docs/src/components.d.ts b/packages/docs/src/components.d.ts index d713694..6b2015b 100644 --- a/packages/docs/src/components.d.ts +++ b/packages/docs/src/components.d.ts @@ -4,31 +4,39 @@ declare module 'vue' { export interface GlobalComponents { + 'AntDesign:apiTwotone': typeof import('~icons/ant-design/api-twotone')['default'] 'AntDesign:fileMarkdownOutlined': typeof import('~icons/ant-design/file-markdown-outlined')['default'] + 'Bx:bxCheckbox': typeof import('~icons/bx/bx-checkbox')['default'] + 'Bx:bxCheckboxChecked': typeof import('~icons/bx/bx-checkbox-checked')['default'] + 'Bx:bxCheckboxMinus': typeof import('~icons/bx/bx-checkbox-minus')['default'] 'Bx:bxSearchAlt': typeof import('~icons/bx/bx-search-alt')['default'] 'Carbon:document': typeof import('~icons/carbon/document')['default'] - 'Carbon:errorFilled': typeof import('~icons/carbon/error-filled')['default'] 'Carbon:listDropdown': typeof import('~icons/carbon/list-dropdown')['default'] - 'Carbon:nextFilled': typeof import('~icons/carbon/next-filled')['default'] CarbonLanguage: typeof import('~icons/carbon/language')['default'] CarbonMoon: typeof import('~icons/carbon/moon')['default'] CarbonSun: typeof import('~icons/carbon/sun')['default'] + Checkbox: typeof import('./components/Checkbox.vue')['default'] Counter: typeof import('./components/Counter.vue')['default'] + CurrentIndex: typeof import('./components/CurrentIndex.vue')['default'] 'Fluent:databaseSearch24Regular': typeof import('~icons/fluent/database-search24-regular')['default'] Footer: typeof import('./components/Footer.vue')['default'] + 'Iconoir:download': typeof import('~icons/iconoir/download')['default'] 'Mdi:folderHome': typeof import('~icons/mdi/folder-home')['default'] 'Mdi:github': typeof import('~icons/mdi/github')['default'] 'Mdi:languageRust': typeof import('~icons/mdi/language-rust')['default'] 'Mdi:languageTypescript': typeof import('~icons/mdi/language-typescript')['default'] + MissingIndex: typeof import('./components/MissingIndex.vue')['default'] + 'Octicon:trash16': typeof import('~icons/octicon/trash16')['default'] + 'Ph:info': typeof import('~icons/ph/info')['default'] 'Ph:linkLight': typeof import('~icons/ph/link-light')['default'] README: typeof import('./components/README.md')['default'] - SearchActions: typeof import('./components/SearchActions.vue')['default'] SearchHit: typeof import('./components/SearchHit.vue')['default'] + SearchIndexes: typeof import('./components/SearchIndexes.vue')['default'] SearchResults: typeof import('./components/SearchResults.vue')['default'] - SearchStats: typeof import('./components/SearchStats.vue')['default'] SimpleCard: typeof import('./components/SimpleCard.vue')['default'] 'Tabler:databaseImport': typeof import('~icons/tabler/database-import')['default'] 'Teenyicons:dockerOutline': typeof import('~icons/teenyicons/docker-outline')['default'] + 'Teenyicons:textDocumentSolid': typeof import('~icons/teenyicons/text-document-solid')['default'] 'VscodeIcons:fileTypeRust': typeof import('~icons/vscode-icons/file-type-rust')['default'] 'VscodeIcons:fileTypeTypescriptOfficial': typeof import('~icons/vscode-icons/file-type-typescript-official')['default'] } diff --git a/packages/docs/src/components/Checkbox.vue b/packages/docs/src/components/Checkbox.vue new file mode 100644 index 0000000..817debb --- /dev/null +++ b/packages/docs/src/components/Checkbox.vue @@ -0,0 +1,19 @@ + + + diff --git a/packages/docs/src/components/CurrentIndex.vue b/packages/docs/src/components/CurrentIndex.vue new file mode 100644 index 0000000..81bf7de --- /dev/null +++ b/packages/docs/src/components/CurrentIndex.vue @@ -0,0 +1,112 @@ + + + + + diff --git a/packages/docs/src/components/MissingIndex.vue b/packages/docs/src/components/MissingIndex.vue new file mode 100644 index 0000000..87c39fe --- /dev/null +++ b/packages/docs/src/components/MissingIndex.vue @@ -0,0 +1,41 @@ + + + + diff --git a/packages/docs/src/components/SearchActions.vue b/packages/docs/src/components/SearchActions.vue deleted file mode 100644 index 83d7399..0000000 --- a/packages/docs/src/components/SearchActions.vue +++ /dev/null @@ -1,38 +0,0 @@ - - - - - \ No newline at end of file diff --git a/packages/docs/src/components/SearchHit.vue b/packages/docs/src/components/SearchHit.vue index 2867a1a..7ac7ca7 100644 --- a/packages/docs/src/components/SearchHit.vue +++ b/packages/docs/src/components/SearchHit.vue @@ -33,18 +33,53 @@ const details = () => {