mirror of
https://github.com/tauri-apps/tauri-search.git
synced 2026-02-04 02:41:20 +01:00
chore: reduced number of mappers needed and removed the createMapper abstraction
This commit is contained in:
83
.eslintrc
Normal file
83
.eslintrc
Normal file
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"extends": ["plugin:vue/vue3-recommended", "prettier"],
|
||||
"plugins": ["cypress", "@typescript-eslint"],
|
||||
"parserOptions": {
|
||||
"parser": "@typescript-eslint/parser"
|
||||
},
|
||||
"settings": {
|
||||
// https://github.com/meteorlxy/eslint-plugin-prettier-vue#eslint-config
|
||||
"SFCBlocks": {}
|
||||
},
|
||||
"rules": {
|
||||
// "prettier-vue/prettier": [
|
||||
// "error",
|
||||
// {
|
||||
// "printWidth": 120,
|
||||
// "singleQuote": false,
|
||||
// "semi": true
|
||||
// }
|
||||
// ],
|
||||
"vue/no-lone-template": "off",
|
||||
"vue/no-multiple-template-root": "off",
|
||||
"vue/no-vmodel-argument": "off",
|
||||
/**
|
||||
* turn this back on once the false warning for using string literals goes away;
|
||||
* for now, so long as we express an "emits" clause then TS should keep us honest
|
||||
* here rather than relying on lint rules.
|
||||
*/
|
||||
"vue/require-explicit-emits": "off",
|
||||
"no-trailing-spaces": ["warn", { "skipBlankLines": true, "ignoreComments": true }],
|
||||
"no-console": "warn",
|
||||
"prefer-const": "error",
|
||||
|
||||
"semi": "off",
|
||||
"quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
|
||||
"no-unused-vars": "off",
|
||||
"curly": ["warn", "multi-line"],
|
||||
"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
|
||||
// our use of `-spec` files for testing prevents us using this
|
||||
"unicorn/filename-case": "off",
|
||||
// reduce has been getting a bad rap lately; its true that often
|
||||
// a filter or map would be clearer and equally as effective but
|
||||
// there are still some legit cases to use reduce
|
||||
"unicorn/no-array-reduce": "off",
|
||||
"unicorn/prevent-abbreviations": "off",
|
||||
"unicorn/no-null": "off",
|
||||
"no-nested-ternary": "off",
|
||||
// doesn't play well with prettier
|
||||
"unicorn/no-nested-ternary": "off",
|
||||
// this is kind of nice sometimes
|
||||
"unicorn/no-array-callback-reference": "off",
|
||||
|
||||
// we need exceptions to be only "warn" because
|
||||
// there are valid use cases for generic variables being
|
||||
// used before being defined
|
||||
"no-use-before-define": ["warn"],
|
||||
"@typescript-eslint/semi": ["error"],
|
||||
"@typescript-eslint/no-unsafe-member-access": "off",
|
||||
"@typescript-eslint/no-unsafe-call": "off",
|
||||
"@typescript-eslint/no-unsafe-assignment": "off",
|
||||
"@typescript-eslint/member-delimiter-style": [
|
||||
"error",
|
||||
{
|
||||
"multiline": {
|
||||
"delimiter": "semi",
|
||||
"requireLast": true
|
||||
},
|
||||
"singleline": {
|
||||
"delimiter": "semi",
|
||||
"requireLast": false
|
||||
}
|
||||
}
|
||||
],
|
||||
// "cases" allows for graceful use of that variable
|
||||
// name in Typescript test cases
|
||||
"@typescript-eslint/no-unused-vars": [
|
||||
"error",
|
||||
{
|
||||
"varsIgnorePattern": "cases|^_",
|
||||
"argsIgnorePattern": "^_"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,17 @@ services:
|
||||
# We are not using the scraper for anything currently but keeping it here
|
||||
# for now in case we decide to
|
||||
|
||||
# scraper:
|
||||
# image: getmeili/docs-scraper:latest
|
||||
# container_name: scraper
|
||||
# command: pipenv run ./docs_scraper config.json -d
|
||||
# depends_on:
|
||||
# - search
|
||||
# environment:
|
||||
# - MEILISEARCH_HOST_URL=localhost:7700
|
||||
# - MEILISEARCH_API_KEY=""
|
||||
# volumes:
|
||||
# - ./scraper:/data.ms
|
||||
scraper:
|
||||
image: getmeili/docs-scraper:latest
|
||||
container_name: scraper
|
||||
command: pipenv run ./docs_scraper config.json -d
|
||||
depends_on:
|
||||
- search
|
||||
environment:
|
||||
- MEILISEARCH_HOST_URL=localhost:7700
|
||||
- MEILISEARCH_API_KEY=""
|
||||
volumes:
|
||||
- ./scraper:/data.ms
|
||||
|
||||
search:
|
||||
image: getmeili/meilisearch:latest
|
||||
|
||||
@@ -8,13 +8,14 @@
|
||||
"start": "pnpm -r install && pnpm run start:tauri-search && pnpm run start:docs && pnpm run up",
|
||||
"start:tauri-search": "pnpm -C ./packages/tauri-search run watch",
|
||||
"start:docs": "pnpm -C ./packages/docs run watch",
|
||||
"build:cli": "pnpm -C ./packages/tauri-search run build:cli",
|
||||
"watch": "npx pnpm run -r watch",
|
||||
"test": "npx pnpm run -r test",
|
||||
"up": "docker compose up -d",
|
||||
"down": "docker compose down",
|
||||
"into:scraper": "docker exec -it scraper bash",
|
||||
"into:search": "docker exec -it search bash",
|
||||
"lint": "eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
|
||||
"lint": "pnpm -r eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
|
||||
"lint:docs": "pnpm -C docs run lint",
|
||||
"ping": "npx http GET localhost:7700/health --timeout 2",
|
||||
"prune": "docker system prune",
|
||||
|
||||
@@ -1,83 +1,3 @@
|
||||
{
|
||||
"extends": ["plugin:vue/vue3-recommended", "prettier"],
|
||||
"plugins": ["cypress", "@typescript-eslint"],
|
||||
"parserOptions": {
|
||||
"parser": "@typescript-eslint/parser"
|
||||
},
|
||||
"settings": {
|
||||
// https://github.com/meteorlxy/eslint-plugin-prettier-vue#eslint-config
|
||||
"SFCBlocks": {}
|
||||
},
|
||||
"rules": {
|
||||
// "prettier-vue/prettier": [
|
||||
// "error",
|
||||
// {
|
||||
// "printWidth": 120,
|
||||
// "singleQuote": false,
|
||||
// "semi": true
|
||||
// }
|
||||
// ],
|
||||
"vue/no-lone-template": "off",
|
||||
"vue/no-multiple-template-root": "off",
|
||||
"vue/no-vmodel-argument": "off",
|
||||
/**
|
||||
* turn this back on once the false warning for using string literals goes away;
|
||||
* for now, so long as we express an "emits" clause then TS should keep us honest
|
||||
* here rather than relying on lint rules.
|
||||
*/
|
||||
"vue/require-explicit-emits": "off",
|
||||
"no-trailing-spaces": ["warn", { "skipBlankLines": true, "ignoreComments": true }],
|
||||
"no-console": "warn",
|
||||
"prefer-const": "error",
|
||||
|
||||
"semi": "off",
|
||||
"quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
|
||||
"no-unused-vars": "off",
|
||||
"curly": ["warn", "multi-line"],
|
||||
"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
|
||||
// our use of `-spec` files for testing prevents us using this
|
||||
"unicorn/filename-case": "off",
|
||||
// reduce has been getting a bad rap lately; its true that often
|
||||
// a filter or map would be clearer and equally as effective but
|
||||
// there are still some legit cases to use reduce
|
||||
"unicorn/no-array-reduce": "off",
|
||||
"unicorn/prevent-abbreviations": "off",
|
||||
"unicorn/no-null": "off",
|
||||
"no-nested-ternary": "off",
|
||||
// doesn't play well with prettier
|
||||
"unicorn/no-nested-ternary": "off",
|
||||
// this is kind of nice sometimes
|
||||
"unicorn/no-array-callback-reference": "off",
|
||||
|
||||
// we need exceptions to be only "warn" because
|
||||
// there are valid use cases for generic variables being
|
||||
// used before being defined
|
||||
"no-use-before-define": ["warn"],
|
||||
"@typescript-eslint/semi": ["error"],
|
||||
"@typescript-eslint/no-unsafe-member-access": "off",
|
||||
"@typescript-eslint/no-unsafe-call": "off",
|
||||
"@typescript-eslint/no-unsafe-assignment": "off",
|
||||
"@typescript-eslint/member-delimiter-style": [
|
||||
"error",
|
||||
{
|
||||
"multiline": {
|
||||
"delimiter": "semi",
|
||||
"requireLast": true
|
||||
},
|
||||
"singleline": {
|
||||
"delimiter": "semi",
|
||||
"requireLast": false
|
||||
}
|
||||
}
|
||||
],
|
||||
// "cases" allows for graceful use of that variable
|
||||
// name in Typescript test cases
|
||||
"@typescript-eslint/no-unused-vars": [
|
||||
"error",
|
||||
{
|
||||
"varsIgnorePattern": "cases|^_",
|
||||
"argsIgnorePattern": "^_"
|
||||
}
|
||||
]
|
||||
}
|
||||
"extends": "../../.eslintrc"
|
||||
}
|
||||
|
||||
3
packages/tauri-search/.eslintrc
Normal file
3
packages/tauri-search/.eslintrc
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"extends": "../../.eslintrc"
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
"lint": "eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
|
||||
"prune": "docker system prune",
|
||||
"restart": "docker compose restart",
|
||||
"build:cli": "tsup src/cli/*.ts --format=esm,cjs --clean --sourcemap -d bin",
|
||||
"test": "vitest --ui",
|
||||
"ts-ast": "node ./bin/ts-ast.js",
|
||||
"ts-ast-overview": "node ./bin/ts-ast-overview.js",
|
||||
@@ -30,6 +31,7 @@
|
||||
"devDependencies": {
|
||||
"@jest/types": "^27.4.2",
|
||||
"@octokit/types": "^6.34.0",
|
||||
"@type-challenges/utils": "^0.1.1",
|
||||
"@types/jest": "^27.4.0",
|
||||
"@types/markdown-it": "^12.2.3",
|
||||
"@types/node": "^14.18.9",
|
||||
|
||||
@@ -1,31 +1,39 @@
|
||||
/* eslint-disable no-console */
|
||||
/* eslint-disable no-use-before-define */
|
||||
import { readFile } from "fs/promises";
|
||||
import xxhash from "xxhash-wasm";
|
||||
// import xxhash from "xxhash-wasm";
|
||||
import matter from "gray-matter";
|
||||
import smd from "simple-markdown";
|
||||
import { ITauriFrontmatter, MarkdownAst } from "~/types/markdown";
|
||||
|
||||
export function isHeading(something: string): something is "h1" | "h2" | "h3" {
|
||||
return ["h1", "h2", "h3"].includes(something);
|
||||
}
|
||||
|
||||
export type MarkdownAst = {
|
||||
filename: string;
|
||||
filepath: string;
|
||||
/** the full text of the markdown content */
|
||||
text: string;
|
||||
/** a hash indicating the current state of the document */
|
||||
hash: number;
|
||||
/** a key-value dictionary of frontmatter variables */
|
||||
frontmatter: Record<string, any>;
|
||||
h1: { content: string; type: string }[];
|
||||
h2: { content: string; type: string }[];
|
||||
h3: { content: string; type: string }[];
|
||||
/** boolean flag indicating whether there was a code block */
|
||||
hasCodeBlock: boolean;
|
||||
/** if there were code blocks, this will list the languages found */
|
||||
programmingLanguages: string[];
|
||||
/** other symbols found in the markdown that weren't specifically expressed */
|
||||
otherSymbols: string[];
|
||||
};
|
||||
function validateFrontmatter(f: string, matter: Record<string, any>) {
|
||||
const typedMatter = { ...matter } as ITauriFrontmatter;
|
||||
if (matter?.title && typeof matter.title !== "string") {
|
||||
console.error(
|
||||
`The frontmatter for "title" property needs to be a string but was a "${typeof matter.title}" in file ${f}.`
|
||||
);
|
||||
typedMatter.title = "UNKNOWN";
|
||||
}
|
||||
|
||||
if (matter?.tags && Array.isArray(matter.tags)) {
|
||||
console.error(
|
||||
`The frontmatter for "tags" property needs to be an array of strings but was detected as "${typeof matter.title}" in file ${f}.`
|
||||
);
|
||||
typedMatter.tags = [];
|
||||
}
|
||||
if (matter?.category && typeof matter.category !== "string") {
|
||||
console.error(
|
||||
`The frontmatter for "category" property needs to be a string but was a "${typeof matter.title}" in file ${f}.`
|
||||
);
|
||||
typedMatter.category = undefined;
|
||||
}
|
||||
|
||||
return typedMatter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes in a list of files and parses them in three ways:
|
||||
@@ -40,7 +48,7 @@ export type MarkdownAst = {
|
||||
* 3. finally it will also add the `filepath` and `filename` along with a content `hash`
|
||||
* which can be used detect whether content has changed
|
||||
*/
|
||||
export async function markdownParser(files: string[]) {
|
||||
export async function parseMarkdown(files: string[]) {
|
||||
// const { h32 } = await xxhash();
|
||||
|
||||
const tokens: MarkdownAst[] = [];
|
||||
@@ -58,7 +66,7 @@ export async function markdownParser(files: string[]) {
|
||||
filename,
|
||||
filepath,
|
||||
hash,
|
||||
frontmatter,
|
||||
frontmatter: validateFrontmatter(f, frontmatter),
|
||||
text,
|
||||
h1,
|
||||
h2,
|
||||
@@ -68,7 +76,7 @@ export async function markdownParser(files: string[]) {
|
||||
otherSymbols,
|
||||
});
|
||||
} catch (err) {
|
||||
err.message = `Problem parsing file ${f}: ${err.message}`;
|
||||
(err as Error).message = `Problem parsing file ${f}: ${(err as Error).message}`;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -96,7 +104,6 @@ function simpleParse(f: string, content: string) {
|
||||
|
||||
const extract = (nodeArray: smd.SingleASTNode[]) => {
|
||||
if (!Array.isArray(nodeArray)) {
|
||||
console.log("not an array", nodeArray);
|
||||
return;
|
||||
}
|
||||
for (const node of nodeArray) {
|
||||
@@ -109,7 +116,7 @@ function simpleParse(f: string, content: string) {
|
||||
if (Array.isArray(node.content)) {
|
||||
headings[tag].push(node.content[0] as { content: string; type: string });
|
||||
if (node.content.length > 1) {
|
||||
console.warn(
|
||||
console.error(
|
||||
`A heading tag in "${f}" was found which accumulated ${
|
||||
node.content.length
|
||||
} content elements in a single entry; only expected 1: ${node.content
|
||||
@@ -118,7 +125,7 @@ function simpleParse(f: string, content: string) {
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.warn(
|
||||
console.error(
|
||||
`The file ${f} got a headings tag that wasn't wrapped in an array element; this wasn't expected.`
|
||||
);
|
||||
}
|
||||
@@ -10,8 +10,9 @@ function parseModule(mod: TypescriptBlock) {
|
||||
name: mod.name,
|
||||
module: mod.name,
|
||||
type: mod.type,
|
||||
fileName: mod.sources?.shift()?.fileName,
|
||||
comment: mod.comment,
|
||||
fileName: mod.sources?.shift()?.fileName || "UNKNOWN",
|
||||
comment: mod?.comment?.text || mod?.comment?.text,
|
||||
commentTags: mod?.comment?.tags,
|
||||
children: [],
|
||||
};
|
||||
const symbols: TypescriptSymbol[] = [modDefn];
|
||||
@@ -21,9 +22,10 @@ function parseModule(mod: TypescriptBlock) {
|
||||
kind: i.kindString,
|
||||
name: i.name,
|
||||
module: mod.name,
|
||||
comment: i.comment,
|
||||
comment: i?.comment?.text || i?.comment?.text,
|
||||
commentTags: i?.comment?.tags,
|
||||
type: i.type,
|
||||
fileName: i.sources?.shift()?.fileName || "",
|
||||
fileName: i.sources?.shift()?.fileName || "UNKNOWN",
|
||||
signatures: i.signatures?.map((s) => ({
|
||||
name: s.name,
|
||||
kind: s.kindString,
|
||||
@@ -54,23 +56,17 @@ export async function parseTypescriptAst(
|
||||
): Promise<TsDocProject> {
|
||||
const content = JSON.parse(await fetchContent(source)) as TypescriptBlock;
|
||||
/**
|
||||
* The top level isn't probably worth putting into the index,
|
||||
* The top level "project" isn't probably worth putting into the index,
|
||||
* but instead we'll start at the modules level.
|
||||
*/
|
||||
const project: TsDocProject = {
|
||||
project: content.name,
|
||||
comment: content.comment,
|
||||
/**
|
||||
* flattened list of all TS symbols (including modules
|
||||
* and the symbols owned by them)
|
||||
*/
|
||||
symbols: [],
|
||||
};
|
||||
|
||||
for (const mod of content.children || []) {
|
||||
if (mod.kindString === "Namespace") {
|
||||
// each module will insert a row for itself
|
||||
// then then one each for each symbol it owns
|
||||
project.symbols.push(...parseModule(mod));
|
||||
} else {
|
||||
console.error(
|
||||
|
||||
20
packages/tauri-search/src/cli/get-symbol.ts
Normal file
20
packages/tauri-search/src/cli/get-symbol.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
/* eslint-disable no-console */
|
||||
import process from "node:process";
|
||||
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
|
||||
import { TypescriptMapper } from "~/mappers/TypescriptMapper";
|
||||
|
||||
(async () => {
|
||||
const symbols = (await parseTypescriptAst()).symbols;
|
||||
|
||||
const find = process.argv[2];
|
||||
const found = symbols.find((i) => i.name === find);
|
||||
if (found) {
|
||||
console.log({
|
||||
symbol: find,
|
||||
ast: found,
|
||||
doc: TypescriptMapper(found),
|
||||
});
|
||||
} else {
|
||||
console.error(`Didn't find the symbol ${find}!`);
|
||||
}
|
||||
})();
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// summarizes results from the `parseTypescriptAst()` function and returns to stdout
|
||||
|
||||
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
|
||||
|
||||
(async () => {
|
||||
const ast = await parseTypescriptAst();
|
||||
const overview = ast.modules.map((m) => ({
|
||||
module: m.name,
|
||||
fns: m.functions?.map((f) => f.name),
|
||||
interfaces: m.interfaces?.map((f) => f.name),
|
||||
typeAliases: m.typeAliases?.map((t) => t.name),
|
||||
classes: m.classes?.map((c) => c.name),
|
||||
variables: m.variables?.map((v) => v.name),
|
||||
references: m.references?.map((v) => v.name),
|
||||
other: m.other.map((o) => `${o.name} [${o.kindString}]`),
|
||||
}));
|
||||
|
||||
console.log(JSON.stringify(overview));
|
||||
})();
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
// summarizes results from the `parseTypescriptAst()` function and returns to stdout
|
||||
|
||||
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
|
||||
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
|
||||
|
||||
(async () => {
|
||||
const ast = await parseTypescriptAst();
|
||||
|
||||
4
packages/tauri-search/src/mappers/ConsolidatedMapper.ts
Normal file
4
packages/tauri-search/src/mappers/ConsolidatedMapper.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
import { ModelMapper } from "../types";
|
||||
import { IApiModel, IRepoModel } from "..";
|
||||
|
||||
export const ConsolidatedMapper: ModelMapper<IApiModel | IRepoModel, any> = (i) => ({});
|
||||
37
packages/tauri-search/src/mappers/GithubMapper.ts
Normal file
37
packages/tauri-search/src/mappers/GithubMapper.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { IRepoModel } from "~/models/RepoModel";
|
||||
import { url } from "~/types/aliases";
|
||||
import { GithubRepoResp } from "~/utils/github/getRepo";
|
||||
import { ModelMapper } from "..";
|
||||
|
||||
/**
|
||||
* Maps Github Repo's API response to the appropriate document response for a Repo
|
||||
*/
|
||||
export const GithubMapper: ModelMapper<GithubRepoResp["data"], IRepoModel> = (i) => ({
|
||||
id: `github_${i.full_name.replace("/", "_")}`,
|
||||
name: i.name,
|
||||
description: i.description,
|
||||
kind: i.name.includes("plugin")
|
||||
? "plugin"
|
||||
: i.language?.toLowerCase().includes("rust")
|
||||
? "code"
|
||||
: "unknown",
|
||||
|
||||
stars: i.stargazers_count,
|
||||
watchers: i.watchers_count,
|
||||
subscribers: i.subscribers_count,
|
||||
openIssues: i.open_issues_count,
|
||||
forks: i.forks_count,
|
||||
|
||||
defaultBranch: i.default_branch,
|
||||
language: i.language,
|
||||
topics: i.topics,
|
||||
isTemplate: i.is_template,
|
||||
|
||||
lastUpdated: i.updated_at,
|
||||
createdAt: i.created_at,
|
||||
license: i.license?.name,
|
||||
|
||||
text: "",
|
||||
|
||||
url: i.html_url as url,
|
||||
});
|
||||
19
packages/tauri-search/src/mappers/ProseMapper.ts
Normal file
19
packages/tauri-search/src/mappers/ProseMapper.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { TAURI_BASE_URL } from "~/constants";
|
||||
import { IProseModel } from "~/models/ProseModel";
|
||||
import { MarkdownAst } from "~/types/markdown";
|
||||
import { ModelMapper } from "../types";
|
||||
|
||||
/**
|
||||
* Map markdown AST to the appropriate document structure
|
||||
*/
|
||||
export const ProseMapper: ModelMapper<MarkdownAst, IProseModel> = (i) => ({
|
||||
id: `prose_${i.filepath.replace("/", "_")}_${i.filename}`,
|
||||
title: i.frontmatter.title || i.h1.shift() || "UNKNOWN",
|
||||
tags: i.frontmatter.tags as string[],
|
||||
category: i.frontmatter.section as string,
|
||||
sections: i.h2.map((i) => i.content),
|
||||
subSections: i.h3.map((i) => i.content),
|
||||
code: i.programmingLanguages,
|
||||
text: i.text,
|
||||
url: `${TAURI_BASE_URL}/docs/${i.filepath}/${i.filename.replace(".md", "")}`,
|
||||
});
|
||||
6
packages/tauri-search/src/mappers/RustMapper.ts
Normal file
6
packages/tauri-search/src/mappers/RustMapper.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
import { ModelMapper } from "../types";
|
||||
|
||||
export interface IRustApi {}
|
||||
export interface IRustAst {}
|
||||
|
||||
export const RustMapper: ModelMapper<IRustAst, IRustApi> = (i) => ({});
|
||||
57
packages/tauri-search/src/mappers/TypescriptMapper.ts
Normal file
57
packages/tauri-search/src/mappers/TypescriptMapper.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import { IApiModel } from "~/models/ApiModel";
|
||||
import { ModelMapper, TypescriptSymbol } from "~/types";
|
||||
|
||||
function symbolToUrl(module: string, kind: TypescriptKind, symbol: string) {
|
||||
switch (kind) {
|
||||
case TypescriptKind.Class:
|
||||
return `${TAURI_JS_DOCS_URL}/classes/${module}.${symbol}`;
|
||||
default:
|
||||
return `${TAURI_JS_DOCS_URL}/modules/${module}#${symbol}`;
|
||||
}
|
||||
}
|
||||
|
||||
function symbolToDeclaration(i: TypescriptSymbol) {
|
||||
switch (i.kind) {
|
||||
case TypescriptKind.Reference:
|
||||
return `type ${i.name} = {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
|
||||
case TypescriptKind.Enumeration:
|
||||
return `enum ${i.name} {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
|
||||
case TypescriptKind.Class:
|
||||
return `Class ${i.name} {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
|
||||
case TypescriptKind.Interface:
|
||||
return `interface ${i.name} {\n\t${i.children
|
||||
?.map((s) => s.name)
|
||||
.join(",\n\t")}\n}`;
|
||||
case TypescriptKind.Namespace:
|
||||
return `Module ${i.name}`;
|
||||
case TypescriptKind.Function:
|
||||
// TODO: see if we can get this filled in
|
||||
const returnType = "";
|
||||
const parameters = i.signatures?.map((s) => `${s.name}: ${s.type.name}`).join(", ");
|
||||
return `function ${i.name}(${parameters})${returnType} { ... }`;
|
||||
|
||||
default:
|
||||
return `${i.kind} ${i.name}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Typescript symbol definition to an API document modeled as `IApiModel`
|
||||
*/
|
||||
export const TypescriptMapper: ModelMapper<TypescriptSymbol, IApiModel> = (i) => ({
|
||||
id: `ts_${i.module}_${i.kind}_${i.name}`,
|
||||
name: i.name,
|
||||
kind: i.kind,
|
||||
module: i.module,
|
||||
language: "typescript",
|
||||
type: i.type,
|
||||
|
||||
commentTags: i?.commentTags,
|
||||
comment: i.comment,
|
||||
|
||||
// Type Specific
|
||||
url: symbolToUrl(i.module, i.kind, i.name),
|
||||
declaration: symbolToDeclaration(i),
|
||||
});
|
||||
@@ -1,42 +0,0 @@
|
||||
import { IRepoModel } from "~/models/RepoModel";
|
||||
import { url } from "~/types/aliases";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
import { GithubRepoResp } from "~/utils/github/getRepo";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const githubMapper = createMapper<GithubRepoResp["data"], IRepoModel>(
|
||||
"githubMapper",
|
||||
(i) => ({
|
||||
id: `github_${i.full_name.replace("/", "_")}`,
|
||||
name: i.name,
|
||||
description: i.description,
|
||||
kind: i.name.includes("plugin")
|
||||
? "plugin"
|
||||
: i.language?.toLowerCase().includes("rust")
|
||||
? "code"
|
||||
: "unknown",
|
||||
|
||||
stars: i.stargazers_count,
|
||||
watchers: i.watchers_count,
|
||||
subscribers: i.subscribers_count,
|
||||
openIssues: i.open_issues_count,
|
||||
forks: i.forks_count,
|
||||
|
||||
defaultBranch: i.default_branch,
|
||||
language: i.language,
|
||||
topics: i.topics,
|
||||
isTemplate: i.is_template,
|
||||
|
||||
lastUpdated: i.updated_at,
|
||||
createdAt: i.created_at,
|
||||
license: i.license?.name,
|
||||
|
||||
text: "",
|
||||
|
||||
url: i.html_url as url,
|
||||
})
|
||||
);
|
||||
|
||||
const g = githubMapper.map();
|
||||
@@ -1,6 +1,3 @@
|
||||
export * from "./tsClass";
|
||||
export * from "./tsFunction";
|
||||
export * from "./tsModule";
|
||||
export * from "./tsTypeAlias";
|
||||
export * from "./tsInterface";
|
||||
export * from "./tsReference";
|
||||
export * from "./TypescriptMapper";
|
||||
export * from "./GithubMapper";
|
||||
export * from "./ProseMapper";
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstClass } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsClass = createMapper<TsAstClass, ApiModel>("tsClass", (i) => ({
|
||||
id: `module_${i.module}_class_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags
|
||||
? i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
)
|
||||
: undefined,
|
||||
parameters: i.properties.map((s) => ({
|
||||
name: s.name,
|
||||
kind: s.kind,
|
||||
type: s.type.name,
|
||||
comment: s.comment?.text || s.comment?.shortText,
|
||||
})),
|
||||
declaration: `Class ${i.name} {\n\t${i.properties
|
||||
?.map((s) => `s.name`)
|
||||
.join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
|
||||
}));
|
||||
@@ -1,35 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstEnumeration } from "~/types";
|
||||
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsEnumeration = createMapper<TsAstEnumeration, ApiModel>(
|
||||
"tsEnumeration",
|
||||
(i) => ({
|
||||
id: `module_${i.module}_enumeration_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
parameters: i.properties.map((s) => ({
|
||||
name: s.name,
|
||||
kind: s.kind,
|
||||
type: s.type.name,
|
||||
comment: s.comment?.text || s.comment?.shortText,
|
||||
})),
|
||||
declaration: `enum ${i.name} {\n\t${i.properties
|
||||
.map((s) => `s.name`)
|
||||
.join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
|
||||
})
|
||||
);
|
||||
@@ -1,29 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstFunction } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsFunction = createMapper<TsAstFunction, ApiModel>("tsFunction", (i) => ({
|
||||
id: `module_${i.module}_function_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
parameters: i.signature.map((s) => ({
|
||||
name: s.name,
|
||||
kind: s.kind,
|
||||
type: s.type.name,
|
||||
comment: s.comment?.text || s.comment?.shortText,
|
||||
})),
|
||||
declaration: `function ${i.name}(${i.signature.map((s) => `${s.name}`).join(", ")})`,
|
||||
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
|
||||
}));
|
||||
@@ -1,31 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstInterface } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsInterface = createMapper<TsAstInterface, ApiModel>("tsInterface", (i) => ({
|
||||
id: `module_${i.module}_interface_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
parameters: i.properties.map((s) => ({
|
||||
name: s.name,
|
||||
kind: s.kind,
|
||||
type: s.type.name,
|
||||
comment: s.comment?.text || s.comment?.shortText,
|
||||
})),
|
||||
declaration: `interface ${i.name} {\n\t${i.properties
|
||||
.map((s) => `s.name`)
|
||||
.join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
|
||||
}));
|
||||
@@ -1,24 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstModule } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsModule = createMapper<TsAstModule, ApiModel>("tsModule", (i) => {
|
||||
return {
|
||||
id: `module_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
module: i.name,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
comments: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
url: `${TAURI_JS_DOCS_URL}/modules/${i.name}`,
|
||||
};
|
||||
});
|
||||
@@ -1,24 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstReference } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Type Alias** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsReference = createMapper<TsAstReference, ApiModel>("tsReference", (i) => ({
|
||||
id: `module_${i.module}_reference_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
|
||||
declaration: `Type ${i.name} = {\n\t${i.children.map((s) => s.name).join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
|
||||
}));
|
||||
@@ -1,26 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstTypeAlias } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Type Alias** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsTypeAlias = createMapper<TsAstTypeAlias, ApiModel>("tsTypeAlias", (i) => ({
|
||||
id: `module_${i.module}_type-alias_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
|
||||
// declaration: `Type ${i.name} = {\n\t${i.properties
|
||||
// .map((s) => `s.name`)
|
||||
// .join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
|
||||
}));
|
||||
@@ -1,31 +0,0 @@
|
||||
import { TAURI_JS_DOCS_URL } from "~/constants";
|
||||
import { ApiModel } from "~/models/api";
|
||||
import { TsAstVariable } from "~/types";
|
||||
import { createMapper } from "~/utils/createMapper";
|
||||
|
||||
/**
|
||||
* Maps a **Typescript Module** to the Meilisearch documents for that module.
|
||||
*/
|
||||
export const tsVariable = createMapper<TsAstVariable, ApiModel>("tsVariable", (i) => ({
|
||||
id: `module_${i.module}_variable_${i.name}`,
|
||||
language: "typescript",
|
||||
kind: i.kind,
|
||||
name: i.name,
|
||||
fileName: i.fileName,
|
||||
module: i.module,
|
||||
comment: i.comment?.text || i.comment?.shortText,
|
||||
tags: i.comment?.tags?.map(
|
||||
(t) =>
|
||||
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
|
||||
),
|
||||
// parameters: i.properties.map((s) => ({
|
||||
// name: s.name,
|
||||
// kind: s.kind,
|
||||
// type: s.type.name,
|
||||
// comment: s.comment?.text || s.comment?.shortText,
|
||||
// })),
|
||||
// declaration: `interface ${i.name} {\n\t${i.properties
|
||||
// .map((s) => `s.name`)
|
||||
// .join(",\n\t")}\n}`,
|
||||
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
|
||||
}));
|
||||
@@ -1,10 +1,14 @@
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import { en } from "~/stop-words";
|
||||
import { createModel } from "~/utils/createModel";
|
||||
import { TsComment } from "..";
|
||||
|
||||
export interface IApiModel {
|
||||
id: string;
|
||||
language: "rust" | "typescript";
|
||||
/** the name of the Symbol */
|
||||
name: string;
|
||||
|
||||
/**
|
||||
* The symbol's type (aka, Interface, Function, etc.)
|
||||
*
|
||||
@@ -25,8 +29,11 @@ export interface IApiModel {
|
||||
tags?: string[];
|
||||
|
||||
comment?: string;
|
||||
commentTags?: TsComment["tags"];
|
||||
|
||||
parameters?: { name: string; type?: string; comment?: string; kind: TypescriptKind }[];
|
||||
|
||||
url: string;
|
||||
}
|
||||
|
||||
export const ApiModel = createModel<IApiModel>("api", (c) =>
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
import { createModel } from "~/utils/createModel";
|
||||
|
||||
export interface ProseModel {
|
||||
export interface IProseModel {
|
||||
id: string;
|
||||
/** comes from frontmatter or the H1 tag; ideally frontmatter */
|
||||
title: string;
|
||||
/** taken from frontmatter, it allows authors to bring in words which relate to the content */
|
||||
tags?: string[];
|
||||
/** the broad area in the documentation this doc sits */
|
||||
category?: string;
|
||||
/** the sections of a document; represented as an H2 */
|
||||
sections?: string[];
|
||||
/** the sub-sections of a document; represented as an H3 */
|
||||
subSections?: string[];
|
||||
/** the programming languages which have code examples in this document */
|
||||
code?: string[];
|
||||
|
||||
kind: "core" | "plugin" | "documentation" | "other";
|
||||
stars: number;
|
||||
latestVersion: string;
|
||||
description: string;
|
||||
body: string;
|
||||
text: string;
|
||||
url: `https://${string}`;
|
||||
}
|
||||
|
||||
const model = createModel<ProseModel>("prose");
|
||||
|
||||
export default model;
|
||||
export const ProseModel = createModel<IProseModel>("prose");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { githubMapper } from "~/mappers/githubMapper";
|
||||
import { GithubMapper } from "~/mappers/GithubMapper";
|
||||
import { RepoModel } from "~/models/RepoModel";
|
||||
import { MsAddOrReplace } from "~/types";
|
||||
import { getRepo } from "~/utils/github/getRepo";
|
||||
@@ -47,7 +47,7 @@ export async function githubPipeline() {
|
||||
const waitFor: Promise<MsAddOrReplace>[] = [];
|
||||
for (const repo of REPOS) {
|
||||
const resp = await getRepo(repo);
|
||||
waitFor.push(model.query.addOrReplaceDoc(githubMapper.map(resp)));
|
||||
waitFor.push(model.query.addOrReplaceDoc(GithubMapper(resp)));
|
||||
}
|
||||
|
||||
await Promise.all(waitFor);
|
||||
|
||||
@@ -1,16 +1,7 @@
|
||||
import { ApiModel } from "~/models";
|
||||
import {
|
||||
tsClass,
|
||||
tsFunction,
|
||||
tsModule,
|
||||
tsTypeAlias,
|
||||
tsReference,
|
||||
tsInterface,
|
||||
} from "~/mappers";
|
||||
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
|
||||
import { MsAddOrReplace } from "~/types";
|
||||
import { tsVariable } from "~/mappers/tsVariable";
|
||||
import { tsEnumeration } from "~/mappers/tsEnumeration";
|
||||
// import { ApiModel } from "~/models";
|
||||
// import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
|
||||
// import { TypescriptMapper } from "~/mappers/TypescriptMapper";
|
||||
// import { MsAddOrReplace } from "~/types";
|
||||
|
||||
/**
|
||||
* Will iterate over each Typescript _module_ and all of the
|
||||
@@ -19,47 +10,42 @@ import { tsEnumeration } from "~/mappers/tsEnumeration";
|
||||
*/
|
||||
export async function typescriptPipeline() {
|
||||
// get AST from output of TSDoc's JSON option
|
||||
const ast = await parseTypescriptAst();
|
||||
const model = await ApiModel;
|
||||
|
||||
const waitFor: Promise<MsAddOrReplace>[] = [];
|
||||
|
||||
for (const mod of ast.modules) {
|
||||
// map AST model to the Document Model and add/update in MeiliSearch
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsModule.map(mod)));
|
||||
// functions
|
||||
for (const fn of mod.functions) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsFunction.map(fn)));
|
||||
}
|
||||
// classes
|
||||
for (const c of mod.classes) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsClass.map(c)));
|
||||
}
|
||||
// interfaces
|
||||
for (const i of mod.interfaces) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsInterface.map(i)));
|
||||
}
|
||||
|
||||
// variables
|
||||
for (const v of mod.variables) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsVariable.map(v)));
|
||||
}
|
||||
|
||||
// enumerations
|
||||
for (const e of mod.enums) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsEnumeration.map(e)));
|
||||
}
|
||||
|
||||
// type-aliases
|
||||
for (const ta of mod.typeAliases) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsTypeAlias.map(ta)));
|
||||
}
|
||||
// references
|
||||
for (const ref of mod.references) {
|
||||
waitFor.push(model.query.addOrReplaceDoc(tsReference.map(ref)));
|
||||
}
|
||||
}
|
||||
|
||||
const results = await Promise.all(waitFor);
|
||||
console.log(results.map((i) => `${i?.indexUid}: ${i?.status}`));
|
||||
// const ast = await parseTypescriptAst();
|
||||
// const model = ApiModel;
|
||||
// const waitFor: Promise<MsAddOrReplace>[] = [];
|
||||
// for (const sym of ast.symbols) {
|
||||
// waitFor.push(TypescriptMapper())
|
||||
// // map AST model to the Document Model and add/update in MeiliSearch
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsModule.map(mod)));
|
||||
// // functions
|
||||
// for (const fn of mod.functions) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsFunction.map(fn)));
|
||||
// }
|
||||
// // classes
|
||||
// for (const c of mod.classes) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsClass.map(c)));
|
||||
// }
|
||||
// // interfaces
|
||||
// for (const i of mod.interfaces) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsInterface.map(i)));
|
||||
// }
|
||||
// // variables
|
||||
// for (const v of mod.variables) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsVariable.map(v)));
|
||||
// }
|
||||
// // enumerations
|
||||
// for (const e of mod.enums) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsEnumeration.map(e)));
|
||||
// }
|
||||
// // type-aliases
|
||||
// for (const ta of mod.typeAliases) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsTypeAlias.map(ta)));
|
||||
// }
|
||||
// // references
|
||||
// for (const ref of mod.references) {
|
||||
// waitFor.push(model.query.addOrReplaceDoc(tsReference.map(ref)));
|
||||
// }
|
||||
// }
|
||||
// const results = await Promise.all(waitFor);
|
||||
// console.log(results.map((i) => `${i?.indexUid}: ${i?.status}`));
|
||||
}
|
||||
|
||||
@@ -1,15 +1,25 @@
|
||||
export type MarkdownAst = {
|
||||
// File info
|
||||
|
||||
filename: string;
|
||||
path: string;
|
||||
lastUpdated: number;
|
||||
createdAt: number;
|
||||
|
||||
title?: string;
|
||||
tags?: string[];
|
||||
|
||||
h1: string;
|
||||
h2: string[];
|
||||
h3: string[];
|
||||
filepath: string;
|
||||
/** the full text of the markdown content */
|
||||
text: string;
|
||||
/** a hash indicating the current state of the document */
|
||||
hash: number;
|
||||
/** a key-value dictionary of frontmatter variables */
|
||||
frontmatter: Record<string, any>;
|
||||
h1: { content: string; type: string }[];
|
||||
h2: { content: string; type: string }[];
|
||||
h3: { content: string; type: string }[];
|
||||
/** boolean flag indicating whether there was a code block */
|
||||
hasCodeBlock: boolean;
|
||||
/** if there were code blocks, this will list the languages found */
|
||||
programmingLanguages: string[];
|
||||
/** other symbols found in the markdown that weren't specifically expressed */
|
||||
otherSymbols: string[];
|
||||
};
|
||||
|
||||
export interface ITauriFrontmatter {
|
||||
title?: string;
|
||||
category?: string;
|
||||
tags?: string[];
|
||||
}
|
||||
|
||||
@@ -120,8 +120,9 @@ export type TypescriptSymbol = {
|
||||
name: string;
|
||||
module: string;
|
||||
type: TsType;
|
||||
fileName?: string;
|
||||
comment?: TsComment;
|
||||
fileName: string;
|
||||
comment?: string;
|
||||
commentTags?: TsComment["tags"];
|
||||
signatures?: { name: string; kind: string; comment: TsComment; type: TsType }[];
|
||||
children?: TypescriptBlock[];
|
||||
};
|
||||
|
||||
@@ -45,7 +45,7 @@ export function MeiliSearchApi<TDoc extends {}>(
|
||||
const get = <T>(url: string, options: AxiosRequestConfig = {}) => {
|
||||
return call("get", `${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`, options);
|
||||
};
|
||||
const put = <T, D extends any>(
|
||||
const put = <T>(
|
||||
url: string,
|
||||
data?: AxiosRequestConfig["data"],
|
||||
options: AxiosRequestConfig = {}
|
||||
@@ -79,7 +79,7 @@ export function MeiliSearchApi<TDoc extends {}>(
|
||||
getDocument: (docId: string) => get<TDoc>(`indexes/${idx}/documents/${docId}`),
|
||||
deleteDocument: (docId: string) =>
|
||||
del<MsTaskStatus>(`indexes/${idx}/documents/${docId}`),
|
||||
getDocuments: (paging: PagingOptions = {}) => get<TDoc[]>(`indexes/${idx}/documents`),
|
||||
getDocuments: (o: AxiosRequestConfig = {}) => get<TDoc[]>(`indexes/${idx}/documents`, o),
|
||||
deleteAllDocuments: del<MsTaskStatus>(`indexes/${idx}/documents`),
|
||||
addOrReplaceDocuments: (doc: TDoc, o: ApiOptions = {}) =>
|
||||
post<MsAddOrReplace>(`indexes/${idx}/documents`, JSON.stringify(doc), o),
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
import { ModelMapper } from "~/types";
|
||||
|
||||
/**
|
||||
* Provides a simple API surface for mapping between two types
|
||||
*/
|
||||
export type Mapper<TInput extends {}, TOutput extends {}> = {
|
||||
name: Readonly<string>;
|
||||
props: Readonly<ModelMapper<TInput, TOutput>>;
|
||||
map: <I extends TInput | TInput[]>(
|
||||
input: I
|
||||
) => I extends TInput[] ? TOutput[] : TOutput;
|
||||
};
|
||||
|
||||
/**
|
||||
* Facilitates the mapping from one data type to another.
|
||||
*
|
||||
* ```ts
|
||||
* export default createMapper<Input, Output>("my-mapper", i => {
|
||||
* foo: i.bar,
|
||||
* static: 42
|
||||
* })
|
||||
* ```
|
||||
*/
|
||||
export const createMapper = <TInput extends {}, TOutput extends {}>(
|
||||
name: string,
|
||||
props: ModelMapper<TInput, TOutput>
|
||||
): Mapper<TInput, TOutput> => {
|
||||
return {
|
||||
name,
|
||||
props,
|
||||
map(input: TInput | TInput[]) {
|
||||
return (
|
||||
Array.isArray(input)
|
||||
? (input.map((i) => props(i)) as TOutput[])
|
||||
: (props(input) as TOutput)
|
||||
) as typeof input extends TInput[] ? TOutput[] : TOutput;
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Endpoints } from "@octokit/types";
|
||||
import fetch from "node-fetch";
|
||||
import axios from "axios";
|
||||
import { GITHUB_API_BASE } from "~/constants";
|
||||
|
||||
export type OrgRepoLisReq = Endpoints["GET /orgs/{org}/repos"]["request"];
|
||||
@@ -7,16 +7,9 @@ export type OrgRepoLisResp = Endpoints["GET /orgs/{org}/repos"]["response"];
|
||||
|
||||
export async function getOrgRepoList(org: string = "tauri-apps") {
|
||||
const url = `${GITHUB_API_BASE}/orgs/${org}/repos?type=public&sort=updated&direction=desc&per_page=30&page=1`;
|
||||
const res = await fetch(url);
|
||||
|
||||
if (res.ok) {
|
||||
const repos = (await res.json()) as OrgRepoLisResp;
|
||||
console.log(
|
||||
repos
|
||||
// repos.map(
|
||||
// (i) =>
|
||||
// `${i.name} [${i.license?.name}] - stars ${i.stargazers_count}, watchers ${i.watchers_count}`
|
||||
// )
|
||||
);
|
||||
}
|
||||
const repos = axios.get<OrgRepoLisResp>(url);
|
||||
|
||||
return repos;
|
||||
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import c, { Cheerio, Element } from "cheerio";
|
||||
import c from "cheerio";
|
||||
import fetch from "node-fetch";
|
||||
|
||||
export async function scrapeRustDocs() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { markdownParser } from "../../src/ast/markdownParser";
|
||||
import { parseMarkdown } from "../../src/ast/parseMarkdown";
|
||||
|
||||
describe("markdownParser()", () => {
|
||||
describe("Fixture tests", async () => {
|
||||
@@ -9,8 +9,17 @@ describe("markdownParser()", () => {
|
||||
"guides/cli.md", //
|
||||
].map((i) => `test/fixtures/prose/${i}`);
|
||||
|
||||
const fileMeta = await markdownParser(files);
|
||||
const expectations = {
|
||||
const fileMeta = await parseMarkdown(files);
|
||||
const expectations: Record<
|
||||
string,
|
||||
{
|
||||
h1?: string[];
|
||||
h2: string[];
|
||||
h3: string[];
|
||||
programmingLanguages: string[];
|
||||
frontmatter: Record<string, any>;
|
||||
}
|
||||
> = {
|
||||
"cli.md": {
|
||||
h1: [],
|
||||
h2: [
|
||||
@@ -94,5 +103,9 @@ describe("markdownParser()", () => {
|
||||
}
|
||||
});
|
||||
}
|
||||
it("json", async () => {
|
||||
const meta = (await parseMarkdown(["test/fixtures/guides/updater.md"]))[0];
|
||||
console.log(meta.otherSymbols);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, beforeAll } from "vitest";
|
||||
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
|
||||
import { TypescriptKind } from "~/enums";
|
||||
import type { Expect, Equal } from "@type-challenges/utils";
|
||||
import { TsDocProject, TypescriptSymbol } from "~/types";
|
||||
|
||||
describe("typescriptParser() - AST to List", () => {
|
||||
let prj: TsDocProject;
|
||||
|
||||
describe.only("typescriptParser() - AST to List", () => {
|
||||
beforeAll(async () => {
|
||||
prj = await parseTypescriptAst();
|
||||
});
|
||||
// Initiation Tests
|
||||
|
||||
it("calling parser without parameters defaults to fixture data", async () => {
|
||||
@@ -39,9 +46,82 @@ describe("typescriptParser() - AST to List", () => {
|
||||
|
||||
// HL Structural Tests
|
||||
|
||||
it.todo("parsed data comes back as a list", () => {});
|
||||
it.todo("all items in the list have a tag 'module' to indicate ownership", () => {});
|
||||
it.todo("type information for each item is correctly set as 'ITypescriptSymbol'");
|
||||
it("all items in the list have a 'module' property to indicate ownership", async () => {
|
||||
expect(prj.symbols.every((i) => i.module && i.module.length > 0));
|
||||
});
|
||||
it("type information for each item is correctly set as 'ITypescriptSymbol'", async () => {
|
||||
type Received = typeof prj;
|
||||
|
||||
type cases = [
|
||||
Expect<Equal<Received, TsDocProject>>,
|
||||
Expect<Equal<Received["symbols"], TypescriptSymbol[]>>
|
||||
];
|
||||
const cases: cases = [true, true];
|
||||
expect(cases).toBeTruthy();
|
||||
});
|
||||
|
||||
// Specific "Depth Charge" Tests (using fixtures)
|
||||
// 1. symbol existance tests
|
||||
const modules = [
|
||||
"app",
|
||||
"cli",
|
||||
"clipboard",
|
||||
"dialog",
|
||||
"event",
|
||||
"fs",
|
||||
"globalShortcut",
|
||||
"http",
|
||||
"notification",
|
||||
"os",
|
||||
"path",
|
||||
"process",
|
||||
"shell",
|
||||
"tauri",
|
||||
"updater",
|
||||
"window",
|
||||
];
|
||||
|
||||
it("found at least as many top level modules as was statically known in test", () => {
|
||||
const foundModules = Array.from(new Set(prj.symbols.map((i) => i.module)));
|
||||
|
||||
expect(
|
||||
foundModules.length,
|
||||
`the modules found were: ${foundModules.join(", ")}`
|
||||
).toBeGreaterThanOrEqual(modules.length);
|
||||
});
|
||||
|
||||
it("all top level modules found", async () => {
|
||||
const foundModules = Array.from(new Set(prj.symbols.map((i) => i.module)));
|
||||
|
||||
for (const f of modules) {
|
||||
expect(foundModules.includes(f), `module "${f}" was not found!`).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
const expectedSymbols = [
|
||||
{
|
||||
m: "notification",
|
||||
s: ["isPermissionGranted", "requestPermission", "sendNotification", "Permission"],
|
||||
},
|
||||
{
|
||||
m: "http",
|
||||
s: ["FetchOptions", "HttpVerb", "Part", "RequestOptions", "fetch", "getClient"],
|
||||
},
|
||||
{
|
||||
m: "clipboard",
|
||||
s: ["readText", "writeText"],
|
||||
},
|
||||
];
|
||||
|
||||
for (const s of expectedSymbols) {
|
||||
it(`Module "${s.m}" has all expected symbols: ${s.s.join(", ")}`, () => {
|
||||
const found = prj.symbols.filter((i) => i.module === s.m).map((i) => i.name);
|
||||
for (const lookFor of s.s) {
|
||||
expect(
|
||||
found.includes(lookFor),
|
||||
`expected module ${s.m} to have the symbol "${s.s}"`
|
||||
).toBeTruthy();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
import { readFileSync } from "fs";
|
||||
import matter from "gray-matter";
|
||||
import smd from "simple-markdown";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
export function isHeading(something: string): something is "h1" | "h2" | "h3" {
|
||||
return ["h1", "h2", "h3"].includes(something);
|
||||
}
|
||||
|
||||
const contributorGuide = () =>
|
||||
readFileSync("test/fixtures/prose/guides/contributor-guide.md", "utf-8");
|
||||
|
||||
describe("markdown parsing tools", () => {
|
||||
it("frontmatter extracted by greymatter", () => {
|
||||
const output = matter(contributorGuide());
|
||||
expect(Object.keys(output.data)).toContain("title");
|
||||
});
|
||||
|
||||
it("simple-markdown extracts heading tags", () => {
|
||||
const ast = smd.defaultBlockParse(contributorGuide());
|
||||
const headings = {
|
||||
h1: [] as any[],
|
||||
h2: [] as any[],
|
||||
h3: [] as any[],
|
||||
};
|
||||
const otherSymbols = new Set<string>();
|
||||
let hasCodeBlock = false;
|
||||
const programmingLanguages = new Set<string>();
|
||||
|
||||
const extract = (nodeArray: smd.SingleASTNode[]) => {
|
||||
if (!Array.isArray(nodeArray)) {
|
||||
console.log("not an array", nodeArray);
|
||||
return;
|
||||
}
|
||||
for (const node of nodeArray) {
|
||||
switch (node.type) {
|
||||
case "heading":
|
||||
const tag = `h${Number(node.level)}`;
|
||||
if (isHeading(tag)) {
|
||||
headings[tag].push(node.content[0]);
|
||||
} else {
|
||||
otherSymbols.add(tag);
|
||||
}
|
||||
break;
|
||||
case "codeBlock":
|
||||
hasCodeBlock = true;
|
||||
programmingLanguages.add(node.lang);
|
||||
break;
|
||||
|
||||
default:
|
||||
otherSymbols.add(node.type);
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
for (const child of node.content) {
|
||||
if (Array.isArray(child)) {
|
||||
extract(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
extract(ast);
|
||||
|
||||
console.log("h1", headings.h1);
|
||||
console.log("h2", headings.h2);
|
||||
console.log("h3", headings.h3);
|
||||
console.log(otherSymbols);
|
||||
console.log(hasCodeBlock, programmingLanguages);
|
||||
});
|
||||
});
|
||||
6
pnpm-lock.yaml
generated
6
pnpm-lock.yaml
generated
@@ -124,6 +124,7 @@ importers:
|
||||
specifiers:
|
||||
'@jest/types': ^27.4.2
|
||||
'@octokit/types': ^6.34.0
|
||||
'@type-challenges/utils': ^0.1.1
|
||||
'@types/jest': ^27.4.0
|
||||
'@types/markdown-it': ^12.2.3
|
||||
'@types/node': ^14.18.9
|
||||
@@ -171,6 +172,7 @@ importers:
|
||||
devDependencies:
|
||||
'@jest/types': 27.4.2
|
||||
'@octokit/types': 6.34.0
|
||||
'@type-challenges/utils': 0.1.1
|
||||
'@types/jest': 27.4.0
|
||||
'@types/markdown-it': 12.2.3
|
||||
'@types/node': 14.18.9
|
||||
@@ -2033,6 +2035,10 @@ packages:
|
||||
resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==}
|
||||
dev: true
|
||||
|
||||
/@type-challenges/utils/0.1.1:
|
||||
resolution: {integrity: sha512-A7ljYfBM+FLw+NDyuYvGBJiCEV9c0lPWEAdzfOAkb3JFqfLl0Iv/WhWMMARHiRKlmmiD1g8gz/507yVvHdQUYA==}
|
||||
dev: true
|
||||
|
||||
/@types/chai-subset/1.3.3:
|
||||
resolution: {integrity: sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==}
|
||||
dependencies:
|
||||
|
||||
Reference in New Issue
Block a user