chore: reduced number of mappers needed and removed the createMapper abstraction

This commit is contained in:
Ken Snyder
2022-01-26 16:47:59 -08:00
parent 8b753b4cc8
commit 39cf3f84e7
41 changed files with 494 additions and 646 deletions

83
.eslintrc Normal file
View File

@@ -0,0 +1,83 @@
{
"extends": ["plugin:vue/vue3-recommended", "prettier"],
"plugins": ["cypress", "@typescript-eslint"],
"parserOptions": {
"parser": "@typescript-eslint/parser"
},
"settings": {
// https://github.com/meteorlxy/eslint-plugin-prettier-vue#eslint-config
"SFCBlocks": {}
},
"rules": {
// "prettier-vue/prettier": [
// "error",
// {
// "printWidth": 120,
// "singleQuote": false,
// "semi": true
// }
// ],
"vue/no-lone-template": "off",
"vue/no-multiple-template-root": "off",
"vue/no-vmodel-argument": "off",
/**
* turn this back on once the false warning for using string literals goes away;
* for now, so long as we express an "emits" clause then TS should keep us honest
* here rather than relying on lint rules.
*/
"vue/require-explicit-emits": "off",
"no-trailing-spaces": ["warn", { "skipBlankLines": true, "ignoreComments": true }],
"no-console": "warn",
"prefer-const": "error",
"semi": "off",
"quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
"no-unused-vars": "off",
"curly": ["warn", "multi-line"],
"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
// our use of `-spec` files for testing prevents us using this
"unicorn/filename-case": "off",
// reduce has been getting a bad rap lately; its true that often
// a filter or map would be clearer and equally as effective but
// there are still some legit cases to use reduce
"unicorn/no-array-reduce": "off",
"unicorn/prevent-abbreviations": "off",
"unicorn/no-null": "off",
"no-nested-ternary": "off",
// doesn't play well with prettier
"unicorn/no-nested-ternary": "off",
// this is kind of nice sometimes
"unicorn/no-array-callback-reference": "off",
// we need exceptions to be only "warn" because
// there are valid use cases for generic variables being
// used before being defined
"no-use-before-define": ["warn"],
"@typescript-eslint/semi": ["error"],
"@typescript-eslint/no-unsafe-member-access": "off",
"@typescript-eslint/no-unsafe-call": "off",
"@typescript-eslint/no-unsafe-assignment": "off",
"@typescript-eslint/member-delimiter-style": [
"error",
{
"multiline": {
"delimiter": "semi",
"requireLast": true
},
"singleline": {
"delimiter": "semi",
"requireLast": false
}
}
],
// "cases" allows for graceful use of that variable
// name in Typescript test cases
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "cases|^_",
"argsIgnorePattern": "^_"
}
]
}
}

View File

@@ -2,17 +2,17 @@ services:
# We are not using the scraper for anything currently but keeping it here
# for now in case we decide to
# scraper:
# image: getmeili/docs-scraper:latest
# container_name: scraper
# command: pipenv run ./docs_scraper config.json -d
# depends_on:
# - search
# environment:
# - MEILISEARCH_HOST_URL=localhost:7700
# - MEILISEARCH_API_KEY=""
# volumes:
# - ./scraper:/data.ms
scraper:
image: getmeili/docs-scraper:latest
container_name: scraper
command: pipenv run ./docs_scraper config.json -d
depends_on:
- search
environment:
- MEILISEARCH_HOST_URL=localhost:7700
- MEILISEARCH_API_KEY=""
volumes:
- ./scraper:/data.ms
search:
image: getmeili/meilisearch:latest

View File

@@ -8,13 +8,14 @@
"start": "pnpm -r install && pnpm run start:tauri-search && pnpm run start:docs && pnpm run up",
"start:tauri-search": "pnpm -C ./packages/tauri-search run watch",
"start:docs": "pnpm -C ./packages/docs run watch",
"build:cli": "pnpm -C ./packages/tauri-search run build:cli",
"watch": "npx pnpm run -r watch",
"test": "npx pnpm run -r test",
"up": "docker compose up -d",
"down": "docker compose down",
"into:scraper": "docker exec -it scraper bash",
"into:search": "docker exec -it search bash",
"lint": "eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
"lint": "pnpm -r eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
"lint:docs": "pnpm -C docs run lint",
"ping": "npx http GET localhost:7700/health --timeout 2",
"prune": "docker system prune",

View File

@@ -1,83 +1,3 @@
{
"extends": ["plugin:vue/vue3-recommended", "prettier"],
"plugins": ["cypress", "@typescript-eslint"],
"parserOptions": {
"parser": "@typescript-eslint/parser"
},
"settings": {
// https://github.com/meteorlxy/eslint-plugin-prettier-vue#eslint-config
"SFCBlocks": {}
},
"rules": {
// "prettier-vue/prettier": [
// "error",
// {
// "printWidth": 120,
// "singleQuote": false,
// "semi": true
// }
// ],
"vue/no-lone-template": "off",
"vue/no-multiple-template-root": "off",
"vue/no-vmodel-argument": "off",
/**
* turn this back on once the false warning for using string literals goes away;
* for now, so long as we express an "emits" clause then TS should keep us honest
* here rather than relying on lint rules.
*/
"vue/require-explicit-emits": "off",
"no-trailing-spaces": ["warn", { "skipBlankLines": true, "ignoreComments": true }],
"no-console": "warn",
"prefer-const": "error",
"semi": "off",
"quotes": ["warn", "double", { "avoidEscape": true, "allowTemplateLiterals": true }],
"no-unused-vars": "off",
"curly": ["warn", "multi-line"],
"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
// our use of `-spec` files for testing prevents us using this
"unicorn/filename-case": "off",
// reduce has been getting a bad rap lately; its true that often
// a filter or map would be clearer and equally as effective but
// there are still some legit cases to use reduce
"unicorn/no-array-reduce": "off",
"unicorn/prevent-abbreviations": "off",
"unicorn/no-null": "off",
"no-nested-ternary": "off",
// doesn't play well with prettier
"unicorn/no-nested-ternary": "off",
// this is kind of nice sometimes
"unicorn/no-array-callback-reference": "off",
// we need exceptions to be only "warn" because
// there are valid use cases for generic variables being
// used before being defined
"no-use-before-define": ["warn"],
"@typescript-eslint/semi": ["error"],
"@typescript-eslint/no-unsafe-member-access": "off",
"@typescript-eslint/no-unsafe-call": "off",
"@typescript-eslint/no-unsafe-assignment": "off",
"@typescript-eslint/member-delimiter-style": [
"error",
{
"multiline": {
"delimiter": "semi",
"requireLast": true
},
"singleline": {
"delimiter": "semi",
"requireLast": false
}
}
],
// "cases" allows for graceful use of that variable
// name in Typescript test cases
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "cases|^_",
"argsIgnorePattern": "^_"
}
]
}
"extends": "../../.eslintrc"
}

View File

@@ -0,0 +1,3 @@
{
"extends": "../../.eslintrc"
}

View File

@@ -10,6 +10,7 @@
"lint": "eslint src --ext ts,js,tsx,jsx --fix --no-error-on-unmatched-pattern",
"prune": "docker system prune",
"restart": "docker compose restart",
"build:cli": "tsup src/cli/*.ts --format=esm,cjs --clean --sourcemap -d bin",
"test": "vitest --ui",
"ts-ast": "node ./bin/ts-ast.js",
"ts-ast-overview": "node ./bin/ts-ast-overview.js",
@@ -30,6 +31,7 @@
"devDependencies": {
"@jest/types": "^27.4.2",
"@octokit/types": "^6.34.0",
"@type-challenges/utils": "^0.1.1",
"@types/jest": "^27.4.0",
"@types/markdown-it": "^12.2.3",
"@types/node": "^14.18.9",

View File

@@ -1,31 +1,39 @@
/* eslint-disable no-console */
/* eslint-disable no-use-before-define */
import { readFile } from "fs/promises";
import xxhash from "xxhash-wasm";
// import xxhash from "xxhash-wasm";
import matter from "gray-matter";
import smd from "simple-markdown";
import { ITauriFrontmatter, MarkdownAst } from "~/types/markdown";
export function isHeading(something: string): something is "h1" | "h2" | "h3" {
return ["h1", "h2", "h3"].includes(something);
}
export type MarkdownAst = {
filename: string;
filepath: string;
/** the full text of the markdown content */
text: string;
/** a hash indicating the current state of the document */
hash: number;
/** a key-value dictionary of frontmatter variables */
frontmatter: Record<string, any>;
h1: { content: string; type: string }[];
h2: { content: string; type: string }[];
h3: { content: string; type: string }[];
/** boolean flag indicating whether there was a code block */
hasCodeBlock: boolean;
/** if there were code blocks, this will list the languages found */
programmingLanguages: string[];
/** other symbols found in the markdown that weren't specifically expressed */
otherSymbols: string[];
};
function validateFrontmatter(f: string, matter: Record<string, any>) {
const typedMatter = { ...matter } as ITauriFrontmatter;
if (matter?.title && typeof matter.title !== "string") {
console.error(
`The frontmatter for "title" property needs to be a string but was a "${typeof matter.title}" in file ${f}.`
);
typedMatter.title = "UNKNOWN";
}
if (matter?.tags && Array.isArray(matter.tags)) {
console.error(
`The frontmatter for "tags" property needs to be an array of strings but was detected as "${typeof matter.title}" in file ${f}.`
);
typedMatter.tags = [];
}
if (matter?.category && typeof matter.category !== "string") {
console.error(
`The frontmatter for "category" property needs to be a string but was a "${typeof matter.title}" in file ${f}.`
);
typedMatter.category = undefined;
}
return typedMatter;
}
/**
* Takes in a list of files and parses them in three ways:
@@ -40,7 +48,7 @@ export type MarkdownAst = {
* 3. finally it will also add the `filepath` and `filename` along with a content `hash`
* which can be used detect whether content has changed
*/
export async function markdownParser(files: string[]) {
export async function parseMarkdown(files: string[]) {
// const { h32 } = await xxhash();
const tokens: MarkdownAst[] = [];
@@ -58,7 +66,7 @@ export async function markdownParser(files: string[]) {
filename,
filepath,
hash,
frontmatter,
frontmatter: validateFrontmatter(f, frontmatter),
text,
h1,
h2,
@@ -68,7 +76,7 @@ export async function markdownParser(files: string[]) {
otherSymbols,
});
} catch (err) {
err.message = `Problem parsing file ${f}: ${err.message}`;
(err as Error).message = `Problem parsing file ${f}: ${(err as Error).message}`;
throw err;
}
}
@@ -96,7 +104,6 @@ function simpleParse(f: string, content: string) {
const extract = (nodeArray: smd.SingleASTNode[]) => {
if (!Array.isArray(nodeArray)) {
console.log("not an array", nodeArray);
return;
}
for (const node of nodeArray) {
@@ -109,7 +116,7 @@ function simpleParse(f: string, content: string) {
if (Array.isArray(node.content)) {
headings[tag].push(node.content[0] as { content: string; type: string });
if (node.content.length > 1) {
console.warn(
console.error(
`A heading tag in "${f}" was found which accumulated ${
node.content.length
} content elements in a single entry; only expected 1: ${node.content
@@ -118,7 +125,7 @@ function simpleParse(f: string, content: string) {
);
}
} else {
console.warn(
console.error(
`The file ${f} got a headings tag that wasn't wrapped in an array element; this wasn't expected.`
);
}

View File

@@ -10,8 +10,9 @@ function parseModule(mod: TypescriptBlock) {
name: mod.name,
module: mod.name,
type: mod.type,
fileName: mod.sources?.shift()?.fileName,
comment: mod.comment,
fileName: mod.sources?.shift()?.fileName || "UNKNOWN",
comment: mod?.comment?.text || mod?.comment?.text,
commentTags: mod?.comment?.tags,
children: [],
};
const symbols: TypescriptSymbol[] = [modDefn];
@@ -21,9 +22,10 @@ function parseModule(mod: TypescriptBlock) {
kind: i.kindString,
name: i.name,
module: mod.name,
comment: i.comment,
comment: i?.comment?.text || i?.comment?.text,
commentTags: i?.comment?.tags,
type: i.type,
fileName: i.sources?.shift()?.fileName || "",
fileName: i.sources?.shift()?.fileName || "UNKNOWN",
signatures: i.signatures?.map((s) => ({
name: s.name,
kind: s.kindString,
@@ -54,23 +56,17 @@ export async function parseTypescriptAst(
): Promise<TsDocProject> {
const content = JSON.parse(await fetchContent(source)) as TypescriptBlock;
/**
* The top level isn't probably worth putting into the index,
* The top level "project" isn't probably worth putting into the index,
* but instead we'll start at the modules level.
*/
const project: TsDocProject = {
project: content.name,
comment: content.comment,
/**
* flattened list of all TS symbols (including modules
* and the symbols owned by them)
*/
symbols: [],
};
for (const mod of content.children || []) {
if (mod.kindString === "Namespace") {
// each module will insert a row for itself
// then then one each for each symbol it owns
project.symbols.push(...parseModule(mod));
} else {
console.error(

View File

@@ -0,0 +1,20 @@
/* eslint-disable no-console */
import process from "node:process";
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
import { TypescriptMapper } from "~/mappers/TypescriptMapper";
(async () => {
const symbols = (await parseTypescriptAst()).symbols;
const find = process.argv[2];
const found = symbols.find((i) => i.name === find);
if (found) {
console.log({
symbol: find,
ast: found,
doc: TypescriptMapper(found),
});
} else {
console.error(`Didn't find the symbol ${find}!`);
}
})();

View File

@@ -1,20 +0,0 @@
#!/usr/bin/env node
// summarizes results from the `parseTypescriptAst()` function and returns to stdout
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
(async () => {
const ast = await parseTypescriptAst();
const overview = ast.modules.map((m) => ({
module: m.name,
fns: m.functions?.map((f) => f.name),
interfaces: m.interfaces?.map((f) => f.name),
typeAliases: m.typeAliases?.map((t) => t.name),
classes: m.classes?.map((c) => c.name),
variables: m.variables?.map((v) => v.name),
references: m.references?.map((v) => v.name),
other: m.other.map((o) => `${o.name} [${o.kindString}]`),
}));
console.log(JSON.stringify(overview));
})();

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env node
// summarizes results from the `parseTypescriptAst()` function and returns to stdout
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
(async () => {
const ast = await parseTypescriptAst();

View File

@@ -0,0 +1,4 @@
import { ModelMapper } from "../types";
import { IApiModel, IRepoModel } from "..";
export const ConsolidatedMapper: ModelMapper<IApiModel | IRepoModel, any> = (i) => ({});

View File

@@ -0,0 +1,37 @@
import { IRepoModel } from "~/models/RepoModel";
import { url } from "~/types/aliases";
import { GithubRepoResp } from "~/utils/github/getRepo";
import { ModelMapper } from "..";
/**
* Maps Github Repo's API response to the appropriate document response for a Repo
*/
export const GithubMapper: ModelMapper<GithubRepoResp["data"], IRepoModel> = (i) => ({
id: `github_${i.full_name.replace("/", "_")}`,
name: i.name,
description: i.description,
kind: i.name.includes("plugin")
? "plugin"
: i.language?.toLowerCase().includes("rust")
? "code"
: "unknown",
stars: i.stargazers_count,
watchers: i.watchers_count,
subscribers: i.subscribers_count,
openIssues: i.open_issues_count,
forks: i.forks_count,
defaultBranch: i.default_branch,
language: i.language,
topics: i.topics,
isTemplate: i.is_template,
lastUpdated: i.updated_at,
createdAt: i.created_at,
license: i.license?.name,
text: "",
url: i.html_url as url,
});

View File

@@ -0,0 +1,19 @@
import { TAURI_BASE_URL } from "~/constants";
import { IProseModel } from "~/models/ProseModel";
import { MarkdownAst } from "~/types/markdown";
import { ModelMapper } from "../types";
/**
* Map markdown AST to the appropriate document structure
*/
export const ProseMapper: ModelMapper<MarkdownAst, IProseModel> = (i) => ({
id: `prose_${i.filepath.replace("/", "_")}_${i.filename}`,
title: i.frontmatter.title || i.h1.shift() || "UNKNOWN",
tags: i.frontmatter.tags as string[],
category: i.frontmatter.section as string,
sections: i.h2.map((i) => i.content),
subSections: i.h3.map((i) => i.content),
code: i.programmingLanguages,
text: i.text,
url: `${TAURI_BASE_URL}/docs/${i.filepath}/${i.filename.replace(".md", "")}`,
});

View File

@@ -0,0 +1,6 @@
import { ModelMapper } from "../types";
export interface IRustApi {}
export interface IRustAst {}
export const RustMapper: ModelMapper<IRustAst, IRustApi> = (i) => ({});

View File

@@ -0,0 +1,57 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { TypescriptKind } from "~/enums";
import { IApiModel } from "~/models/ApiModel";
import { ModelMapper, TypescriptSymbol } from "~/types";
function symbolToUrl(module: string, kind: TypescriptKind, symbol: string) {
switch (kind) {
case TypescriptKind.Class:
return `${TAURI_JS_DOCS_URL}/classes/${module}.${symbol}`;
default:
return `${TAURI_JS_DOCS_URL}/modules/${module}#${symbol}`;
}
}
function symbolToDeclaration(i: TypescriptSymbol) {
switch (i.kind) {
case TypescriptKind.Reference:
return `type ${i.name} = {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
case TypescriptKind.Enumeration:
return `enum ${i.name} {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
case TypescriptKind.Class:
return `Class ${i.name} {\n\t${i.children?.map((s) => s.name).join(",\n\t")}\n}`;
case TypescriptKind.Interface:
return `interface ${i.name} {\n\t${i.children
?.map((s) => s.name)
.join(",\n\t")}\n}`;
case TypescriptKind.Namespace:
return `Module ${i.name}`;
case TypescriptKind.Function:
// TODO: see if we can get this filled in
const returnType = "";
const parameters = i.signatures?.map((s) => `${s.name}: ${s.type.name}`).join(", ");
return `function ${i.name}(${parameters})${returnType} { ... }`;
default:
return `${i.kind} ${i.name}`;
}
}
/**
* Maps a Typescript symbol definition to an API document modeled as `IApiModel`
*/
export const TypescriptMapper: ModelMapper<TypescriptSymbol, IApiModel> = (i) => ({
id: `ts_${i.module}_${i.kind}_${i.name}`,
name: i.name,
kind: i.kind,
module: i.module,
language: "typescript",
type: i.type,
commentTags: i?.commentTags,
comment: i.comment,
// Type Specific
url: symbolToUrl(i.module, i.kind, i.name),
declaration: symbolToDeclaration(i),
});

View File

@@ -1,42 +0,0 @@
import { IRepoModel } from "~/models/RepoModel";
import { url } from "~/types/aliases";
import { createMapper } from "~/utils/createMapper";
import { GithubRepoResp } from "~/utils/github/getRepo";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const githubMapper = createMapper<GithubRepoResp["data"], IRepoModel>(
"githubMapper",
(i) => ({
id: `github_${i.full_name.replace("/", "_")}`,
name: i.name,
description: i.description,
kind: i.name.includes("plugin")
? "plugin"
: i.language?.toLowerCase().includes("rust")
? "code"
: "unknown",
stars: i.stargazers_count,
watchers: i.watchers_count,
subscribers: i.subscribers_count,
openIssues: i.open_issues_count,
forks: i.forks_count,
defaultBranch: i.default_branch,
language: i.language,
topics: i.topics,
isTemplate: i.is_template,
lastUpdated: i.updated_at,
createdAt: i.created_at,
license: i.license?.name,
text: "",
url: i.html_url as url,
})
);
const g = githubMapper.map();

View File

@@ -1,6 +1,3 @@
export * from "./tsClass";
export * from "./tsFunction";
export * from "./tsModule";
export * from "./tsTypeAlias";
export * from "./tsInterface";
export * from "./tsReference";
export * from "./TypescriptMapper";
export * from "./GithubMapper";
export * from "./ProseMapper";

View File

@@ -1,33 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstClass } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsClass = createMapper<TsAstClass, ApiModel>("tsClass", (i) => ({
id: `module_${i.module}_class_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags
? i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
)
: undefined,
parameters: i.properties.map((s) => ({
name: s.name,
kind: s.kind,
type: s.type.name,
comment: s.comment?.text || s.comment?.shortText,
})),
declaration: `Class ${i.name} {\n\t${i.properties
?.map((s) => `s.name`)
.join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
}));

View File

@@ -1,35 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstEnumeration } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsEnumeration = createMapper<TsAstEnumeration, ApiModel>(
"tsEnumeration",
(i) => ({
id: `module_${i.module}_enumeration_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
parameters: i.properties.map((s) => ({
name: s.name,
kind: s.kind,
type: s.type.name,
comment: s.comment?.text || s.comment?.shortText,
})),
declaration: `enum ${i.name} {\n\t${i.properties
.map((s) => `s.name`)
.join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
})
);

View File

@@ -1,29 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstFunction } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsFunction = createMapper<TsAstFunction, ApiModel>("tsFunction", (i) => ({
id: `module_${i.module}_function_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
parameters: i.signature.map((s) => ({
name: s.name,
kind: s.kind,
type: s.type.name,
comment: s.comment?.text || s.comment?.shortText,
})),
declaration: `function ${i.name}(${i.signature.map((s) => `${s.name}`).join(", ")})`,
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
}));

View File

@@ -1,31 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstInterface } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsInterface = createMapper<TsAstInterface, ApiModel>("tsInterface", (i) => ({
id: `module_${i.module}_interface_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
parameters: i.properties.map((s) => ({
name: s.name,
kind: s.kind,
type: s.type.name,
comment: s.comment?.text || s.comment?.shortText,
})),
declaration: `interface ${i.name} {\n\t${i.properties
.map((s) => `s.name`)
.join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/classes/${i.module}.${i.name}`,
}));

View File

@@ -1,24 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstModule } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsModule = createMapper<TsAstModule, ApiModel>("tsModule", (i) => {
return {
id: `module_${i.name}`,
language: "typescript",
kind: i.kind,
module: i.name,
name: i.name,
fileName: i.fileName,
comments: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
url: `${TAURI_JS_DOCS_URL}/modules/${i.name}`,
};
});

View File

@@ -1,24 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstReference } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Type Alias** to the Meilisearch documents for that module.
*/
export const tsReference = createMapper<TsAstReference, ApiModel>("tsReference", (i) => ({
id: `module_${i.module}_reference_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
declaration: `Type ${i.name} = {\n\t${i.children.map((s) => s.name).join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
}));

View File

@@ -1,26 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstTypeAlias } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Type Alias** to the Meilisearch documents for that module.
*/
export const tsTypeAlias = createMapper<TsAstTypeAlias, ApiModel>("tsTypeAlias", (i) => ({
id: `module_${i.module}_type-alias_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
// declaration: `Type ${i.name} = {\n\t${i.properties
// .map((s) => `s.name`)
// .join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
}));

View File

@@ -1,31 +0,0 @@
import { TAURI_JS_DOCS_URL } from "~/constants";
import { ApiModel } from "~/models/api";
import { TsAstVariable } from "~/types";
import { createMapper } from "~/utils/createMapper";
/**
* Maps a **Typescript Module** to the Meilisearch documents for that module.
*/
export const tsVariable = createMapper<TsAstVariable, ApiModel>("tsVariable", (i) => ({
id: `module_${i.module}_variable_${i.name}`,
language: "typescript",
kind: i.kind,
name: i.name,
fileName: i.fileName,
module: i.module,
comment: i.comment?.text || i.comment?.shortText,
tags: i.comment?.tags?.map(
(t) =>
`<span class="tag">@${t.tag}:</span> <span class="tag-description">${t?.text}</span>`
),
// parameters: i.properties.map((s) => ({
// name: s.name,
// kind: s.kind,
// type: s.type.name,
// comment: s.comment?.text || s.comment?.shortText,
// })),
// declaration: `interface ${i.name} {\n\t${i.properties
// .map((s) => `s.name`)
// .join(",\n\t")}\n}`,
url: `${TAURI_JS_DOCS_URL}/modules/${i.module}#${i.name}`,
}));

View File

@@ -1,10 +1,14 @@
import { TypescriptKind } from "~/enums";
import { en } from "~/stop-words";
import { createModel } from "~/utils/createModel";
import { TsComment } from "..";
export interface IApiModel {
id: string;
language: "rust" | "typescript";
/** the name of the Symbol */
name: string;
/**
* The symbol's type (aka, Interface, Function, etc.)
*
@@ -25,8 +29,11 @@ export interface IApiModel {
tags?: string[];
comment?: string;
commentTags?: TsComment["tags"];
parameters?: { name: string; type?: string; comment?: string; kind: TypescriptKind }[];
url: string;
}
export const ApiModel = createModel<IApiModel>("api", (c) =>

View File

@@ -1,17 +1,22 @@
import { createModel } from "~/utils/createModel";
export interface ProseModel {
export interface IProseModel {
id: string;
/** comes from frontmatter or the H1 tag; ideally frontmatter */
title: string;
/** taken from frontmatter, it allows authors to bring in words which relate to the content */
tags?: string[];
/** the broad area in the documentation this doc sits */
category?: string;
/** the sections of a document; represented as an H2 */
sections?: string[];
/** the sub-sections of a document; represented as an H3 */
subSections?: string[];
/** the programming languages which have code examples in this document */
code?: string[];
kind: "core" | "plugin" | "documentation" | "other";
stars: number;
latestVersion: string;
description: string;
body: string;
text: string;
url: `https://${string}`;
}
const model = createModel<ProseModel>("prose");
export default model;
export const ProseModel = createModel<IProseModel>("prose");

View File

@@ -1,4 +1,4 @@
import { githubMapper } from "~/mappers/githubMapper";
import { GithubMapper } from "~/mappers/GithubMapper";
import { RepoModel } from "~/models/RepoModel";
import { MsAddOrReplace } from "~/types";
import { getRepo } from "~/utils/github/getRepo";
@@ -47,7 +47,7 @@ export async function githubPipeline() {
const waitFor: Promise<MsAddOrReplace>[] = [];
for (const repo of REPOS) {
const resp = await getRepo(repo);
waitFor.push(model.query.addOrReplaceDoc(githubMapper.map(resp)));
waitFor.push(model.query.addOrReplaceDoc(GithubMapper(resp)));
}
await Promise.all(waitFor);

View File

@@ -1,16 +1,7 @@
import { ApiModel } from "~/models";
import {
tsClass,
tsFunction,
tsModule,
tsTypeAlias,
tsReference,
tsInterface,
} from "~/mappers";
import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
import { MsAddOrReplace } from "~/types";
import { tsVariable } from "~/mappers/tsVariable";
import { tsEnumeration } from "~/mappers/tsEnumeration";
// import { ApiModel } from "~/models";
// import { parseTypescriptAst } from "~/utils/parseTypescriptAst";
// import { TypescriptMapper } from "~/mappers/TypescriptMapper";
// import { MsAddOrReplace } from "~/types";
/**
* Will iterate over each Typescript _module_ and all of the
@@ -19,47 +10,42 @@ import { tsEnumeration } from "~/mappers/tsEnumeration";
*/
export async function typescriptPipeline() {
// get AST from output of TSDoc's JSON option
const ast = await parseTypescriptAst();
const model = await ApiModel;
const waitFor: Promise<MsAddOrReplace>[] = [];
for (const mod of ast.modules) {
// map AST model to the Document Model and add/update in MeiliSearch
waitFor.push(model.query.addOrReplaceDoc(tsModule.map(mod)));
// functions
for (const fn of mod.functions) {
waitFor.push(model.query.addOrReplaceDoc(tsFunction.map(fn)));
}
// classes
for (const c of mod.classes) {
waitFor.push(model.query.addOrReplaceDoc(tsClass.map(c)));
}
// interfaces
for (const i of mod.interfaces) {
waitFor.push(model.query.addOrReplaceDoc(tsInterface.map(i)));
}
// variables
for (const v of mod.variables) {
waitFor.push(model.query.addOrReplaceDoc(tsVariable.map(v)));
}
// enumerations
for (const e of mod.enums) {
waitFor.push(model.query.addOrReplaceDoc(tsEnumeration.map(e)));
}
// type-aliases
for (const ta of mod.typeAliases) {
waitFor.push(model.query.addOrReplaceDoc(tsTypeAlias.map(ta)));
}
// references
for (const ref of mod.references) {
waitFor.push(model.query.addOrReplaceDoc(tsReference.map(ref)));
}
}
const results = await Promise.all(waitFor);
console.log(results.map((i) => `${i?.indexUid}: ${i?.status}`));
// const ast = await parseTypescriptAst();
// const model = ApiModel;
// const waitFor: Promise<MsAddOrReplace>[] = [];
// for (const sym of ast.symbols) {
// waitFor.push(TypescriptMapper())
// // map AST model to the Document Model and add/update in MeiliSearch
// waitFor.push(model.query.addOrReplaceDoc(tsModule.map(mod)));
// // functions
// for (const fn of mod.functions) {
// waitFor.push(model.query.addOrReplaceDoc(tsFunction.map(fn)));
// }
// // classes
// for (const c of mod.classes) {
// waitFor.push(model.query.addOrReplaceDoc(tsClass.map(c)));
// }
// // interfaces
// for (const i of mod.interfaces) {
// waitFor.push(model.query.addOrReplaceDoc(tsInterface.map(i)));
// }
// // variables
// for (const v of mod.variables) {
// waitFor.push(model.query.addOrReplaceDoc(tsVariable.map(v)));
// }
// // enumerations
// for (const e of mod.enums) {
// waitFor.push(model.query.addOrReplaceDoc(tsEnumeration.map(e)));
// }
// // type-aliases
// for (const ta of mod.typeAliases) {
// waitFor.push(model.query.addOrReplaceDoc(tsTypeAlias.map(ta)));
// }
// // references
// for (const ref of mod.references) {
// waitFor.push(model.query.addOrReplaceDoc(tsReference.map(ref)));
// }
// }
// const results = await Promise.all(waitFor);
// console.log(results.map((i) => `${i?.indexUid}: ${i?.status}`));
}

View File

@@ -1,15 +1,25 @@
export type MarkdownAst = {
// File info
filename: string;
path: string;
lastUpdated: number;
createdAt: number;
title?: string;
tags?: string[];
h1: string;
h2: string[];
h3: string[];
filepath: string;
/** the full text of the markdown content */
text: string;
/** a hash indicating the current state of the document */
hash: number;
/** a key-value dictionary of frontmatter variables */
frontmatter: Record<string, any>;
h1: { content: string; type: string }[];
h2: { content: string; type: string }[];
h3: { content: string; type: string }[];
/** boolean flag indicating whether there was a code block */
hasCodeBlock: boolean;
/** if there were code blocks, this will list the languages found */
programmingLanguages: string[];
/** other symbols found in the markdown that weren't specifically expressed */
otherSymbols: string[];
};
export interface ITauriFrontmatter {
title?: string;
category?: string;
tags?: string[];
}

View File

@@ -120,8 +120,9 @@ export type TypescriptSymbol = {
name: string;
module: string;
type: TsType;
fileName?: string;
comment?: TsComment;
fileName: string;
comment?: string;
commentTags?: TsComment["tags"];
signatures?: { name: string; kind: string; comment: TsComment; type: TsType }[];
children?: TypescriptBlock[];
};

View File

@@ -45,7 +45,7 @@ export function MeiliSearchApi<TDoc extends {}>(
const get = <T>(url: string, options: AxiosRequestConfig = {}) => {
return call("get", `${baseUrl}/${url.startsWith("/" ? url.slice(1) : url)}`, options);
};
const put = <T, D extends any>(
const put = <T>(
url: string,
data?: AxiosRequestConfig["data"],
options: AxiosRequestConfig = {}
@@ -79,7 +79,7 @@ export function MeiliSearchApi<TDoc extends {}>(
getDocument: (docId: string) => get<TDoc>(`indexes/${idx}/documents/${docId}`),
deleteDocument: (docId: string) =>
del<MsTaskStatus>(`indexes/${idx}/documents/${docId}`),
getDocuments: (paging: PagingOptions = {}) => get<TDoc[]>(`indexes/${idx}/documents`),
getDocuments: (o: AxiosRequestConfig = {}) => get<TDoc[]>(`indexes/${idx}/documents`, o),
deleteAllDocuments: del<MsTaskStatus>(`indexes/${idx}/documents`),
addOrReplaceDocuments: (doc: TDoc, o: ApiOptions = {}) =>
post<MsAddOrReplace>(`indexes/${idx}/documents`, JSON.stringify(doc), o),

View File

@@ -1,39 +0,0 @@
import { ModelMapper } from "~/types";
/**
* Provides a simple API surface for mapping between two types
*/
export type Mapper<TInput extends {}, TOutput extends {}> = {
name: Readonly<string>;
props: Readonly<ModelMapper<TInput, TOutput>>;
map: <I extends TInput | TInput[]>(
input: I
) => I extends TInput[] ? TOutput[] : TOutput;
};
/**
* Facilitates the mapping from one data type to another.
*
* ```ts
* export default createMapper<Input, Output>("my-mapper", i => {
* foo: i.bar,
* static: 42
* })
* ```
*/
export const createMapper = <TInput extends {}, TOutput extends {}>(
name: string,
props: ModelMapper<TInput, TOutput>
): Mapper<TInput, TOutput> => {
return {
name,
props,
map(input: TInput | TInput[]) {
return (
Array.isArray(input)
? (input.map((i) => props(i)) as TOutput[])
: (props(input) as TOutput)
) as typeof input extends TInput[] ? TOutput[] : TOutput;
},
};
};

View File

@@ -1,5 +1,5 @@
import { Endpoints } from "@octokit/types";
import fetch from "node-fetch";
import axios from "axios";
import { GITHUB_API_BASE } from "~/constants";
export type OrgRepoLisReq = Endpoints["GET /orgs/{org}/repos"]["request"];
@@ -7,16 +7,9 @@ export type OrgRepoLisResp = Endpoints["GET /orgs/{org}/repos"]["response"];
export async function getOrgRepoList(org: string = "tauri-apps") {
const url = `${GITHUB_API_BASE}/orgs/${org}/repos?type=public&sort=updated&direction=desc&per_page=30&page=1`;
const res = await fetch(url);
if (res.ok) {
const repos = (await res.json()) as OrgRepoLisResp;
console.log(
repos
// repos.map(
// (i) =>
// `${i.name} [${i.license?.name}] - stars ${i.stargazers_count}, watchers ${i.watchers_count}`
// )
);
}
const repos = axios.get<OrgRepoLisResp>(url);
return repos;
}

View File

@@ -1,4 +1,4 @@
import c, { Cheerio, Element } from "cheerio";
import c from "cheerio";
import fetch from "node-fetch";
export async function scrapeRustDocs() {

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { markdownParser } from "../../src/ast/markdownParser";
import { parseMarkdown } from "../../src/ast/parseMarkdown";
describe("markdownParser()", () => {
describe("Fixture tests", async () => {
@@ -9,8 +9,17 @@ describe("markdownParser()", () => {
"guides/cli.md", //
].map((i) => `test/fixtures/prose/${i}`);
const fileMeta = await markdownParser(files);
const expectations = {
const fileMeta = await parseMarkdown(files);
const expectations: Record<
string,
{
h1?: string[];
h2: string[];
h3: string[];
programmingLanguages: string[];
frontmatter: Record<string, any>;
}
> = {
"cli.md": {
h1: [],
h2: [
@@ -94,5 +103,9 @@ describe("markdownParser()", () => {
}
});
}
it("json", async () => {
const meta = (await parseMarkdown(["test/fixtures/guides/updater.md"]))[0];
console.log(meta.otherSymbols);
});
});
});

View File

@@ -1,8 +1,15 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, beforeAll } from "vitest";
import { parseTypescriptAst } from "~/ast/parseTypescriptAst";
import { TypescriptKind } from "~/enums";
import type { Expect, Equal } from "@type-challenges/utils";
import { TsDocProject, TypescriptSymbol } from "~/types";
describe("typescriptParser() - AST to List", () => {
let prj: TsDocProject;
describe.only("typescriptParser() - AST to List", () => {
beforeAll(async () => {
prj = await parseTypescriptAst();
});
// Initiation Tests
it("calling parser without parameters defaults to fixture data", async () => {
@@ -39,9 +46,82 @@ describe("typescriptParser() - AST to List", () => {
// HL Structural Tests
it.todo("parsed data comes back as a list", () => {});
it.todo("all items in the list have a tag 'module' to indicate ownership", () => {});
it.todo("type information for each item is correctly set as 'ITypescriptSymbol'");
it("all items in the list have a 'module' property to indicate ownership", async () => {
expect(prj.symbols.every((i) => i.module && i.module.length > 0));
});
it("type information for each item is correctly set as 'ITypescriptSymbol'", async () => {
type Received = typeof prj;
type cases = [
Expect<Equal<Received, TsDocProject>>,
Expect<Equal<Received["symbols"], TypescriptSymbol[]>>
];
const cases: cases = [true, true];
expect(cases).toBeTruthy();
});
// Specific "Depth Charge" Tests (using fixtures)
// 1. symbol existance tests
const modules = [
"app",
"cli",
"clipboard",
"dialog",
"event",
"fs",
"globalShortcut",
"http",
"notification",
"os",
"path",
"process",
"shell",
"tauri",
"updater",
"window",
];
it("found at least as many top level modules as was statically known in test", () => {
const foundModules = Array.from(new Set(prj.symbols.map((i) => i.module)));
expect(
foundModules.length,
`the modules found were: ${foundModules.join(", ")}`
).toBeGreaterThanOrEqual(modules.length);
});
it("all top level modules found", async () => {
const foundModules = Array.from(new Set(prj.symbols.map((i) => i.module)));
for (const f of modules) {
expect(foundModules.includes(f), `module "${f}" was not found!`).toBeTruthy();
}
});
const expectedSymbols = [
{
m: "notification",
s: ["isPermissionGranted", "requestPermission", "sendNotification", "Permission"],
},
{
m: "http",
s: ["FetchOptions", "HttpVerb", "Part", "RequestOptions", "fetch", "getClient"],
},
{
m: "clipboard",
s: ["readText", "writeText"],
},
];
for (const s of expectedSymbols) {
it(`Module "${s.m}" has all expected symbols: ${s.s.join(", ")}`, () => {
const found = prj.symbols.filter((i) => i.module === s.m).map((i) => i.name);
for (const lookFor of s.s) {
expect(
found.includes(lookFor),
`expected module ${s.m} to have the symbol "${s.s}"`
).toBeTruthy();
}
});
}
});

View File

@@ -1,71 +0,0 @@
import { readFileSync } from "fs";
import matter from "gray-matter";
import smd from "simple-markdown";
import { describe, expect, it } from "vitest";
export function isHeading(something: string): something is "h1" | "h2" | "h3" {
return ["h1", "h2", "h3"].includes(something);
}
const contributorGuide = () =>
readFileSync("test/fixtures/prose/guides/contributor-guide.md", "utf-8");
describe("markdown parsing tools", () => {
it("frontmatter extracted by greymatter", () => {
const output = matter(contributorGuide());
expect(Object.keys(output.data)).toContain("title");
});
it("simple-markdown extracts heading tags", () => {
const ast = smd.defaultBlockParse(contributorGuide());
const headings = {
h1: [] as any[],
h2: [] as any[],
h3: [] as any[],
};
const otherSymbols = new Set<string>();
let hasCodeBlock = false;
const programmingLanguages = new Set<string>();
const extract = (nodeArray: smd.SingleASTNode[]) => {
if (!Array.isArray(nodeArray)) {
console.log("not an array", nodeArray);
return;
}
for (const node of nodeArray) {
switch (node.type) {
case "heading":
const tag = `h${Number(node.level)}`;
if (isHeading(tag)) {
headings[tag].push(node.content[0]);
} else {
otherSymbols.add(tag);
}
break;
case "codeBlock":
hasCodeBlock = true;
programmingLanguages.add(node.lang);
break;
default:
otherSymbols.add(node.type);
}
if (node.content) {
for (const child of node.content) {
if (Array.isArray(child)) {
extract(child);
}
}
}
}
};
extract(ast);
console.log("h1", headings.h1);
console.log("h2", headings.h2);
console.log("h3", headings.h3);
console.log(otherSymbols);
console.log(hasCodeBlock, programmingLanguages);
});
});

6
pnpm-lock.yaml generated
View File

@@ -124,6 +124,7 @@ importers:
specifiers:
'@jest/types': ^27.4.2
'@octokit/types': ^6.34.0
'@type-challenges/utils': ^0.1.1
'@types/jest': ^27.4.0
'@types/markdown-it': ^12.2.3
'@types/node': ^14.18.9
@@ -171,6 +172,7 @@ importers:
devDependencies:
'@jest/types': 27.4.2
'@octokit/types': 6.34.0
'@type-challenges/utils': 0.1.1
'@types/jest': 27.4.0
'@types/markdown-it': 12.2.3
'@types/node': 14.18.9
@@ -2033,6 +2035,10 @@ packages:
resolution: {integrity: sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==}
dev: true
/@type-challenges/utils/0.1.1:
resolution: {integrity: sha512-A7ljYfBM+FLw+NDyuYvGBJiCEV9c0lPWEAdzfOAkb3JFqfLl0Iv/WhWMMARHiRKlmmiD1g8gz/507yVvHdQUYA==}
dev: true
/@types/chai-subset/1.3.3:
resolution: {integrity: sha512-frBecisrNGz+F4T6bcc+NLeolfiojh5FxW2klu669+8BARtyQv2C/GkNW6FUodVe4BroGMP/wER/YDGc7rEllw==}
dependencies: