import { basename, extname, join, relative } from "path/mod.ts"; import { readMarkdownDoc } from "./readDoc.ts"; import { Index } from "./client_search.ts"; export interface Doc { path: string; content: string; attributes: { title?: string; japanese_title?: string; tags?: string[]; rjcode?: string; author?: string; }; } export async function loadDocuments(path: string): Promise { const doc_json = await Deno.readTextFile(path); return JSON.parse(doc_json) as Doc[]; } export interface DocCollectorOptions { summaryOnly?: boolean; dropContent?: boolean; } export class DocCollector { private doc_map: Map; private options: DocCollectorOptions; constructor(options: DocCollectorOptions = {}) { this.doc_map = new Map(); this.options = options; } public getDocs(): Doc[] { return [...this.doc_map.values()]; } public setDoc(doc: Doc) { if (this.options.dropContent) { doc.content = ""; } this.doc_map.set(doc.path, doc); } public setDocs(docs: Doc[]) { for (const doc of docs) { this.setDoc(doc); } } public removeDoc(path: string) { this.doc_map.delete(path); } public async walkDir(path: string) { const dir = Deno.readDir(path); const fileList = []; for await (const entry of dir) { fileList.push(entry); } if (fileList.some((entry) => entry.name === "SUMMARY.md")) { const { content, metadata } = await readMarkdownDoc( join(path, "SUMMARY.md"), ); this.setDoc({ path: join(path, "SUMMARY.md"), content: content, attributes: metadata, }); } else { for (const entry of fileList) { if (entry.isDirectory) { await this.walkDir(join(path, entry.name)); } else if (entry.isFile && !this.options.summaryOnly) { const doc = await this.readDoc(join(path, entry.name)); this.setDoc(doc); } } } } public async readDoc(path: string): Promise { const ext = extname(path); if (ext === ".md") { return await this.readMarkdown(path); } else if (ext === ".html" || ext === ".htm" || ext === ".xhtml") { return await this.readHTML(path); } else if (ext === ".txt") { return await this.readText(path); } else { return { path: path, content: "", attributes: {}, }; } } public async readHTML(path: string): Promise { const content = await Deno.readTextFile(path); return { path: path, content: content, attributes: {}, }; } public async readText(path: string): Promise { const content = await Deno.readTextFile(path); return { path: path, content: content, attributes: {}, }; } public async readMarkdown(path: string): Promise { const { content, metadata } = await readMarkdownDoc(path); return { path: path, content: content, attributes: metadata, }; } async watchDir(path: string, { onRemove = (_path: string) => {}, onAdd = (_doc: Doc) => {}, onChange = (_doc: Doc) => {}, abort = undefined, }: { onRemove?: (path: string) => void | Promise; onAdd?: (doc: Doc) => void | Promise; onChange?: (doc: Doc) => void | Promise; abort?: AbortSignal; }) { const watcher = Deno.watchFs(path); if (abort) { abort.addEventListener("abort", () => { watcher.close(); }); } for await (const event of watcher) { if ( event.kind === "access" || event.kind === "other" || event.kind === "any" ) { continue; } if (event.paths.length === 0) { continue; } for (const path of event.paths) { const relpath = relative(Deno.cwd(), path); const filename = basename(relpath); if (filename === "SUMMARY.md") { if (event.kind === "remove") { this.doc_map.delete(relpath); await onRemove(relpath); } else if (event.kind === "create" || event.kind === "modify") { const { content, metadata } = await readMarkdownDoc(relpath); const doc = { path: relpath, content: content, attributes: metadata, }; this.setDoc(doc); if (event.kind === "create") { await onAdd(doc); } else if (event.kind === "modify") { await onChange(doc); } } } } } } makeIndex(options?: { onUpdate?: (() => void) | (() => Promise); abort?: AbortSignal; watch?: boolean; }) { const opt = options ?? {}; const index = Index.createIndex(this.getDocs()); if (!opt.watch) { return index; } const update = async () => { index.setDocs(this.getDocs()); if (opt.onUpdate) { await opt.onUpdate(); } }; this.watchDir(".", { async onAdd(_doc) { await update(); }, async onRemove(_path) { await update(); }, async onChange(_doc) { await update(); }, abort: opt.abort, }); return index; } }