From cade73da87bdbbc07987f56e1ca6c88f3f008161 Mon Sep 17 00:00:00 2001 From: monoid Date: Fri, 15 Jan 2021 18:43:36 +0900 Subject: [PATCH] add diff --- app.ts | 38 ++++++++- gen_conf_schema.ts | 48 ++++++++++++ loading.html | 32 -------- migrations/initial.ts | 3 +- package.json | 4 +- preload.ts | 1 + src/SettingConfig.schema.json | 66 ++++++++++++++++ src/{setting.ts => SettingConfig.ts} | 15 ++-- src/client/accessor/document.ts | 1 + src/client/page/difference.tsx | 46 ++++++++++- src/content/file.ts | 46 ++++++++--- src/content/manga.ts | 7 +- src/content/video.ts | 4 +- src/database.ts | 2 +- src/db/doc.ts | 30 +++++-- src/diff/MangaConfig.schema.json | 1 + src/diff/MangaConfig.ts | 6 ++ src/diff/content_handler.ts | 74 ++++++++++++++++++ src/diff/content_list.ts | 62 +++++++++++++++ src/diff/diff.ts | 113 +++++++++------------------ src/diff/mod.ts | 2 + src/diff/router.ts | 61 +++++++++++++++ src/diff/watcher.ts | 78 ++++++++++++++++++ src/login.ts | 2 +- src/model/doc.ts | 18 +++-- src/route/contents.ts | 6 +- src/route/context.ts | 2 - src/server.ts | 20 +++-- src/types/db.d.ts | 6 +- src/types/json.d.ts | 5 ++ test.ts | 10 +++ tsconfig.json | 1 + 32 files changed, 639 insertions(+), 171 deletions(-) create mode 100644 gen_conf_schema.ts delete mode 100644 loading.html create mode 100644 preload.ts create mode 100644 src/SettingConfig.schema.json rename src/{setting.ts => SettingConfig.ts} (85%) create mode 100644 src/diff/MangaConfig.schema.json create mode 100644 src/diff/MangaConfig.ts create mode 100644 src/diff/content_handler.ts create mode 100644 src/diff/content_list.ts create mode 100644 src/diff/mod.ts create mode 100644 src/diff/router.ts create mode 100644 src/diff/watcher.ts create mode 100644 src/types/json.d.ts create mode 100644 test.ts diff --git a/app.ts b/app.ts index b1a9d4d..a6a1a2e 100644 --- a/app.ts +++ b/app.ts @@ -1,5 +1,5 @@ import { app, BrowserWindow, session, dialog } from "electron"; -import { get_setting } from "./src/setting"; +import { get_setting } from "./src/SettingConfig"; import { create_server, start_server } from "./src/server"; import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login"; @@ -14,8 +14,8 @@ if (!setting.cli) { center: true, useContentSize: true, }); - //await window.loadURL(`data:text/html;base64,`+Buffer.from(get_loading_html()).toString('base64')); - await wnd.loadFile('../loading.html'); + await wnd.loadURL(`data:text/html;base64,`+Buffer.from(loading_html).toString('base64')); + //await wnd.loadURL('../loading.html'); await session.defaultSession.cookies.set({ url:`http://localhost:${setting.port}`, name:accessTokenName, @@ -88,3 +88,35 @@ if (!setting.cli) { start_server(server); })(); } +const loading_html = ` + + +loading + + + + + +

Loading...

+
+ +`; \ No newline at end of file diff --git a/gen_conf_schema.ts b/gen_conf_schema.ts new file mode 100644 index 0000000..5a2071d --- /dev/null +++ b/gen_conf_schema.ts @@ -0,0 +1,48 @@ +import { promises } from 'fs'; +const { readdir, writeFile } = promises; +import {createGenerator} from 'ts-json-schema-generator'; +import {dirname,join} from 'path'; + +async function genSchema(path:string,typename:string){ + const gen = createGenerator({ + path:path, + type:typename, + tsconfig:"tsconfig.json" + }); + const schema = gen.createSchema(typename); + if(schema.definitions != undefined){ + const definitions = schema.definitions; + const definition = definitions[typename]; + if(typeof definition == "object" ){ + let property = definition.properties; + if(property){ + property['$schema'] = { + type:"string" + }; + } + } + } + const text = JSON.stringify(schema); + await writeFile(join(dirname(path),`${typename}.schema.json`),text); +} +function capitalize(s:string){ + return s.charAt(0).toUpperCase() + s.slice(1); +} +async function setToALL(path:string) { + console.log(`scan ${path}`) + const direntry = await readdir(path,{withFileTypes:true}); + const works = direntry.filter(x=>x.isFile()&&x.name.endsWith("Config.ts")).map(x=>{ + const name = x.name; + const m = /(.+)\.ts/.exec(name); + if(m !== null){ + const typename = m[1]; + return genSchema(join(path,typename),capitalize(typename)); + } + }) + await Promise.all(works); + const subdir = direntry.filter(x=>x.isDirectory()).map(x=>x.name); + for(const x of subdir){ + await setToALL(join(path,x)); + } +} +setToALL("src") \ No newline at end of file diff --git a/loading.html b/loading.html deleted file mode 100644 index e0ac6a5..0000000 --- a/loading.html +++ /dev/null @@ -1,32 +0,0 @@ - - - -loading - - - - - -

Loading...

-
- - \ No newline at end of file diff --git a/migrations/initial.ts b/migrations/initial.ts index 86a16d3..36178b6 100644 --- a/migrations/initial.ts +++ b/migrations/initial.ts @@ -14,7 +14,8 @@ export async function up(knex:Knex) { b.string("filename",256).notNullable().comment("filename"); b.string("content_hash").nullable(); b.json("additional").nullable(); - b.timestamps(); + b.integer("created_at").notNullable(); + b.integer("deleted_at"); b.index("content_type","content_type_index"); }); await knex.schema.createTable("tags", (b)=>{ diff --git a/package.json b/package.json index 52e83ab..85c9989 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "files": [ "build/**/*", "node_modules/**/*", - "package.json" + "package.json", + "!node_modules/@material-ui/**/*" ], "appId": "com.prelude.ionian.app", "productName": "Ionian", @@ -82,6 +83,7 @@ "eslint-plugin-node": "^11.1.0", "mini-css-extract-plugin": "^1.3.3", "style-loader": "^2.0.0", + "ts-json-schema-generator": "^0.82.0", "ts-node": "^9.1.1", "typescript": "^4.1.3", "webpack": "^5.11.0", diff --git a/preload.ts b/preload.ts new file mode 100644 index 0000000..fd3d7bf --- /dev/null +++ b/preload.ts @@ -0,0 +1 @@ +import {} from 'electron'; \ No newline at end of file diff --git a/src/SettingConfig.schema.json b/src/SettingConfig.schema.json new file mode 100644 index 0000000..aedb7ab --- /dev/null +++ b/src/SettingConfig.schema.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/SettingConfig", + "definitions": { + "SettingConfig": { + "type": "object", + "properties": { + "localmode": { + "type": "boolean", + "description": "if true, server will bind on '127.0.0.1' rather than '0.0.0.0'" + }, + "guest": { + "type": "array", + "items": { + "$ref": "#/definitions/Permission" + }, + "description": "guest permission" + }, + "jwt_secretkey": { + "type": "string", + "description": "JWT secret key. if you change its value, all access tokens are invalidated." + }, + "port": { + "type": "number", + "description": "the port which running server is binding on." + }, + "mode": { + "type": "string", + "enum": [ + "development", + "production" + ] + }, + "cli": { + "type": "boolean", + "description": "if true, do not show 'electron' window and show terminal only." + }, + "forbid_remote_admin_login": { + "type": "boolean", + "description": "forbid to login admin from remote client. but, it do not invalidate access token. \r if you want to invalidate access token, change 'jwt_secretkey'." + }, + "$schema": { + "type": "string" + } + }, + "required": [ + "localmode", + "guest", + "jwt_secretkey", + "port", + "mode", + "cli", + "forbid_remote_admin_login" + ], + "additionalProperties": false + }, + "Permission": { + "type": "string", + "enum": [ + "ModifyTag", + "QueryContent", + "ModifyTagDesc" + ] + } + } +} \ No newline at end of file diff --git a/src/setting.ts b/src/SettingConfig.ts similarity index 85% rename from src/setting.ts rename to src/SettingConfig.ts index 6378071..944f911 100644 --- a/src/setting.ts +++ b/src/SettingConfig.ts @@ -1,9 +1,8 @@ -import { Settings } from '@material-ui/icons'; import { randomBytes } from 'crypto'; import { existsSync, readFileSync, writeFileSync } from 'fs'; import { Permission } from './permission/permission'; -export type Setting = { +export interface SettingConfig { /** * if true, server will bind on '127.0.0.1' rather than '0.0.0.0' */ @@ -31,7 +30,7 @@ export type Setting = { * if you want to invalidate access token, change 'jwt_secretkey'.*/ forbid_remote_admin_login:boolean, } -const default_setting:Setting = { +const default_setting:SettingConfig = { localmode: true, guest:[], @@ -41,15 +40,15 @@ const default_setting:Setting = { cli:false, forbid_remote_admin_login:true, } -let setting: null|Setting = null; +let setting: null|SettingConfig = null; -const setEmptyToDefault = (target:any,default_table:Setting)=>{ +const setEmptyToDefault = (target:any,default_table:SettingConfig)=>{ let diff_occur = false; for(const key in default_table){ if(key === undefined || key in target){ continue; } - target[key] = default_table[key as keyof Setting]; + target[key] = default_table[key as keyof SettingConfig]; diff_occur = true; } return diff_occur; @@ -61,9 +60,9 @@ export const read_setting_from_file = ()=>{ if(partial_occur){ writeFileSync("settings.json",JSON.stringify(ret)); } - return ret as Setting; + return ret as SettingConfig; } -export function get_setting():Setting{ +export function get_setting():SettingConfig{ if(setting === null){ setting = read_setting_from_file(); const env = process.env.NODE_ENV || 'development'; diff --git a/src/client/accessor/document.ts b/src/client/accessor/document.ts index 98d442f..36d25fa 100644 --- a/src/client/accessor/document.ts +++ b/src/client/accessor/document.ts @@ -44,6 +44,7 @@ export class ClientDocumentAccessor implements DocumentAccessor{ return ret; } async add(c: DocumentBody): Promise{ + throw new Error("not allow"); const res = await fetch(`${baseurl}`,{ method: "POST", body: JSON.stringify(c) diff --git a/src/client/page/difference.tsx b/src/client/page/difference.tsx index ca7ffed..8257eac 100644 --- a/src/client/page/difference.tsx +++ b/src/client/page/difference.tsx @@ -1,11 +1,51 @@ -import React from 'react'; +import React, { useContext, useEffect, useState } from 'react'; import { CommonMenuList, Headline } from "../component/mod"; import { UserContext } from "../state"; -import { Grid, Typography } from "@material-ui/core"; +import { Grid, Paper, Typography } from "@material-ui/core"; export function DifferencePage(){ + const ctx = useContext(UserContext); + const [diffList,setDiffList] = useState< + {type:string,value:{path:string,type:string}[]}[] + >([]); + const doLoad = async ()=>{ + const list = await fetch('/api/diff/list'); + if(list.ok){ + const inner = await list.json(); + setDiffList(inner); + } + else{ + //setDiffList([]); + } + }; + useEffect( + ()=>{ + doLoad(); + const i = setInterval(doLoad,5000); + return ()=>{ + clearInterval(i); + } + },[] + ) + const Commit = async(x:{type:string,path:string})=>{ + const res = await fetch('/api/diff/commit',{ + method:'POST', + body: JSON.stringify([{...x}]), + headers:{ + 'content-type':'application/json' + } + }); + const bb = await res.json(); + if(bb.ok){ + doLoad(); + } + } const menu = CommonMenuList(); + return ( -
Not implemented
+ {diffList.map(x=> + {x.type} + {x.value.map(y=>Commit(y)}>{y.path})} + )} ) } \ No newline at end of file diff --git a/src/content/file.ts b/src/content/file.ts index 894cbce..8339c59 100644 --- a/src/content/file.ts +++ b/src/content/file.ts @@ -3,37 +3,65 @@ import Router from 'koa-router'; import {createHash} from 'crypto'; import {promises} from 'fs' import {extname} from 'path'; +import { DocumentBody } from '../model/mod'; +import path from 'path'; /** * content file or directory referrer */ export interface ContentFile{ getHash():Promise; - getDesc():Promise; + createDocumentBody():Promise; readonly path: string; readonly type: string; } -type ContentFileConstructor = (new (path:string,desc?:object) => ContentFile)&{content_type:string}; +export type ContentConstructOption = { + hash: string, + tags: string[], + title: string, + additional: JSONMap +} +type ContentFileConstructor = (new (path:string,option?:ContentConstructOption) => ContentFile)&{content_type:string}; export const createDefaultClass = (type:string):ContentFileConstructor=>{ let cons = class implements ContentFile{ readonly path: string; - type = type; + //type = type; static content_type = type; + protected hash: string| undefined; - constructor(path:string,option?:object){ + constructor(path:string,option?:ContentConstructOption){ this.path = path; + this.hash = option?.hash; + } + async createDocumentBody(): Promise { + const {base,dir, name} = path.parse(this.path); + const ret = { + title : name, + basepath : dir, + additional: {}, + content_type: cons.content_type, + filename: base, + tags: [], + content_hash: await this.getHash(), + } as DocumentBody; + return ret; + } + get type():string{ + return cons.content_type; } async getDesc(): Promise { return null; } async getHash():Promise{ + if(this.hash !== undefined) return this.hash; const stat = await promises.stat(this.path); const hash = createHash("sha512"); - hash.update(extname(this.type)); + hash.update(extname(this.path)); hash.update(stat.mode.toString()); //if(this.desc !== undefined) // hash.update(JSON.stringify(this.desc)); hash.update(stat.size.toString()); - return hash.digest("base64"); + this.hash = hash.digest("base64"); + return this.hash; } }; return cons; @@ -43,11 +71,11 @@ export function registerContentReferrer(s: ContentFileConstructor){ console.log(`registered content type: ${s.content_type}`) ContstructorTable[s.content_type] = s; } -export function createContentFile(type:string,path:string,option?:object){ +export function createContentFile(type:string,path:string,option?:ContentConstructOption){ const constructorMethod = ContstructorTable[type]; if(constructorMethod === undefined){ - console.log(type); - throw new Error("undefined"); + console.log(`${type} are not in ${JSON.stringify(ContstructorTable)}`); + throw new Error("construction method of the content type is undefined"); } return new constructorMethod(path,option); } diff --git a/src/content/manga.ts b/src/content/manga.ts index 3e4e433..f01e2d5 100644 --- a/src/content/manga.ts +++ b/src/content/manga.ts @@ -1,10 +1,11 @@ -import {ContentFile} from './file'; -import {createDefaultClass,registerContentReferrer} from './file'; +import {ContentFile, createDefaultClass,registerContentReferrer, ContentConstructOption} from './file'; import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap'; export class MangaReferrer extends createDefaultClass("manga"){ desc: object|null|undefined; - constructor(path:string,option?:object|undefined){ + additional: object| undefined; + constructor(path:string,option?:ContentConstructOption){ super(path); + this.additional = option; } async getDesc(){ if(this.desc !== undefined){ diff --git a/src/content/video.ts b/src/content/video.ts index 6d2ca2b..edf2095 100644 --- a/src/content/video.ts +++ b/src/content/video.ts @@ -1,8 +1,8 @@ -import {ContentFile, registerContentReferrer} from './file'; +import {ContentFile, registerContentReferrer, ContentConstructOption} from './file'; import {createDefaultClass} from './file'; export class VideoReferrer extends createDefaultClass("video"){ - constructor(path:string,desc?:object){ + constructor(path:string,desc?:ContentConstructOption){ super(path,desc); } } diff --git a/src/database.ts b/src/database.ts index 388451c..e13afde 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,7 +1,7 @@ import { existsSync } from 'fs'; import Knex from 'knex'; import {Knex as KnexConfig} from './config'; -import { get_setting } from './setting'; +import { get_setting } from './SettingConfig'; export async function connectDB(){ const config = KnexConfig.config; diff --git a/src/db/doc.ts b/src/db/doc.ts index 8230072..d60ea47 100644 --- a/src/db/doc.ts +++ b/src/db/doc.ts @@ -8,7 +8,7 @@ type DBTagContentRelation = { tag_name:string } -class KnexContentsAccessor implements DocumentAccessor{ +class KnexDocumentAccessor implements DocumentAccessor{ knex : Knex; tagController: TagAccessor; constructor(knex : Knex){ @@ -19,6 +19,7 @@ class KnexContentsAccessor implements DocumentAccessor{ const {tags,additional, ...rest} = c; const id_lst = await this.knex.insert({ additional:JSON.stringify(additional), + created_at:Date.now(), ...rest }).into('document'); const id = id_lst[0]; @@ -53,9 +54,20 @@ class KnexContentsAccessor implements DocumentAccessor{ return { ...first, tags:ret_tags, - additional: JSON.parse(first.additional || "{}"), + additional: first.additional !== null ? JSON.parse(first.additional) : {}, }; }; + async findDeleted(content_type:string){ + const s = await this.knex.select("*") + .where({content_type:content_type}) + .whereNotNull("update_at") + .from("document"); + return s.map(x=>({ + ...x, + tags:[], + additional:{} + })); + } async findList(option?:QueryListOption){ option = option || {}; const allow_tag = option.allow_tag || []; @@ -94,6 +106,7 @@ class KnexContentsAccessor implements DocumentAccessor{ } } query = query.limit(limit); + query = query.orderBy('id',"desc"); return query; } let query = buildquery(); @@ -119,13 +132,14 @@ class KnexContentsAccessor implements DocumentAccessor{ } return result; }; - async findListByBasePath(path:string):Promise{ - let results = await this.knex.select("*").from("document").where({basepath:path}); + async findByPath(path:string,filename?:string):Promise{ + const e = filename == undefined ? {} : {filename:filename} + const results = await this.knex.select("*").from("document").where({basepath:path,...e}); return results.map(x=>({ ...x, tags:[], - additional:JSON.parse(x.additional || "{}"), - })); + additional:{} + })) } async update(c:Partial & { id:number }){ const {id,tags,...rest} = c; @@ -150,6 +164,6 @@ class KnexContentsAccessor implements DocumentAccessor{ return true; } } -export const createKnexContentsAccessor = (knex:Knex): DocumentAccessor=>{ - return new KnexContentsAccessor(knex); +export const createKnexDocumentAccessor = (knex:Knex): DocumentAccessor=>{ + return new KnexDocumentAccessor(knex); } \ No newline at end of file diff --git a/src/diff/MangaConfig.schema.json b/src/diff/MangaConfig.schema.json new file mode 100644 index 0000000..0e9bb2f --- /dev/null +++ b/src/diff/MangaConfig.schema.json @@ -0,0 +1 @@ +{"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/MangaConfig","definitions":{"MangaConfig":{"type":"object","properties":{"watch":{"type":"array","items":{"type":"string"}},"$schema":{"type":"string"}},"required":["watch"],"additionalProperties":false}}} \ No newline at end of file diff --git a/src/diff/MangaConfig.ts b/src/diff/MangaConfig.ts new file mode 100644 index 0000000..d3964d0 --- /dev/null +++ b/src/diff/MangaConfig.ts @@ -0,0 +1,6 @@ +import Schema from './MangaConfig.schema.json'; + +export interface MangaConfig{ + watch:string[] +} + diff --git a/src/diff/content_handler.ts b/src/diff/content_handler.ts new file mode 100644 index 0000000..b240086 --- /dev/null +++ b/src/diff/content_handler.ts @@ -0,0 +1,74 @@ +import {join as pathjoin} from 'path'; +import {Document, DocumentAccessor} from '../model/mod'; +import { ContentFile, createContentFile } from '../content/mod'; +import {IDiffWatcher} from './watcher'; +import {ContentList} from './content_list'; + +//refactoring needed. +export class ContentDiffHandler{ + waiting_list:ContentList; + tombstone: Map;//hash, contentfile + doc_cntr: DocumentAccessor; + content_type: string; + constructor(cntr: DocumentAccessor,content_type:string){ + this.waiting_list = new ContentList(); + this.tombstone = new Map(); + this.doc_cntr = cntr; + this.content_type = content_type; + } + async setup(){ + const deleted = await this.doc_cntr.findDeleted(this.content_type); + for (const it of deleted) { + this.tombstone.set(it.content_hash,it); + } + } + register(diff:IDiffWatcher){ + diff.on('create',(filename)=>this.OnCreated(diff.path,filename)) + .on('delete',(filename)=>this.OnDeleted(diff.path,filename)) + .on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename)); + } + private async OnDeleted(basepath:string,filename:string){ + const cpath = pathjoin(basepath,filename); + if(this.waiting_list.hasPath(cpath)){ + this.waiting_list.deletePath(cpath); + return; + } + const dbc = await this.doc_cntr.findByPath(basepath,filename); + if(dbc.length === 0) return; //ignore + if(this.waiting_list.hasHash(dbc[0].content_hash)){ + //if path changed, update changed path. + await this.doc_cntr.update({ + id:dbc[0].id, + deleted_at: null, + filename:filename, + basepath:basepath + }); + return; + } + //db invalidate + await this.doc_cntr.update({ + id:dbc[0].id, + deleted_at: Date.now(), + }); + this.tombstone.set(dbc[0].content_hash, dbc[0]); + } + private async OnCreated(basepath:string,filename:string){ + const content = createContentFile(this.content_type,pathjoin(basepath,filename)); + const hash = await content.getHash(); + const c = this.tombstone.get(hash); + if(c !== undefined){ + this.doc_cntr.update({ + id: c.id, + deleted_at: null, + filename:filename, + basepath:basepath + }); + return; + } + this.waiting_list.set(content); + } + private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){ + const doc = await this.doc_cntr.findByPath(basepath,prev_filename); + await this.doc_cntr.update({...doc[0],filename:cur_filename}); + } +} \ No newline at end of file diff --git a/src/diff/content_list.ts b/src/diff/content_list.ts new file mode 100644 index 0000000..a3b268c --- /dev/null +++ b/src/diff/content_list.ts @@ -0,0 +1,62 @@ +import { ContentFile } from '../content/mod'; +import event from 'events'; + +interface ContentListEvent{ + 'set':(c:ContentFile)=>void, + 'delete':(c:ContentFile)=>void, +} + +export class ContentList extends event.EventEmitter{ + cl:Map; + hl:Map; + on(event:U,listener:ContentListEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + return super.emit(event,...arg); + } + constructor(){ + super(); + this.cl = new Map; + this.hl = new Map; + } + hasHash(s:string){ + return this.hl.has(s); + } + hasPath(p:string){ + return this.cl.has(p); + } + getHash(s:string){ + return this.hl.get(s) + } + getPath(p:string){ + return this.cl.get(p); + } + async set(c:ContentFile){ + const path = c.path; + const hash = await c.getHash(); + this.cl.set(path,c); + this.hl.set(hash,c); + this.emit('set',c); + } + async delete(c:ContentFile){ + let r = true; + r &&= this.cl.delete(c.path); + r &&= this.hl.delete(await c.getHash()); + this.emit('delete',c); + return r; + } + async deletePath(p:string){ + const o = this.getPath(p); + if(o === undefined) return false; + return this.delete(o); + } + async deleteHash(s:string){ + const o = this.getHash(s); + if(o === undefined) return false; + return this.delete(o); + } + getAll(){ + return [...this.cl.values()]; + } +} diff --git a/src/diff/diff.ts b/src/diff/diff.ts index d9578ea..354e5a4 100644 --- a/src/diff/diff.ts +++ b/src/diff/diff.ts @@ -1,82 +1,39 @@ -import { watch } from 'fs'; -import { promises } from 'fs'; -import { ContentFile, createContentReferrer, getContentRefererConstructor } from '../content/referrer' -import path from 'path'; - -const readdir = promises.readdir; - - -export class Watcher{ - private _type: string; - private _path:string; - /** - * @todo : alter type Map - */ - private _added: ContentFile[]; - private _deleted: ContentFile[]; - constructor(path:string,type:string){ - this._path = path; - this._added =[]; - this._deleted =[]; - this._type = type; +import { DocumentAccessor } from '../model/doc'; +import {ContentDiffHandler} from './content_handler'; +import { CommonDiffWatcher } from './watcher'; +//import {join as pathjoin} from 'path'; +export class DiffManager{ + watching: {[content_type:string]:ContentDiffHandler}; + doc_cntr: DocumentAccessor; + constructor(contorller: DocumentAccessor){ + this.watching = {}; + this.doc_cntr = contorller; } - public get added() : ContentFile[] { - return this._added; + async register(content_type:string,path:string){ + if(this.watching[content_type] === undefined){ + this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type); + } + const watcher = new CommonDiffWatcher(path); + this.watching[content_type].register(watcher); + const initial_doc = await this.doc_cntr.findByPath(path); + await watcher.setup(initial_doc.map(x=>x.filename)); + watcher.watch(); } - /*public set added(diff : FileDiff[]) { - this._added = diff; - }*/ - public get deleted(): ContentFile[]{ - return this._deleted; + async commit(type:string,path:string){ + const list = this.watching[type].waiting_list; + const c = list.getPath(path); + if(c===undefined){ + throw new Error("path is not exist"); + } + await list.delete(c); + const body = await c.createDocumentBody(); + const id = await this.doc_cntr.add(body); + return id; } - /*public set deleted(diff : FileDiff[]){ - this._deleted = diff; - }*/ - public get path(){ - return this._path; + getAdded(){ + return Object.keys(this.watching).map(x=>({ + type:x, + value:this.watching[x].waiting_list.getAll(), + })); } - public get type(){ - return this._type; - } - private createCR(filename: string){ - return createContentReferrer(`${this.path}/${filename}`,this.type); - } - /** - * - */ - async setup(initial_filenames:string[]){ - const cur = (await readdir(this._path,{ - encoding:"utf8", - withFileTypes: true, - })).filter(x=>x.isFile).map(x=>x.name); - let added = cur.filter(x => !initial_filenames.includes(x)); - let deleted = initial_filenames.filter(x=>!cur.includes(x)); - this._added = added.map(x=>this.createCR(x)); - this._deleted = deleted.map(x=>this.createCR(x)); - watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{ - if(eventType === "rename"){ - const cur = (await readdir(this._path,{ - encoding:"utf8", - withFileTypes: true, - })).filter(x=>x.isFile).map(x=>x.name); - //add - if(cur.includes(filename)){ - this._added.push(this.createCR(filename)); - } - else{ - //added has one - if(this._added.map(x=>x.path).includes(path.join(this.path,filename))){ - this._added = this._added.filter(x=> x.path !== path.join(this.path,filename)); - } - else { - this._deleted.push(this.createCR(filename)); - } - } - } - }); - } -} - -export class DiffWatcher{ - Watchers: {[basepath:string]:Watcher} = {}; -} \ No newline at end of file +}; \ No newline at end of file diff --git a/src/diff/mod.ts b/src/diff/mod.ts new file mode 100644 index 0000000..cb8a6cd --- /dev/null +++ b/src/diff/mod.ts @@ -0,0 +1,2 @@ +export * from './router'; +export * from './diff'; \ No newline at end of file diff --git a/src/diff/router.ts b/src/diff/router.ts new file mode 100644 index 0000000..1992056 --- /dev/null +++ b/src/diff/router.ts @@ -0,0 +1,61 @@ +import Koa from 'koa'; +import Router from 'koa-router'; +import { ContentFile } from '../content/mod'; +import { sendError } from '../route/error_handler'; +import {DiffManager} from './diff'; + +function content_file_to_return(x:ContentFile){ + return {path:x.path,type:x.type}; +} + +export const getAdded = (diffmgr:DiffManager)=> (ctx:Koa.Context,next:Koa.Next)=>{ + const ret = diffmgr.getAdded(); + ctx.body = ret.map(x=>({ + type:x.type, + value:x.value.map(x=>({path:x.path,type:x.type})), + })); + ctx.type = 'json'; +} + +type PostAddedBody = { + type:string, + path:string, +}[]; + +function checkPostAddedBody(body: any): body is PostAddedBody{ + if(body instanceof Array){ + return body.map(x=> 'type' in x && 'path' in x).every(x=>x); + } + return false; +} + +export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next)=>{ + const reqbody = ctx.request.body; + console.log(reqbody); + if(!checkPostAddedBody(reqbody)){ + sendError(400,"format exception"); + return; + } + const allWork = reqbody.map(op=>diffmgr.commit(op.type,op.path)); + const results = await Promise.all(allWork); + ctx.body = { + ok:true, + docs:results, + } + ctx.type = 'json'; +} +/* +export const getNotWatched = (diffmgr : DiffManager)=> (ctx:Router.IRouterContext,next:Koa.Next)=>{ + ctx.body = { + added: diffmgr.added.map(content_file_to_return), + deleted: diffmgr.deleted.map(content_file_to_return), + }; + ctx.type = 'json'; +}*/ + +export function createDiffRouter(diffmgr: DiffManager){ + const ret = new Router(); + ret.get("/list",getAdded(diffmgr)); + ret.post("/commit",postAdded(diffmgr)); + return ret; +} \ No newline at end of file diff --git a/src/diff/watcher.ts b/src/diff/watcher.ts new file mode 100644 index 0000000..0c69220 --- /dev/null +++ b/src/diff/watcher.ts @@ -0,0 +1,78 @@ +import { FSWatcher, watch } from 'fs'; +import { promises } from 'fs'; +import event from 'events'; + + +const readdir = promises.readdir; + +interface DiffWatcherEvent{ + 'create':(filename:string)=>void, + 'delete':(filename:string)=>void, + 'change':(prev_filename:string,cur_filename:string)=>void, +} + +export interface IDiffWatcher extends event.EventEmitter { + on(event:U,listener:DiffWatcherEvent[U]): this; + emit(event:U,...arg:Parameters): boolean; + readonly path: string; +} + +export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{ + on(event:U,listener:DiffWatcherEvent[U]): this{ + return super.on(event,listener); + } + emit(event:U,...arg:Parameters): boolean{ + return super.emit(event,...arg); + } + private _path:string; + private _watcher: FSWatcher|null; + + constructor(path:string){ + super(); + this._path = path; + this._watcher = null; + } + public get path(){ + return this._path; + } + /** + * setup + * @argument initial_filenames filename in path + */ + async setup(initial_filenames:string[]){ + const cur = (await readdir(this._path,{ + encoding:"utf8", + withFileTypes: true, + })).filter(x=>x.isFile).map(x=>x.name); + //Todo : reduce O(nm) to O(n+m) using hash map. + let added = cur.filter(x => !initial_filenames.includes(x)); + let deleted = initial_filenames.filter(x=>!cur.includes(x)); + for (const iterator of added) { + this.emit('create',iterator); + } + for (const iterator of deleted){ + this.emit('delete',iterator); + } + } + watch():FSWatcher{ + this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{ + if(eventType === "rename"){ + const cur = (await readdir(this._path,{ + encoding:"utf8", + withFileTypes: true, + })).filter(x=>x.isFile).map(x=>x.name); + //add + if(cur.includes(filename)){ + this.emit('create',filename); + } + else{ + this.emit('delete',filename) + } + } + }); + return this._watcher; + } + watchClose(){ + this._watcher?.close() + } +} \ No newline at end of file diff --git a/src/login.ts b/src/login.ts index 6089495..d7af809 100644 --- a/src/login.ts +++ b/src/login.ts @@ -5,7 +5,7 @@ import { sendError } from "./route/error_handler"; import Knex from "knex"; import { createKnexUserController } from "./db/mod"; import { request } from "http"; -import { get_setting } from "./setting"; +import { get_setting } from "./SettingConfig"; import { IUser, UserAccessor } from "./model/mod"; type PayloadInfo = { diff --git a/src/model/doc.ts b/src/model/doc.ts index 3d2b7c5..5134ce7 100644 --- a/src/model/doc.ts +++ b/src/model/doc.ts @@ -1,17 +1,12 @@ import {TagAccessor} from './tag'; import {check_type} from '../util/type_check' -type JSONPrimitive = null|boolean|number|string; -interface JSONMap extends Record{} -interface JSONArray extends Array{}; -type JSONType = JSONMap|JSONPrimitive|JSONArray; - export interface DocumentBody{ title : string, content_type : string, basepath : string, filename : string, - content_hash? : string, + content_hash : string, additional : JSONMap, tags : string[],//eager loading } @@ -32,6 +27,8 @@ export const isDocBody = (c : any):c is DocumentBody =>{ export interface Document extends DocumentBody{ readonly id: number; + readonly created_at:number; + readonly deleted_at:number|null; }; export const isDoc = (c: any):c is Document =>{ @@ -88,9 +85,14 @@ export interface DocumentAccessor{ */ findById: (id:number,tagload?:boolean)=> Promise, /** - * + * find by base path and filename. + * if you call this function with filename, its return array length is 0 or 1. */ - findListByBasePath:(basepath: string)=>Promise; + findByPath:(basepath: string,filename?:string)=>Promise; + /** + * find deleted content + */ + findDeleted:(content_type:string)=>Promise; /** * update document except tag. */ diff --git a/src/route/contents.ts b/src/route/contents.ts index ee370f7..9609f22 100644 --- a/src/route/contents.ts +++ b/src/route/contents.ts @@ -71,7 +71,7 @@ const UpdateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont ctx.body = JSON.stringify(success); ctx.type = 'json'; } -const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => { +/*const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => { const content_desc = ctx.request.body; if(!isDocBody(content_desc)){ return sendError(400,"it is not a valid format"); @@ -79,7 +79,7 @@ const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont const id = await controller.add(content_desc); ctx.body = JSON.stringify(id); ctx.type = 'json'; -}; +};*/ const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{ let tag_name = ctx.params['tag']; const num = Number.parseInt(ctx.params['num']); @@ -137,7 +137,7 @@ export const getContentRouter = (controller: DocumentAccessor)=>{ ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller)); ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller)); //ret.use("/:num(\\d+)/:content_type"); - ret.post("/",AdminOnly,CreateContentHandler(controller)); + //ret.post("/",AdminOnly,CreateContentHandler(controller)); ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller)); ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller)); ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller)); diff --git a/src/route/context.ts b/src/route/context.ts index 2e08aba..eea6123 100644 --- a/src/route/context.ts +++ b/src/route/context.ts @@ -1,5 +1,3 @@ -import {ContentReferrer} from '../content/mod'; - export type ContentLocation = { path:string, type:string, diff --git a/src/server.ts b/src/server.ts index fd96956..7c81d38 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,25 +1,30 @@ import Koa from 'koa'; import Router from 'koa-router'; -import {get_setting} from './setting'; +import {get_setting} from './SettingConfig'; import {connectDB} from './database'; -import {Watcher} from './diff/diff' +import {DiffManager, createDiffRouter} from './diff/mod'; import { createReadStream, readFileSync } from 'fs'; import getContentRouter from './route/contents'; -import { createKnexContentsAccessor } from './db/doc'; +import { createKnexDocumentAccessor } from './db/mod'; import bodyparser from 'koa-bodyparser'; import {error_handler} from './route/error_handler'; - import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login'; import {createInterface as createReadlineInterface} from 'readline'; + //let Koa = require("koa"); export async function create_server(){ - let setting = get_setting(); + const setting = get_setting(); let db = await connectDB(); + + let diffmgr = new DiffManager(createKnexDocumentAccessor(db)); + let diff_router = createDiffRouter(diffmgr); + diffmgr.register("manga","testdata"); + if(setting.cli){ const userAdmin = await getAdmin(db); if(await isAdminFirst(userAdmin)){ @@ -38,9 +43,12 @@ export async function create_server(){ app.use(createUserMiddleWare(db)); //app.use(ctx=>{ctx.state['setting'] = settings}); + const index_html = readFileSync("index.html"); let router = new Router(); + router.use('/api/diff',diff_router.routes()); + router.use('/api/diff',diff_router.allowedMethods()); //let watcher = new Watcher(setting.path[0]); //await watcher.setup([]); @@ -63,7 +71,7 @@ export async function create_server(){ if(setting.mode === "development") static_file_server('dist/js/bundle.js.map','text'); - const content_router = getContentRouter(createKnexContentsAccessor(db)); + const content_router = getContentRouter(createKnexDocumentAccessor(db)); router.use('/api/doc',content_router.routes()); router.use('/api/doc',content_router.allowedMethods()); diff --git a/src/types/db.d.ts b/src/types/db.d.ts index 5545f6a..7452d18 100644 --- a/src/types/db.d.ts +++ b/src/types/db.d.ts @@ -17,8 +17,10 @@ declare module "knex" { content_type: string; basepath: string; filename: string; - content_hash?: string; - additional?: string; + created_at: number; + deleted_at: number|null; + content_hash: string; + additional: string|null; }; doc_tag_relation: { doc_id: number; diff --git a/src/types/json.d.ts b/src/types/json.d.ts new file mode 100644 index 0000000..3ff8968 --- /dev/null +++ b/src/types/json.d.ts @@ -0,0 +1,5 @@ + +type JSONPrimitive = null|boolean|number|string; +interface JSONMap extends Record{} +interface JSONArray extends Array{}; +type JSONType = JSONMap|JSONPrimitive|JSONArray; \ No newline at end of file diff --git a/test.ts b/test.ts new file mode 100644 index 0000000..f89b924 --- /dev/null +++ b/test.ts @@ -0,0 +1,10 @@ +import Knex from 'knex'; +import {connectDB} from './src/database'; + +async function main() { + const db = await connectDB(); + const query = db.update({deleted_at: null}).from('document'); + console.log(query.toSQL()); +} + +main() \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 0867e75..5ed2e68 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -49,6 +49,7 @@ // "types": [], /* Type declaration files to be included in compilation. */ "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + "resolveJsonModule": true, // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */