event driven watcher

.gitignore

@@ -10,4 +10,5 @@ package-lock.json
 devdb.sqlite3
 build/**
 app/**
-settings.json
+settings.json
+*config.json

package.json

@@ -31,14 +31,16 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "chokidar": "^3.5.1",
+    "jsonschema": "^1.4.0",
     "jsonwebtoken": "^8.5.1",
-    "knex": "^0.21.14",
-    "koa": "^2.13.0",
+    "knex": "^0.21.16",
+    "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
     "koa-router": "^10.0.0",
     "natural-orderby": "^2.0.3",
     "node-stream-zip": "^1.12.0",
-    "sqlite3": "^5.0.0"
+    "sqlite3": "^5.0.1"
   },
   "devDependencies": {
     "@types/jsonwebtoken": "^8.5.0",
@@ -46,8 +48,8 @@
     "@types/koa": "^2.11.6",
     "@types/koa-bodyparser": "^4.3.0",
     "@types/koa-router": "^7.4.1",
-    "@types/node": "^14.14.16",
-    "electron": "^11.1.1",
+    "@types/node": "^14.14.22",
+    "electron": "^11.2.0",
     "electron-builder": "^22.9.1",
     "eslint-plugin-node": "^11.1.0",
     "ts-json-schema-generator": "^0.82.0",

src/diff/MangaConfig.ts

@@ -1,6 +0,0 @@
-import Schema from './MangaConfig.schema.json';
-
-export interface MangaConfig{
-    watch:string[]
-}
-

src/diff/content_handler.ts

@@ -1,4 +1,4 @@
-import {join as pathjoin} from 'path';
+import {basename, dirname, join as pathjoin} from 'path';
 import {Document, DocumentAccessor} from '../model/mod';
 import { ContentFile, createContentFile } from '../content/mod';
 import {IDiffWatcher} from './watcher';
@@ -23,12 +23,13 @@ export class ContentDiffHandler{
         }
     }
     register(diff:IDiffWatcher){
-        diff.on('create',(filename)=>this.OnCreated(diff.path,filename))
-            .on('delete',(filename)=>this.OnDeleted(diff.path,filename))
-            .on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename));
+        diff.on('create',(path)=>this.OnCreated(path))
+            .on('delete',(path)=>this.OnDeleted(path))
+            .on('change',(prev,cur)=>this.OnChanged(prev,cur));
     }
-    private async OnDeleted(basepath:string,filename:string){
-        const cpath = pathjoin(basepath,filename);
+    private async OnDeleted(cpath: string){
+        const basepath = dirname(cpath);
+        const filename = basename(cpath);
         if(this.waiting_list.hasPath(cpath)){
             this.waiting_list.deletePath(cpath);
             return;
@@ -52,7 +53,9 @@ export class ContentDiffHandler{
         });
         this.tombstone.set(dbc[0].content_hash, dbc[0]);
     }
-    private async OnCreated(basepath:string,filename:string){
+    private async OnCreated(cpath:string){
+        const basepath = dirname(cpath);
+        const filename = basename(cpath);
         const content = createContentFile(this.content_type,pathjoin(basepath,filename));
         const hash = await content.getHash();
         const c = this.tombstone.get(hash);
@@ -67,8 +70,14 @@ export class ContentDiffHandler{
         }
         this.waiting_list.set(content);
     }
-    private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){
-        const doc = await this.doc_cntr.findByPath(basepath,prev_filename);
-        await this.doc_cntr.update({...doc[0],filename:cur_filename});
+    private async OnChanged(prev_path:string,cur_path:string){
+        const prev_basepath = dirname(prev_path);
+        const prev_filename = basename(prev_path);
+        const cur_basepath = dirname(cur_path);
+        const cur_filename = basename(cur_path);
+        const doc = await this.doc_cntr.findByPath(prev_basepath,prev_filename);
+        await this.doc_cntr.update({...doc[0],
+            basepath:cur_basepath,
+            filename:cur_filename});
     }
 }

src/diff/diff.ts

@@ -1,6 +1,7 @@
 import { DocumentAccessor } from '../model/doc';
 import {ContentDiffHandler} from './content_handler';
-import { CommonDiffWatcher } from './watcher';
+import { IDiffWatcher } from './watcher';
+
 //import {join as pathjoin} from 'path';
 export class DiffManager{
     watching: {[content_type:string]:ContentDiffHandler};
@@ -9,15 +10,12 @@ export class DiffManager{
         this.watching = {};
         this.doc_cntr = contorller;
     }
-    async register(content_type:string,path:string){
+    async register(content_type:string,watcher:IDiffWatcher){
         if(this.watching[content_type] === undefined){
             this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type);
         }
-        const watcher = new CommonDiffWatcher(path);
         this.watching[content_type].register(watcher);
-        const initial_doc = await this.doc_cntr.findByPath(path);
-        await watcher.setup(initial_doc.map(x=>x.filename));
-        watcher.watch();
+        await watcher.setup(this.doc_cntr);
     }
     async commit(type:string,path:string){
         const list = this.watching[type].waiting_list;

src/diff/watcher.ts

@@ -1,78 +1,25 @@
 import { FSWatcher, watch } from 'fs';
 import { promises } from 'fs';
 import event from 'events';
-
+import { join } from 'path';
+import { DocumentAccessor } from '../model/doc';
 
 const readdir = promises.readdir;
 
-interface DiffWatcherEvent{
-    'create':(filename:string)=>void,
-    'delete':(filename:string)=>void,
-    'change':(prev_filename:string,cur_filename:string)=>void,
+export interface DiffWatcherEvent{
+    'create':(path:string)=>void,
+    'delete':(path:string)=>void,
+    'change':(prev_path:string,cur_path:string)=>void,
 }
 
 export interface IDiffWatcher extends event.EventEmitter {
     on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this;
     emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean;
-    readonly path: string;
+    setup(cntr:DocumentAccessor):Promise<void>;
 }
 
-export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
-    on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
-        return super.on(event,listener);
-    }
-    emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
-        return super.emit(event,...arg);
-    }
-    private _path:string;
-    private _watcher: FSWatcher|null;
-
-    constructor(path:string){
-        super();
-        this._path = path;
-        this._watcher = null;
-    }
-    public get path(){
-        return this._path;
-    }
-    /**
-     * setup
-     * @argument initial_filenames filename in path
-     */
-    async setup(initial_filenames:string[]){
-        const cur = (await readdir(this._path,{
-            encoding:"utf8",
-            withFileTypes: true,
-        })).filter(x=>x.isFile).map(x=>x.name);
-        //Todo : reduce O(nm) to O(n+m) using hash map.
-        let added = cur.filter(x => !initial_filenames.includes(x));
-        let deleted = initial_filenames.filter(x=>!cur.includes(x));
-        for (const iterator of added) {
-            this.emit('create',iterator);
-        }
-        for (const iterator of deleted){
-            this.emit('delete',iterator);
-        }
-    }
-    watch():FSWatcher{
-        this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
-            if(eventType === "rename"){
-                const cur = (await readdir(this._path,{
-                    encoding:"utf8",
-                    withFileTypes: true,
-                })).filter(x=>x.isFile).map(x=>x.name);
-                //add
-                if(cur.includes(filename)){
-                    this.emit('create',filename);
-                }
-                else{
-                    this.emit('delete',filename)
-                }
-            }
-        });
-        return this._watcher;
-    }
-    watchClose(){
-        this._watcher?.close()
-    }
+export function linkWatcher(fromWatcher :IDiffWatcher, toWatcher: IDiffWatcher){
+    fromWatcher.on("create",p=>toWatcher.emit("create",p));
+    fromWatcher.on("delete",p=>toWatcher.emit("delete",p));
+    fromWatcher.on("change",(p,c)=>toWatcher.emit("change",p,c));
 }

src/diff/watcher/MangaConfig.ts

@@ -0,0 +1,8 @@
+import {ConfigManager} from '../../util/configRW';
+import MangaSchema from "./MangaConfig.schema.json"
+export interface MangaConfig{
+    watch:string[]
+}
+
+export const MangaConfig = new ConfigManager<MangaConfig>("manga_config.json",{watch:[]},MangaSchema);
+

src/diff/watcher/common_watcher.ts

@@ -0,0 +1,45 @@
+import event from 'events';
+import {FSWatcher,watch,promises} from 'fs';
+import {IDiffWatcher, DiffWatcherEvent} from '../watcher';
+import {join} from 'path';
+import { DocumentAccessor } from '../../model/doc';
+import { setupHelp } from './util';
+
+const {readdir} = promises;
+
+export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
+    on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
+        return super.on(event,listener);
+    }
+    emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
+        return super.emit(event,...arg);
+    }
+    private _path:string;
+    private _watcher: FSWatcher;
+
+    constructor(path:string){
+        super();
+        this._path = path;
+        this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
+            if(eventType === "rename"){
+                const cur = await readdir(this._path);
+                //add
+                if(cur.includes(filename)){
+                    this.emit('create',join(this.path,filename));
+                }
+                else{
+                    this.emit('delete',join(this.path,filename))
+                }
+            }
+        });
+    }
+    async setup(cntr: DocumentAccessor): Promise<void> {
+        await setupHelp(this,this.path,cntr);
+    }
+    public get path(){
+        return this._path;
+    }
+    watchClose(){
+        this._watcher.close()
+    }
+}

src/diff/watcher/compositer.ts

@@ -0,0 +1,24 @@
+import { EventEmitter } from "events";
+import { DocumentAccessor } from "../../model/doc";
+import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher";
+
+
+export class WatcherCompositer extends EventEmitter implements IDiffWatcher{
+    refWatchers : IDiffWatcher[];
+    on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
+        return super.on(event,listener);
+    }
+    emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
+        return super.emit(event,...arg);
+    }
+    constructor(refWatchers:IDiffWatcher[]){
+        super();
+        this.refWatchers = refWatchers;
+        for(const refWatcher of this.refWatchers){
+            linkWatcher(refWatcher,this);
+        }
+    }
+    async setup(cntr: DocumentAccessor): Promise<void> {
+        await Promise.all(this.refWatchers.map(x=>x.setup(cntr)));
+    }
+}

src/diff/watcher/manga_watcher.ts

@@ -0,0 +1,17 @@
+import {IDiffWatcher, DiffWatcherEvent} from '../watcher';
+import {EventEmitter} from 'events';
+import { DocumentAccessor } from '../../model/doc';
+import { WatcherFilter } from './watcher_filter';
+import { RecursiveWatcher } from './recursive_watcher';
+import { MangaConfig } from './MangaConfig';
+import {WatcherCompositer} from './compositer'
+
+
+const createMangaWatcherBase = (path:string)=> {
+    return new WatcherFilter(new RecursiveWatcher(path),(x)=>x.endsWith(".zip"));
+}
+export const createMangaWatcher = ()=>{
+    const file = MangaConfig.get_config_file();
+    console.log(`register manga ${file.watch.join(",")}`)
+    return new WatcherCompositer(file.watch.map(path=>createMangaWatcherBase(path)));
+}

src/diff/watcher/recursive_watcher.ts

@@ -0,0 +1,59 @@
+import {watch, FSWatcher} from 'chokidar';
+import { EventEmitter } from 'events';
+import { join } from 'path';
+import { DocumentAccessor } from '../../model/doc';
+import { DiffWatcherEvent, IDiffWatcher } from '../watcher';
+import { setupHelp, setupRecursive } from './util';
+
+type RecursiveWatcherOption={
+    /** @default true */
+    watchFile?:boolean,
+    /** @default false */
+    watchDir?:boolean,
+}
+
+export class RecursiveWatcher extends EventEmitter implements IDiffWatcher  {
+    on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
+        return super.on(event,listener);
+    }
+    emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
+        return super.emit(event,...arg);
+    }
+    readonly path: string;
+    private watcher: FSWatcher
+
+    constructor(path:string, option?:RecursiveWatcherOption){
+        super();
+        this.path = path;
+        option = option || {
+            watchDir:false,
+            watchFile:true,
+        }
+        this.watcher = watch(path,{
+            persistent:true,
+            ignoreInitial:true,
+            depth:100,
+        });
+        if(option.watchFile === undefined || option.watchFile){
+            this.watcher.on("add",path=>{
+                const cpath = join(this.path,path);
+                this.emit("create",cpath);
+            }).on("unlink",path=>{
+                const cpath = join(this.path,path);
+                this.emit("delete",cpath);
+            });
+        }
+        if(option.watchDir){
+            this.watcher.on("addDir",path=>{
+                const cpath = join(this.path,path);
+                this.emit("create",cpath);
+            }).on("unlinkDir",path=>{
+                const cpath = join(this.path,path);
+                this.emit("delete",cpath);
+            })
+        }
+    }
+    async setup(cntr: DocumentAccessor): Promise<void> {
+        await setupRecursive(this,this.path,cntr);
+    }
+}

src/diff/watcher/util.ts

@@ -0,0 +1,35 @@
+import { EventEmitter } from "events";
+import { promises } from "fs";
+import { join } from "path";
+const {readdir} = promises;
+import { DocumentAccessor } from "../../model/doc";
+import { IDiffWatcher } from "../watcher";
+
+
+function setupCommon(watcher:IDiffWatcher,basepath:string,initial_filenames:string[],cur:string[]){
+    //Todo : reduce O(nm) to O(n+m) using hash map.
+    let added = cur.filter(x => !initial_filenames.includes(x));
+    let deleted = initial_filenames.filter(x=>!cur.includes(x));
+    for (const it of added) {
+        const cpath = join(basepath,it);
+        watcher.emit('create',cpath);
+    }
+    for (const it of deleted){
+        const cpath = join(basepath,it);
+        watcher.emit('delete',cpath);
+    }
+}
+export async function setupHelp(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){
+    const initial_document = await cntr.findByPath(basepath);
+    const initial_filenames = initial_document.map(x=>x.filename);
+    const cur = await readdir(basepath);
+    setupCommon(watcher,basepath,initial_filenames,cur);
+}
+export async function setupRecursive(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){
+    const initial_document = await cntr.findByPath(basepath);
+    const initial_filenames = initial_document.map(x=>x.filename);
+    const cur = await readdir(basepath,{withFileTypes:true});
+    setupCommon(watcher,basepath,initial_filenames,cur.map(x=>x.name));
+    await Promise.all([cur.filter(x=>x.isDirectory())
+        .map(x=>setupHelp(watcher,join(basepath,x.name),cntr))]);
+}

src/diff/watcher/watcher_filter.ts

@@ -0,0 +1,45 @@
+import { EventEmitter } from "events";
+import { DocumentAccessor } from "../../model/doc";
+import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher";
+
+
+export class WatcherFilter extends EventEmitter implements IDiffWatcher{
+    refWatcher : IDiffWatcher;
+    filter : (filename:string)=>boolean;;
+    on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
+        return super.on(event,listener);
+    }
+    emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
+        if(event === "change"){
+            const prev = arg[0];
+            const cur = arg[1] as string;
+            if(this.filter(prev)){
+                if(this.filter(cur)){
+                    return super.emit("change",prev,cur);
+                }
+                else{
+                    return super.emit("delete",cur);
+                }
+            }
+            else{
+                if(this.filter(cur)){
+                    return super.emit("create",cur);
+                }
+            }
+            return false;
+        }
+        else if(!this.filter(arg[0])){
+            return false;
+        }
+        else return super.emit(event,...arg);
+    }
+    constructor(refWatcher:IDiffWatcher, filter:(filename:string)=>boolean){
+        super();
+        this.refWatcher = refWatcher;
+        this.filter = filter;
+        linkWatcher(refWatcher,this);
+    }
+    setup(cntr:DocumentAccessor): Promise<void> {
+        return this.refWatcher.setup(cntr);
+    }
+}

src/server.ts

@@ -14,6 +14,7 @@ import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, Logo
 
 import {createInterface as createReadlineInterface} from 'readline';
 import { DocumentAccessor, UserAccessor } from './model/mod';
+import { createMangaWatcher } from './diff/watcher/manga_watcher';
 
 class ServerApplication{
     readonly userController: UserAccessor;
@@ -21,7 +22,7 @@ class ServerApplication{
     readonly diffManger;
     readonly app: Koa;
     private index_html:Buffer;
-    constructor(userController: UserAccessor,documentController:DocumentAccessor){
+    private constructor(userController: UserAccessor,documentController:DocumentAccessor){
         this.userController = userController;
         this.documentController = documentController;
         this.diffManger = new DiffManager(documentController);
@@ -49,7 +50,7 @@ class ServerApplication{
         app.use(createUserMiddleWare(this.userController));
         
         let diff_router = createDiffRouter(this.diffManger);
-        this.diffManger.register("manga","testdata");
+        this.diffManger.register("manga",createMangaWatcher());
         let router = new Router();
         
         router.use('/api/diff',diff_router.routes());

src/util/configRW.ts

@@ -0,0 +1,51 @@
+import {readFileSync, existsSync, writeFileSync, promises as fs} from 'fs';
+import {validate} from 'jsonschema';
+
+export class ConfigManager<T>{
+    path:string;
+    default_config: T;
+    config: T| null;
+    schema:object;
+    constructor(path:string,default_config:T,schema:object){
+        this.path = path;
+        this.default_config = default_config;
+        this.config = null;
+        this.schema = schema;
+    }
+    get_config_file(): T{
+        if(this.config !== null) return this.config;
+        this.config = {...this.read_config_file()};
+        return this.config;
+    }
+    private emptyToDefault(target:T){
+        let occur = false;
+        for(const key in this.default_config){
+            if(key === undefined || key in target){
+                continue;
+            }
+            target[key] = this.default_config[key];
+            occur = true;
+        }
+        return occur;
+    }
+    read_config_file():T{
+        if(!existsSync(this.path)){
+            writeFileSync(this.path,JSON.stringify(this.default_config));
+            return this.default_config;
+        }
+        const ret = JSON.parse(readFileSync(this.path,{encoding:"utf8"}));
+        if(this.emptyToDefault(ret)){
+            writeFileSync(this.path,JSON.stringify(ret));
+        }
+        const result = validate(ret,this.schema);
+        if(!result.valid){
+            throw new Error(result.toString());
+        }
+        return ret;
+    }
+    async write_config_file(new_config:T){
+        this.config = new_config;
+        await fs.writeFile(`${this.path}.temp`,JSON.stringify(new_config));
+        await fs.rename(`${this.path}.temp`,this.path);
+    }
+}

src/util/type_check.js

@@ -1,22 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.check_type = void 0;
-function check_type(obj, check_proto) {
-    for (const it in check_proto) {
-        let defined = check_proto[it];
-        if (defined === undefined)
-            return false;
-        defined = defined.trim();
-        if (defined.endsWith("[]")) {
-            if (!(obj[it] instanceof Array)) {
-                return false;
-            }
-        }
-        else if (defined !== typeof obj[it]) {
-            return false;
-        }
-    }
-    return true;
-}
-exports.check_type = check_type;
-;