monoid
/
ionian
1
0
Fork 0
Code Issues 9 Pull Requests Projects Releases Wiki Activity

event driven watcher

This commit is contained in:
monoid 2021-01-22 15:13:05 +09:00
parent b4e0e51588
commit 89bc827a7a
17 changed files with 330 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -10,4 +10,5 @@ package-lock.json
devdb.sqlite3
build/**
app/**
settings.json
settings.json
*config.json

12
package.json
View File

@ -31,14 +31,16 @@
"author": "",
"license": "ISC",
"dependencies": {
"chokidar": "^3.5.1",
"jsonschema": "^1.4.0",
"jsonwebtoken": "^8.5.1",
"knex": "^0.21.14",
"koa": "^2.13.0",
"knex": "^0.21.16",
"koa": "^2.13.1",
"koa-bodyparser": "^4.3.0",
"koa-router": "^10.0.0",
"natural-orderby": "^2.0.3",
"node-stream-zip": "^1.12.0",
"sqlite3": "^5.0.0"
"sqlite3": "^5.0.1"
},
"devDependencies": {
"@types/jsonwebtoken": "^8.5.0",
@ -46,8 +48,8 @@
"@types/koa": "^2.11.6",
"@types/koa-bodyparser": "^4.3.0",
"@types/koa-router": "^7.4.1",
"@types/node": "^14.14.16",
"electron": "^11.1.1",
"@types/node": "^14.14.22",
"electron": "^11.2.0",
"electron-builder": "^22.9.1",
"eslint-plugin-node": "^11.1.0",
"ts-json-schema-generator": "^0.82.0",

6
src/diff/MangaConfig.ts
View File

@ -1,6 +0,0 @@
import Schema from './MangaConfig.schema.json';
export interface MangaConfig{
watch:string[]
}

29
src/diff/content_handler.ts
View File

@ -1,4 +1,4 @@
import {join as pathjoin} from 'path';
import {basename, dirname, join as pathjoin} from 'path';
import {Document, DocumentAccessor} from '../model/mod';
import { ContentFile, createContentFile } from '../content/mod';
import {IDiffWatcher} from './watcher';
@ -23,12 +23,13 @@ export class ContentDiffHandler{
}
}
register(diff:IDiffWatcher){
diff.on('create',(filename)=>this.OnCreated(diff.path,filename))
.on('delete',(filename)=>this.OnDeleted(diff.path,filename))
.on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename));
diff.on('create',(path)=>this.OnCreated(path))
.on('delete',(path)=>this.OnDeleted(path))
.on('change',(prev,cur)=>this.OnChanged(prev,cur));
}
private async OnDeleted(basepath:string,filename:string){
const cpath = pathjoin(basepath,filename);
private async OnDeleted(cpath: string){
const basepath = dirname(cpath);
const filename = basename(cpath);
if(this.waiting_list.hasPath(cpath)){
this.waiting_list.deletePath(cpath);
return;
@ -52,7 +53,9 @@ export class ContentDiffHandler{
});
this.tombstone.set(dbc[0].content_hash, dbc[0]);
}
private async OnCreated(basepath:string,filename:string){
private async OnCreated(cpath:string){
const basepath = dirname(cpath);
const filename = basename(cpath);
const content = createContentFile(this.content_type,pathjoin(basepath,filename));
const hash = await content.getHash();
const c = this.tombstone.get(hash);
@ -67,8 +70,14 @@ export class ContentDiffHandler{
}
this.waiting_list.set(content);
}
private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){
const doc = await this.doc_cntr.findByPath(basepath,prev_filename);
await this.doc_cntr.update({...doc[0],filename:cur_filename});
private async OnChanged(prev_path:string,cur_path:string){
const prev_basepath = dirname(prev_path);
const prev_filename = basename(prev_path);
const cur_basepath = dirname(cur_path);
const cur_filename = basename(cur_path);
const doc = await this.doc_cntr.findByPath(prev_basepath,prev_filename);
await this.doc_cntr.update({...doc[0],
basepath:cur_basepath,
filename:cur_filename});
}
}

10
src/diff/diff.ts
View File

@ -1,6 +1,7 @@
import { DocumentAccessor } from '../model/doc';
import {ContentDiffHandler} from './content_handler';
import { CommonDiffWatcher } from './watcher';
import { IDiffWatcher } from './watcher';
//import {join as pathjoin} from 'path';
export class DiffManager{
watching: {[content_type:string]:ContentDiffHandler};
@ -9,15 +10,12 @@ export class DiffManager{
this.watching = {};
this.doc_cntr = contorller;
}
async register(content_type:string,path:string){
async register(content_type:string,watcher:IDiffWatcher){
if(this.watching[content_type] === undefined){
this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type);
}
const watcher = new CommonDiffWatcher(path);
this.watching[content_type].register(watcher);
const initial_doc = await this.doc_cntr.findByPath(path);
await watcher.setup(initial_doc.map(x=>x.filename));
watcher.watch();
await watcher.setup(this.doc_cntr);
}
async commit(type:string,path:string){
const list = this.watching[type].waiting_list;

75
src/diff/watcher.ts
View File

@ -1,78 +1,25 @@
import { FSWatcher, watch } from 'fs';
import { promises } from 'fs';
import event from 'events';
import { join } from 'path';
import { DocumentAccessor } from '../model/doc';
const readdir = promises.readdir;
interface DiffWatcherEvent{
'create':(filename:string)=>void,
'delete':(filename:string)=>void,
'change':(prev_filename:string,cur_filename:string)=>void,
export interface DiffWatcherEvent{
'create':(path:string)=>void,
'delete':(path:string)=>void,
'change':(prev_path:string,cur_path:string)=>void,
}
export interface IDiffWatcher extends event.EventEmitter {
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this;
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean;
readonly path: string;
setup(cntr:DocumentAccessor):Promise<void>;
}
export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
private _path:string;
private _watcher: FSWatcher|null;
constructor(path:string){
super();
this._path = path;
this._watcher = null;
}
public get path(){
return this._path;
}
/**
* setup
* @argument initial_filenames filename in path
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//Todo : reduce O(nm) to O(n+m) using hash map.
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
for (const iterator of added) {
this.emit('create',iterator);
}
for (const iterator of deleted){
this.emit('delete',iterator);
}
}
watch():FSWatcher{
this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this.emit('create',filename);
}
else{
this.emit('delete',filename)
}
}
});
return this._watcher;
}
watchClose(){
this._watcher?.close()
}
export function linkWatcher(fromWatcher :IDiffWatcher, toWatcher: IDiffWatcher){
fromWatcher.on("create",p=>toWatcher.emit("create",p));
fromWatcher.on("delete",p=>toWatcher.emit("delete",p));
fromWatcher.on("change",(p,c)=>toWatcher.emit("change",p,c));
}

0
src/diff/MangaConfig.schema.json → src/diff/watcher/MangaConfig.schema.json
View File

8
src/diff/watcher/MangaConfig.ts Normal file
View File

@ -0,0 +1,8 @@
import {ConfigManager} from '../../util/configRW';
import MangaSchema from "./MangaConfig.schema.json"
export interface MangaConfig{
watch:string[]
}
export const MangaConfig = new ConfigManager<MangaConfig>("manga_config.json",{watch:[]},MangaSchema);

45
src/diff/watcher/common_watcher.ts Normal file
View File

@ -0,0 +1,45 @@
import event from 'events';
import {FSWatcher,watch,promises} from 'fs';
import {IDiffWatcher, DiffWatcherEvent} from '../watcher';
import {join} from 'path';
import { DocumentAccessor } from '../../model/doc';
import { setupHelp } from './util';
const {readdir} = promises;
export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
private _path:string;
private _watcher: FSWatcher;
constructor(path:string){
super();
this._path = path;
this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = await readdir(this._path);
//add
if(cur.includes(filename)){
this.emit('create',join(this.path,filename));
}
else{
this.emit('delete',join(this.path,filename))
}
}
});
}
async setup(cntr: DocumentAccessor): Promise<void> {
await setupHelp(this,this.path,cntr);
}
public get path(){
return this._path;
}
watchClose(){
this._watcher.close()
}
}

24
src/diff/watcher/compositer.ts Normal file
View File

@ -0,0 +1,24 @@
import { EventEmitter } from "events";
import { DocumentAccessor } from "../../model/doc";
import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher";
export class WatcherCompositer extends EventEmitter implements IDiffWatcher{
refWatchers : IDiffWatcher[];
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
constructor(refWatchers:IDiffWatcher[]){
super();
this.refWatchers = refWatchers;
for(const refWatcher of this.refWatchers){
linkWatcher(refWatcher,this);
}
}
async setup(cntr: DocumentAccessor): Promise<void> {
await Promise.all(this.refWatchers.map(x=>x.setup(cntr)));
}
}

17
src/diff/watcher/manga_watcher.ts Normal file
View File

@ -0,0 +1,17 @@
import {IDiffWatcher, DiffWatcherEvent} from '../watcher';
import {EventEmitter} from 'events';
import { DocumentAccessor } from '../../model/doc';
import { WatcherFilter } from './watcher_filter';
import { RecursiveWatcher } from './recursive_watcher';
import { MangaConfig } from './MangaConfig';
import {WatcherCompositer} from './compositer'
const createMangaWatcherBase = (path:string)=> {
return new WatcherFilter(new RecursiveWatcher(path),(x)=>x.endsWith(".zip"));
}
export const createMangaWatcher = ()=>{
const file = MangaConfig.get_config_file();
console.log(`register manga ${file.watch.join(",")}`)
return new WatcherCompositer(file.watch.map(path=>createMangaWatcherBase(path)));
}

59
src/diff/watcher/recursive_watcher.ts Normal file
View File

@ -0,0 +1,59 @@
import {watch, FSWatcher} from 'chokidar';
import { EventEmitter } from 'events';
import { join } from 'path';
import { DocumentAccessor } from '../../model/doc';
import { DiffWatcherEvent, IDiffWatcher } from '../watcher';
import { setupHelp, setupRecursive } from './util';
type RecursiveWatcherOption={
/** @default true */
watchFile?:boolean,
/** @default false */
watchDir?:boolean,
}
export class RecursiveWatcher extends EventEmitter implements IDiffWatcher {
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
readonly path: string;
private watcher: FSWatcher
constructor(path:string, option?:RecursiveWatcherOption){
super();
this.path = path;
option = option || {
watchDir:false,
watchFile:true,
}
this.watcher = watch(path,{
persistent:true,
ignoreInitial:true,
depth:100,
});
if(option.watchFile === undefined || option.watchFile){
this.watcher.on("add",path=>{
const cpath = join(this.path,path);
this.emit("create",cpath);
}).on("unlink",path=>{
const cpath = join(this.path,path);
this.emit("delete",cpath);
});
}
if(option.watchDir){
this.watcher.on("addDir",path=>{
const cpath = join(this.path,path);
this.emit("create",cpath);
}).on("unlinkDir",path=>{
const cpath = join(this.path,path);
this.emit("delete",cpath);
})
}
}
async setup(cntr: DocumentAccessor): Promise<void> {
await setupRecursive(this,this.path,cntr);
}
}

35
src/diff/watcher/util.ts Normal file
View File

@ -0,0 +1,35 @@
import { EventEmitter } from "events";
import { promises } from "fs";
import { join } from "path";
const {readdir} = promises;
import { DocumentAccessor } from "../../model/doc";
import { IDiffWatcher } from "../watcher";
function setupCommon(watcher:IDiffWatcher,basepath:string,initial_filenames:string[],cur:string[]){
//Todo : reduce O(nm) to O(n+m) using hash map.
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
for (const it of added) {
const cpath = join(basepath,it);
watcher.emit('create',cpath);
}
for (const it of deleted){
const cpath = join(basepath,it);
watcher.emit('delete',cpath);
}
}
export async function setupHelp(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){
const initial_document = await cntr.findByPath(basepath);
const initial_filenames = initial_document.map(x=>x.filename);
const cur = await readdir(basepath);
setupCommon(watcher,basepath,initial_filenames,cur);
}
export async function setupRecursive(watcher:IDiffWatcher,basepath:string,cntr:DocumentAccessor){
const initial_document = await cntr.findByPath(basepath);
const initial_filenames = initial_document.map(x=>x.filename);
const cur = await readdir(basepath,{withFileTypes:true});
setupCommon(watcher,basepath,initial_filenames,cur.map(x=>x.name));
await Promise.all([cur.filter(x=>x.isDirectory())
.map(x=>setupHelp(watcher,join(basepath,x.name),cntr))]);
}

45
src/diff/watcher/watcher_filter.ts Normal file
View File

@ -0,0 +1,45 @@
import { EventEmitter } from "events";
import { DocumentAccessor } from "../../model/doc";
import { DiffWatcherEvent, IDiffWatcher, linkWatcher } from "../watcher";
export class WatcherFilter extends EventEmitter implements IDiffWatcher{
refWatcher : IDiffWatcher;
filter : (filename:string)=>boolean;;
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
if(event === "change"){
const prev = arg[0];
const cur = arg[1] as string;
if(this.filter(prev)){
if(this.filter(cur)){
return super.emit("change",prev,cur);
}
else{
return super.emit("delete",cur);
}
}
else{
if(this.filter(cur)){
return super.emit("create",cur);
}
}
return false;
}
else if(!this.filter(arg[0])){
return false;
}
else return super.emit(event,...arg);
}
constructor(refWatcher:IDiffWatcher, filter:(filename:string)=>boolean){
super();
this.refWatcher = refWatcher;
this.filter = filter;
linkWatcher(refWatcher,this);
}
setup(cntr:DocumentAccessor): Promise<void> {
return this.refWatcher.setup(cntr);
}
}

5
src/server.ts
View File

@ -14,6 +14,7 @@ import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, Logo
import {createInterface as createReadlineInterface} from 'readline';
import { DocumentAccessor, UserAccessor } from './model/mod';
import { createMangaWatcher } from './diff/watcher/manga_watcher';
class ServerApplication{
readonly userController: UserAccessor;
@ -21,7 +22,7 @@ class ServerApplication{
readonly diffManger;
readonly app: Koa;
private index_html:Buffer;
constructor(userController: UserAccessor,documentController:DocumentAccessor){
private constructor(userController: UserAccessor,documentController:DocumentAccessor){
this.userController = userController;
this.documentController = documentController;
this.diffManger = new DiffManager(documentController);
@ -49,7 +50,7 @@ class ServerApplication{
app.use(createUserMiddleWare(this.userController));
let diff_router = createDiffRouter(this.diffManger);
this.diffManger.register("manga","testdata");
this.diffManger.register("manga",createMangaWatcher());
let router = new Router();
router.use('/api/diff',diff_router.routes());

51
src/util/configRW.ts Normal file
View File

@ -0,0 +1,51 @@
import {readFileSync, existsSync, writeFileSync, promises as fs} from 'fs';
import {validate} from 'jsonschema';
export class ConfigManager<T>{
path:string;
default_config: T;
config: T| null;
schema:object;
constructor(path:string,default_config:T,schema:object){
this.path = path;
this.default_config = default_config;
this.config = null;
this.schema = schema;
}
get_config_file(): T{
if(this.config !== null) return this.config;
this.config = {...this.read_config_file()};
return this.config;
}
private emptyToDefault(target:T){
let occur = false;
for(const key in this.default_config){
if(key === undefined || key in target){
continue;
}
target[key] = this.default_config[key];
occur = true;
}
return occur;
}
read_config_file():T{
if(!existsSync(this.path)){
writeFileSync(this.path,JSON.stringify(this.default_config));
return this.default_config;
}
const ret = JSON.parse(readFileSync(this.path,{encoding:"utf8"}));
if(this.emptyToDefault(ret)){
writeFileSync(this.path,JSON.stringify(ret));
}
const result = validate(ret,this.schema);
if(!result.valid){
throw new Error(result.toString());
}
return ret;
}
async write_config_file(new_config:T){
this.config = new_config;
await fs.writeFile(`${this.path}.temp`,JSON.stringify(new_config));
await fs.rename(`${this.path}.temp`,this.path);
}
}

22
src/util/type_check.js
View File

@ -1,22 +0,0 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.check_type = void 0;
function check_type(obj, check_proto) {
for (const it in check_proto) {
let defined = check_proto[it];
if (defined === undefined)
return false;
defined = defined.trim();
if (defined.endsWith("[]")) {
if (!(obj[it] instanceof Array)) {
return false;
}
}
else if (defined !== typeof obj[it]) {
return false;
}
}
return true;
}
exports.check_type = check_type;
;