monoid
/
ionian
1
0
Fork 0
Code Issues 9 Pull Requests Projects Releases Wiki Activity

add diff

This commit is contained in:
monoid 2021-01-15 18:43:36 +09:00
parent ec5465f2d2
commit cade73da87
32 changed files with 639 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
app.ts
View File

@ -1,5 +1,5 @@
import { app, BrowserWindow, session, dialog } from "electron";
import { get_setting } from "./src/setting";
import { get_setting } from "./src/SettingConfig";
import { create_server, start_server } from "./src/server";
import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login";
@ -14,8 +14,8 @@ if (!setting.cli) {
center: true,
useContentSize: true,
});
//await window.loadURL(`data:text/html;base64,`+Buffer.from(get_loading_html()).toString('base64'));
await wnd.loadFile('../loading.html');
await wnd.loadURL(`data:text/html;base64,`+Buffer.from(loading_html).toString('base64'));
//await wnd.loadURL('../loading.html');
await session.defaultSession.cookies.set({
url:`http://localhost:${setting.port}`,
name:accessTokenName,
@ -88,3 +88,35 @@ if (!setting.cli) {
start_server(server);
})();
}
const loading_html = `<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>`;

48
gen_conf_schema.ts Normal file
View File

@ -0,0 +1,48 @@
import { promises } from 'fs';
const { readdir, writeFile } = promises;
import {createGenerator} from 'ts-json-schema-generator';
import {dirname,join} from 'path';
async function genSchema(path:string,typename:string){
const gen = createGenerator({
path:path,
type:typename,
tsconfig:"tsconfig.json"
});
const schema = gen.createSchema(typename);
if(schema.definitions != undefined){
const definitions = schema.definitions;
const definition = definitions[typename];
if(typeof definition == "object" ){
let property = definition.properties;
if(property){
property['$schema'] = {
type:"string"
};
}
}
}
const text = JSON.stringify(schema);
await writeFile(join(dirname(path),`${typename}.schema.json`),text);
}
function capitalize(s:string){
return s.charAt(0).toUpperCase() + s.slice(1);
}
async function setToALL(path:string) {
console.log(`scan ${path}`)
const direntry = await readdir(path,{withFileTypes:true});
const works = direntry.filter(x=>x.isFile()&&x.name.endsWith("Config.ts")).map(x=>{
const name = x.name;
const m = /(.+)\.ts/.exec(name);
if(m !== null){
const typename = m[1];
return genSchema(join(path,typename),capitalize(typename));
}
})
await Promise.all(works);
const subdir = direntry.filter(x=>x.isDirectory()).map(x=>x.name);
for(const x of subdir){
await setToALL(join(path,x));
}
}
setToALL("src")

32
loading.html
View File

@ -1,32 +0,0 @@
<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>

3
migrations/initial.ts
View File

@ -14,7 +14,8 @@ export async function up(knex:Knex) {
b.string("filename",256).notNullable().comment("filename");
b.string("content_hash").nullable();
b.json("additional").nullable();
b.timestamps();
b.integer("created_at").notNullable();
b.integer("deleted_at");
b.index("content_type","content_type_index");
});
await knex.schema.createTable("tags", (b)=>{

4
package.json
View File

@ -18,7 +18,8 @@
"files": [
"build/**/*",
"node_modules/**/*",
"package.json"
"package.json",
"!node_modules/@material-ui/**/*"
],
"appId": "com.prelude.ionian.app",
"productName": "Ionian",
@ -82,6 +83,7 @@
"eslint-plugin-node": "^11.1.0",
"mini-css-extract-plugin": "^1.3.3",
"style-loader": "^2.0.0",
"ts-json-schema-generator": "^0.82.0",
"ts-node": "^9.1.1",
"typescript": "^4.1.3",
"webpack": "^5.11.0",

1
preload.ts Normal file
View File

@ -0,0 +1 @@
import {} from 'electron';

66
src/SettingConfig.schema.json Normal file
View File

@ -0,0 +1,66 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/SettingConfig",
"definitions": {
"SettingConfig": {
"type": "object",
"properties": {
"localmode": {
"type": "boolean",
"description": "if true, server will bind on '127.0.0.1' rather than '0.0.0.0'"
},
"guest": {
"type": "array",
"items": {
"$ref": "#/definitions/Permission"
},
"description": "guest permission"
},
"jwt_secretkey": {
"type": "string",
"description": "JWT secret key. if you change its value, all access tokens are invalidated."
},
"port": {
"type": "number",
"description": "the port which running server is binding on."
},
"mode": {
"type": "string",
"enum": [
"development",
"production"
]
},
"cli": {
"type": "boolean",
"description": "if true, do not show 'electron' window and show terminal only."
},
"forbid_remote_admin_login": {
"type": "boolean",
"description": "forbid to login admin from remote client. but, it do not invalidate access token. \r if you want to invalidate access token, change 'jwt_secretkey'."
},
"$schema": {
"type": "string"
}
},
"required": [
"localmode",
"guest",
"jwt_secretkey",
"port",
"mode",
"cli",
"forbid_remote_admin_login"
],
"additionalProperties": false
},
"Permission": {
"type": "string",
"enum": [
"ModifyTag",
"QueryContent",
"ModifyTagDesc"
]
}
}
}

15
src/setting.ts → src/SettingConfig.ts
View File

@ -1,9 +1,8 @@
import { Settings } from '@material-ui/icons';
import { randomBytes } from 'crypto';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { Permission } from './permission/permission';
export type Setting = {
export interface SettingConfig {
/**
* if true, server will bind on '127.0.0.1' rather than '0.0.0.0'
*/
@ -31,7 +30,7 @@ export type Setting = {
* if you want to invalidate access token, change 'jwt_secretkey'.*/
forbid_remote_admin_login:boolean,
}
const default_setting:Setting = {
const default_setting:SettingConfig = {
localmode: true,
guest:[],
@ -41,15 +40,15 @@ const default_setting:Setting = {
cli:false,
forbid_remote_admin_login:true,
}
let setting: null|Setting = null;
let setting: null|SettingConfig = null;
const setEmptyToDefault = (target:any,default_table:Setting)=>{
const setEmptyToDefault = (target:any,default_table:SettingConfig)=>{
let diff_occur = false;
for(const key in default_table){
if(key === undefined || key in target){
continue;
}
target[key] = default_table[key as keyof Setting];
target[key] = default_table[key as keyof SettingConfig];
diff_occur = true;
}
return diff_occur;
@ -61,9 +60,9 @@ export const read_setting_from_file = ()=>{
if(partial_occur){
writeFileSync("settings.json",JSON.stringify(ret));
}
return ret as Setting;
return ret as SettingConfig;
}
export function get_setting():Setting{
export function get_setting():SettingConfig{
if(setting === null){
setting = read_setting_from_file();
const env = process.env.NODE_ENV || 'development';

1
src/client/accessor/document.ts
View File

@ -44,6 +44,7 @@ export class ClientDocumentAccessor implements DocumentAccessor{
return ret;
}
async add(c: DocumentBody): Promise<number>{
throw new Error("not allow");
const res = await fetch(`${baseurl}`,{
method: "POST",
body: JSON.stringify(c)

46
src/client/page/difference.tsx
View File

@ -1,11 +1,51 @@
import React from 'react';
import React, { useContext, useEffect, useState } from 'react';
import { CommonMenuList, Headline } from "../component/mod";
import { UserContext } from "../state";
import { Grid, Typography } from "@material-ui/core";
import { Grid, Paper, Typography } from "@material-ui/core";
export function DifferencePage(){
const ctx = useContext(UserContext);
const [diffList,setDiffList] = useState<
{type:string,value:{path:string,type:string}[]}[]
>([]);
const doLoad = async ()=>{
const list = await fetch('/api/diff/list');
if(list.ok){
const inner = await list.json();
setDiffList(inner);
}
else{
//setDiffList([]);
}
};
useEffect(
()=>{
doLoad();
const i = setInterval(doLoad,5000);
return ()=>{
clearInterval(i);
}
},[]
)
const Commit = async(x:{type:string,path:string})=>{
const res = await fetch('/api/diff/commit',{
method:'POST',
body: JSON.stringify([{...x}]),
headers:{
'content-type':'application/json'
}
});
const bb = await res.json();
if(bb.ok){
doLoad();
}
}
const menu = CommonMenuList();
return (<Headline menu={menu}>
<div>Not implemented</div>
{diffList.map(x=><Paper key={x.type}>
<Typography variant='h3'>{x.type}</Typography>
{x.value.map(y=><Typography variant='h5' onClick={()=>Commit(y)}>{y.path}</Typography>)}
</Paper>)}
</Headline>)
}

46
src/content/file.ts
View File

@ -3,37 +3,65 @@ import Router from 'koa-router';
import {createHash} from 'crypto';
import {promises} from 'fs'
import {extname} from 'path';
import { DocumentBody } from '../model/mod';
import path from 'path';
/**
* content file or directory referrer
*/
export interface ContentFile{
getHash():Promise<string>;
getDesc():Promise<object|null>;
createDocumentBody():Promise<DocumentBody>;
readonly path: string;
readonly type: string;
}
type ContentFileConstructor = (new (path:string,desc?:object) => ContentFile)&{content_type:string};
export type ContentConstructOption = {
hash: string,
tags: string[],
title: string,
additional: JSONMap
}
type ContentFileConstructor = (new (path:string,option?:ContentConstructOption) => ContentFile)&{content_type:string};
export const createDefaultClass = (type:string):ContentFileConstructor=>{
let cons = class implements ContentFile{
readonly path: string;
type = type;
//type = type;
static content_type = type;
protected hash: string| undefined;
constructor(path:string,option?:object){
constructor(path:string,option?:ContentConstructOption){
this.path = path;
this.hash = option?.hash;
}
async createDocumentBody(): Promise<DocumentBody> {
const {base,dir, name} = path.parse(this.path);
const ret = {
title : name,
basepath : dir,
additional: {},
content_type: cons.content_type,
filename: base,
tags: [],
content_hash: await this.getHash(),
} as DocumentBody;
return ret;
}
get type():string{
return cons.content_type;
}
async getDesc(): Promise<object|null> {
return null;
}
async getHash():Promise<string>{
if(this.hash !== undefined) return this.hash;
const stat = await promises.stat(this.path);
const hash = createHash("sha512");
hash.update(extname(this.type));
hash.update(extname(this.path));
hash.update(stat.mode.toString());
//if(this.desc !== undefined)
// hash.update(JSON.stringify(this.desc));
hash.update(stat.size.toString());
return hash.digest("base64");
this.hash = hash.digest("base64");
return this.hash;
}
};
return cons;
@ -43,11 +71,11 @@ export function registerContentReferrer(s: ContentFileConstructor){
console.log(`registered content type: ${s.content_type}`)
ContstructorTable[s.content_type] = s;
}
export function createContentFile(type:string,path:string,option?:object){
export function createContentFile(type:string,path:string,option?:ContentConstructOption){
const constructorMethod = ContstructorTable[type];
if(constructorMethod === undefined){
console.log(type);
throw new Error("undefined");
console.log(`${type} are not in ${JSON.stringify(ContstructorTable)}`);
throw new Error("construction method of the content type is undefined");
}
return new constructorMethod(path,option);
}

7
src/content/manga.ts
View File

@ -1,10 +1,11 @@
import {ContentFile} from './file';
import {createDefaultClass,registerContentReferrer} from './file';
import {ContentFile, createDefaultClass,registerContentReferrer, ContentConstructOption} from './file';
import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap';
export class MangaReferrer extends createDefaultClass("manga"){
desc: object|null|undefined;
constructor(path:string,option?:object|undefined){
additional: object| undefined;
constructor(path:string,option?:ContentConstructOption){
super(path);
this.additional = option;
}
async getDesc(){
if(this.desc !== undefined){

4
src/content/video.ts
View File

@ -1,8 +1,8 @@
import {ContentFile, registerContentReferrer} from './file';
import {ContentFile, registerContentReferrer, ContentConstructOption} from './file';
import {createDefaultClass} from './file';
export class VideoReferrer extends createDefaultClass("video"){
constructor(path:string,desc?:object){
constructor(path:string,desc?:ContentConstructOption){
super(path,desc);
}
}

2
src/database.ts
View File

@ -1,7 +1,7 @@
import { existsSync } from 'fs';
import Knex from 'knex';
import {Knex as KnexConfig} from './config';
import { get_setting } from './setting';
import { get_setting } from './SettingConfig';
export async function connectDB(){
const config = KnexConfig.config;

30
src/db/doc.ts
View File

@ -8,7 +8,7 @@ type DBTagContentRelation = {
tag_name:string
}
class KnexContentsAccessor implements DocumentAccessor{
class KnexDocumentAccessor implements DocumentAccessor{
knex : Knex;
tagController: TagAccessor;
constructor(knex : Knex){
@ -19,6 +19,7 @@ class KnexContentsAccessor implements DocumentAccessor{
const {tags,additional, ...rest} = c;
const id_lst = await this.knex.insert({
additional:JSON.stringify(additional),
created_at:Date.now(),
...rest
}).into('document');
const id = id_lst[0];
@ -53,9 +54,20 @@ class KnexContentsAccessor implements DocumentAccessor{
return {
...first,
tags:ret_tags,
additional: JSON.parse(first.additional || "{}"),
additional: first.additional !== null ? JSON.parse(first.additional) : {},
};
};
async findDeleted(content_type:string){
const s = await this.knex.select("*")
.where({content_type:content_type})
.whereNotNull("update_at")
.from("document");
return s.map(x=>({
...x,
tags:[],
additional:{}
}));
}
async findList(option?:QueryListOption){
option = option || {};
const allow_tag = option.allow_tag || [];
@ -94,6 +106,7 @@ class KnexContentsAccessor implements DocumentAccessor{
}
}
query = query.limit(limit);
query = query.orderBy('id',"desc");
return query;
}
let query = buildquery();
@ -119,13 +132,14 @@ class KnexContentsAccessor implements DocumentAccessor{
}
return result;
};
async findListByBasePath(path:string):Promise<Document[]>{
let results = await this.knex.select("*").from("document").where({basepath:path});
async findByPath(path:string,filename?:string):Promise<Document[]>{
const e = filename == undefined ? {} : {filename:filename}
const results = await this.knex.select("*").from("document").where({basepath:path,...e});
return results.map(x=>({
...x,
tags:[],
additional:JSON.parse(x.additional || "{}"),
}));
additional:{}
}))
}
async update(c:Partial<Document> & { id:number }){
const {id,tags,...rest} = c;
@ -150,6 +164,6 @@ class KnexContentsAccessor implements DocumentAccessor{
return true;
}
}
export const createKnexContentsAccessor = (knex:Knex): DocumentAccessor=>{
return new KnexContentsAccessor(knex);
export const createKnexDocumentAccessor = (knex:Knex): DocumentAccessor=>{
return new KnexDocumentAccessor(knex);
}

1
src/diff/MangaConfig.schema.json Normal file
View File

@ -0,0 +1 @@
{"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/MangaConfig","definitions":{"MangaConfig":{"type":"object","properties":{"watch":{"type":"array","items":{"type":"string"}},"$schema":{"type":"string"}},"required":["watch"],"additionalProperties":false}}}

6
src/diff/MangaConfig.ts Normal file
View File

@ -0,0 +1,6 @@
import Schema from './MangaConfig.schema.json';
export interface MangaConfig{
watch:string[]
}

74
src/diff/content_handler.ts Normal file
View File

@ -0,0 +1,74 @@
import {join as pathjoin} from 'path';
import {Document, DocumentAccessor} from '../model/mod';
import { ContentFile, createContentFile } from '../content/mod';
import {IDiffWatcher} from './watcher';
import {ContentList} from './content_list';
//refactoring needed.
export class ContentDiffHandler{
waiting_list:ContentList;
tombstone: Map<string,Document>;//hash, contentfile
doc_cntr: DocumentAccessor;
content_type: string;
constructor(cntr: DocumentAccessor,content_type:string){
this.waiting_list = new ContentList();
this.tombstone = new Map<string,Document>();
this.doc_cntr = cntr;
this.content_type = content_type;
}
async setup(){
const deleted = await this.doc_cntr.findDeleted(this.content_type);
for (const it of deleted) {
this.tombstone.set(it.content_hash,it);
}
}
register(diff:IDiffWatcher){
diff.on('create',(filename)=>this.OnCreated(diff.path,filename))
.on('delete',(filename)=>this.OnDeleted(diff.path,filename))
.on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename));
}
private async OnDeleted(basepath:string,filename:string){
const cpath = pathjoin(basepath,filename);
if(this.waiting_list.hasPath(cpath)){
this.waiting_list.deletePath(cpath);
return;
}
const dbc = await this.doc_cntr.findByPath(basepath,filename);
if(dbc.length === 0) return; //ignore
if(this.waiting_list.hasHash(dbc[0].content_hash)){
//if path changed, update changed path.
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
//db invalidate
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: Date.now(),
});
this.tombstone.set(dbc[0].content_hash, dbc[0]);
}
private async OnCreated(basepath:string,filename:string){
const content = createContentFile(this.content_type,pathjoin(basepath,filename));
const hash = await content.getHash();
const c = this.tombstone.get(hash);
if(c !== undefined){
this.doc_cntr.update({
id: c.id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
this.waiting_list.set(content);
}
private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){
const doc = await this.doc_cntr.findByPath(basepath,prev_filename);
await this.doc_cntr.update({...doc[0],filename:cur_filename});
}
}

62
src/diff/content_list.ts Normal file
View File

@ -0,0 +1,62 @@
import { ContentFile } from '../content/mod';
import event from 'events';
interface ContentListEvent{
'set':(c:ContentFile)=>void,
'delete':(c:ContentFile)=>void,
}
export class ContentList extends event.EventEmitter{
cl:Map<string,ContentFile>;
hl:Map<string,ContentFile>;
on<U extends keyof ContentListEvent>(event:U,listener:ContentListEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof ContentListEvent>(event:U,...arg:Parameters<ContentListEvent[U]>): boolean{
return super.emit(event,...arg);
}
constructor(){
super();
this.cl = new Map;
this.hl = new Map;
}
hasHash(s:string){
return this.hl.has(s);
}
hasPath(p:string){
return this.cl.has(p);
}
getHash(s:string){
return this.hl.get(s)
}
getPath(p:string){
return this.cl.get(p);
}
async set(c:ContentFile){
const path = c.path;
const hash = await c.getHash();
this.cl.set(path,c);
this.hl.set(hash,c);
this.emit('set',c);
}
async delete(c:ContentFile){
let r = true;
r &&= this.cl.delete(c.path);
r &&= this.hl.delete(await c.getHash());
this.emit('delete',c);
return r;
}
async deletePath(p:string){
const o = this.getPath(p);
if(o === undefined) return false;
return this.delete(o);
}
async deleteHash(s:string){
const o = this.getHash(s);
if(o === undefined) return false;
return this.delete(o);
}
getAll(){
return [...this.cl.values()];
}
}

113
src/diff/diff.ts
View File

@ -1,82 +1,39 @@
import { watch } from 'fs';
import { promises } from 'fs';
import { ContentFile, createContentReferrer, getContentRefererConstructor } from '../content/referrer'
import path from 'path';
const readdir = promises.readdir;
export class Watcher{
private _type: string;
private _path:string;
/**
* @todo : alter type Map<string,ContentReferrer>
*/
private _added: ContentFile[];
private _deleted: ContentFile[];
constructor(path:string,type:string){
this._path = path;
this._added =[];
this._deleted =[];
this._type = type;
import { DocumentAccessor } from '../model/doc';
import {ContentDiffHandler} from './content_handler';
import { CommonDiffWatcher } from './watcher';
//import {join as pathjoin} from 'path';
export class DiffManager{
watching: {[content_type:string]:ContentDiffHandler};
doc_cntr: DocumentAccessor;
constructor(contorller: DocumentAccessor){
this.watching = {};
this.doc_cntr = contorller;
}
public get added() : ContentFile[] {
return this._added;
async register(content_type:string,path:string){
if(this.watching[content_type] === undefined){
this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type);
}
const watcher = new CommonDiffWatcher(path);
this.watching[content_type].register(watcher);
const initial_doc = await this.doc_cntr.findByPath(path);
await watcher.setup(initial_doc.map(x=>x.filename));
watcher.watch();
}
/*public set added(diff : FileDiff[]) {
this._added = diff;
}*/
public get deleted(): ContentFile[]{
return this._deleted;
async commit(type:string,path:string){
const list = this.watching[type].waiting_list;
const c = list.getPath(path);
if(c===undefined){
throw new Error("path is not exist");
}
await list.delete(c);
const body = await c.createDocumentBody();
const id = await this.doc_cntr.add(body);
return id;
}
/*public set deleted(diff : FileDiff[]){
this._deleted = diff;
}*/
public get path(){
return this._path;
getAdded(){
return Object.keys(this.watching).map(x=>({
type:x,
value:this.watching[x].waiting_list.getAll(),
}));
}
public get type(){
return this._type;
}
private createCR(filename: string){
return createContentReferrer(`${this.path}/${filename}`,this.type);
}
/**
*
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
this._added = added.map(x=>this.createCR(x));
this._deleted = deleted.map(x=>this.createCR(x));
watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this._added.push(this.createCR(filename));
}
else{
//added has one
if(this._added.map(x=>x.path).includes(path.join(this.path,filename))){
this._added = this._added.filter(x=> x.path !== path.join(this.path,filename));
}
else {
this._deleted.push(this.createCR(filename));
}
}
}
});
}
}
export class DiffWatcher{
Watchers: {[basepath:string]:Watcher} = {};
}
};

2
src/diff/mod.ts Normal file
View File

@ -0,0 +1,2 @@
export * from './router';
export * from './diff';

61
src/diff/router.ts Normal file
View File

@ -0,0 +1,61 @@
import Koa from 'koa';
import Router from 'koa-router';
import { ContentFile } from '../content/mod';
import { sendError } from '../route/error_handler';
import {DiffManager} from './diff';
function content_file_to_return(x:ContentFile){
return {path:x.path,type:x.type};
}
export const getAdded = (diffmgr:DiffManager)=> (ctx:Koa.Context,next:Koa.Next)=>{
const ret = diffmgr.getAdded();
ctx.body = ret.map(x=>({
type:x.type,
value:x.value.map(x=>({path:x.path,type:x.type})),
}));
ctx.type = 'json';
}
type PostAddedBody = {
type:string,
path:string,
}[];
function checkPostAddedBody(body: any): body is PostAddedBody{
if(body instanceof Array){
return body.map(x=> 'type' in x && 'path' in x).every(x=>x);
}
return false;
}
export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next)=>{
const reqbody = ctx.request.body;
console.log(reqbody);
if(!checkPostAddedBody(reqbody)){
sendError(400,"format exception");
return;
}
const allWork = reqbody.map(op=>diffmgr.commit(op.type,op.path));
const results = await Promise.all(allWork);
ctx.body = {
ok:true,
docs:results,
}
ctx.type = 'json';
}
/*
export const getNotWatched = (diffmgr : DiffManager)=> (ctx:Router.IRouterContext,next:Koa.Next)=>{
ctx.body = {
added: diffmgr.added.map(content_file_to_return),
deleted: diffmgr.deleted.map(content_file_to_return),
};
ctx.type = 'json';
}*/
export function createDiffRouter(diffmgr: DiffManager){
const ret = new Router();
ret.get("/list",getAdded(diffmgr));
ret.post("/commit",postAdded(diffmgr));
return ret;
}

78
src/diff/watcher.ts Normal file
View File

@ -0,0 +1,78 @@
import { FSWatcher, watch } from 'fs';
import { promises } from 'fs';
import event from 'events';
const readdir = promises.readdir;
interface DiffWatcherEvent{
'create':(filename:string)=>void,
'delete':(filename:string)=>void,
'change':(prev_filename:string,cur_filename:string)=>void,
}
export interface IDiffWatcher extends event.EventEmitter {
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this;
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean;
readonly path: string;
}
export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
private _path:string;
private _watcher: FSWatcher|null;
constructor(path:string){
super();
this._path = path;
this._watcher = null;
}
public get path(){
return this._path;
}
/**
* setup
* @argument initial_filenames filename in path
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//Todo : reduce O(nm) to O(n+m) using hash map.
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
for (const iterator of added) {
this.emit('create',iterator);
}
for (const iterator of deleted){
this.emit('delete',iterator);
}
}
watch():FSWatcher{
this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this.emit('create',filename);
}
else{
this.emit('delete',filename)
}
}
});
return this._watcher;
}
watchClose(){
this._watcher?.close()
}
}

2
src/login.ts
View File

@ -5,7 +5,7 @@ import { sendError } from "./route/error_handler";
import Knex from "knex";
import { createKnexUserController } from "./db/mod";
import { request } from "http";
import { get_setting } from "./setting";
import { get_setting } from "./SettingConfig";
import { IUser, UserAccessor } from "./model/mod";
type PayloadInfo = {

18
src/model/doc.ts
View File

@ -1,17 +1,12 @@
import {TagAccessor} from './tag';
import {check_type} from '../util/type_check'
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;
export interface DocumentBody{
title : string,
content_type : string,
basepath : string,
filename : string,
content_hash? : string,
content_hash : string,
additional : JSONMap,
tags : string[],//eager loading
}
@ -32,6 +27,8 @@ export const isDocBody = (c : any):c is DocumentBody =>{
export interface Document extends DocumentBody{
readonly id: number;
readonly created_at:number;
readonly deleted_at:number|null;
};
export const isDoc = (c: any):c is Document =>{
@ -88,9 +85,14 @@ export interface DocumentAccessor{
*/
findById: (id:number,tagload?:boolean)=> Promise<Document| undefined>,
/**
*
* find by base path and filename.
* if you call this function with filename, its return array length is 0 or 1.
*/
findListByBasePath:(basepath: string)=>Promise<Document[]>;
findByPath:(basepath: string,filename?:string)=>Promise<Document[]>;
/**
* find deleted content
*/
findDeleted:(content_type:string)=>Promise<Document[]>;
/**
* update document except tag.
*/

6
src/route/contents.ts
View File

@ -71,7 +71,7 @@ const UpdateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
ctx.body = JSON.stringify(success);
ctx.type = 'json';
}
const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => {
/*const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => {
const content_desc = ctx.request.body;
if(!isDocBody(content_desc)){
return sendError(400,"it is not a valid format");
@ -79,7 +79,7 @@ const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
const id = await controller.add(content_desc);
ctx.body = JSON.stringify(id);
ctx.type = 'json';
};
};*/
const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{
let tag_name = ctx.params['tag'];
const num = Number.parseInt(ctx.params['num']);
@ -137,7 +137,7 @@ export const getContentRouter = (controller: DocumentAccessor)=>{
ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller));
ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller));
//ret.use("/:num(\\d+)/:content_type");
ret.post("/",AdminOnly,CreateContentHandler(controller));
//ret.post("/",AdminOnly,CreateContentHandler(controller));
ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller));
ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller));
ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller));

2
src/route/context.ts
View File

@ -1,5 +1,3 @@
import {ContentReferrer} from '../content/mod';
export type ContentLocation = {
path:string,
type:string,

20
src/server.ts
View File

@ -1,25 +1,30 @@
import Koa from 'koa';
import Router from 'koa-router';
import {get_setting} from './setting';
import {get_setting} from './SettingConfig';
import {connectDB} from './database';
import {Watcher} from './diff/diff'
import {DiffManager, createDiffRouter} from './diff/mod';
import { createReadStream, readFileSync } from 'fs';
import getContentRouter from './route/contents';
import { createKnexContentsAccessor } from './db/doc';
import { createKnexDocumentAccessor } from './db/mod';
import bodyparser from 'koa-bodyparser';
import {error_handler} from './route/error_handler';
import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login';
import {createInterface as createReadlineInterface} from 'readline';
//let Koa = require("koa");
export async function create_server(){
let setting = get_setting();
const setting = get_setting();
let db = await connectDB();
let diffmgr = new DiffManager(createKnexDocumentAccessor(db));
let diff_router = createDiffRouter(diffmgr);
diffmgr.register("manga","testdata");
if(setting.cli){
const userAdmin = await getAdmin(db);
if(await isAdminFirst(userAdmin)){
@ -38,9 +43,12 @@ export async function create_server(){
app.use(createUserMiddleWare(db));
//app.use(ctx=>{ctx.state['setting'] = settings});
const index_html = readFileSync("index.html");
let router = new Router();
router.use('/api/diff',diff_router.routes());
router.use('/api/diff',diff_router.allowedMethods());
//let watcher = new Watcher(setting.path[0]);
//await watcher.setup([]);
@ -63,7 +71,7 @@ export async function create_server(){
if(setting.mode === "development")
static_file_server('dist/js/bundle.js.map','text');
const content_router = getContentRouter(createKnexContentsAccessor(db));
const content_router = getContentRouter(createKnexDocumentAccessor(db));
router.use('/api/doc',content_router.routes());
router.use('/api/doc',content_router.allowedMethods());

6
src/types/db.d.ts vendored
View File

@ -17,8 +17,10 @@ declare module "knex" {
content_type: string;
basepath: string;
filename: string;
content_hash?: string;
additional?: string;
created_at: number;
deleted_at: number|null;
content_hash: string;
additional: string|null;
};
doc_tag_relation: {
doc_id: number;

5
src/types/json.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;

10
test.ts Normal file
View File

@ -0,0 +1,10 @@
import Knex from 'knex';
import {connectDB} from './src/database';
async function main() {
const db = await connectDB();
const query = db.update({deleted_at: null}).from('document');
console.log(query.toSQL());
}
main()

1
tsconfig.json
View File

@ -49,6 +49,7 @@
// "types": [], /* Type declaration files to be included in compilation. */
"allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
"resolveJsonModule": true,
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */