This commit is contained in:
monoid 2021-01-15 18:43:36 +09:00
parent ec5465f2d2
commit cade73da87
32 changed files with 639 additions and 171 deletions

38
app.ts
View File

@ -1,5 +1,5 @@
import { app, BrowserWindow, session, dialog } from "electron"; import { app, BrowserWindow, session, dialog } from "electron";
import { get_setting } from "./src/setting"; import { get_setting } from "./src/SettingConfig";
import { create_server, start_server } from "./src/server"; import { create_server, start_server } from "./src/server";
import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login"; import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, refreshTokenName } from "./src/login";
@ -14,8 +14,8 @@ if (!setting.cli) {
center: true, center: true,
useContentSize: true, useContentSize: true,
}); });
//await window.loadURL(`data:text/html;base64,`+Buffer.from(get_loading_html()).toString('base64')); await wnd.loadURL(`data:text/html;base64,`+Buffer.from(loading_html).toString('base64'));
await wnd.loadFile('../loading.html'); //await wnd.loadURL('../loading.html');
await session.defaultSession.cookies.set({ await session.defaultSession.cookies.set({
url:`http://localhost:${setting.port}`, url:`http://localhost:${setting.port}`,
name:accessTokenName, name:accessTokenName,
@ -88,3 +88,35 @@ if (!setting.cli) {
start_server(server); start_server(server);
})(); })();
} }
const loading_html = `<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>`;

48
gen_conf_schema.ts Normal file
View File

@ -0,0 +1,48 @@
import { promises } from 'fs';
const { readdir, writeFile } = promises;
import {createGenerator} from 'ts-json-schema-generator';
import {dirname,join} from 'path';
async function genSchema(path:string,typename:string){
const gen = createGenerator({
path:path,
type:typename,
tsconfig:"tsconfig.json"
});
const schema = gen.createSchema(typename);
if(schema.definitions != undefined){
const definitions = schema.definitions;
const definition = definitions[typename];
if(typeof definition == "object" ){
let property = definition.properties;
if(property){
property['$schema'] = {
type:"string"
};
}
}
}
const text = JSON.stringify(schema);
await writeFile(join(dirname(path),`${typename}.schema.json`),text);
}
function capitalize(s:string){
return s.charAt(0).toUpperCase() + s.slice(1);
}
async function setToALL(path:string) {
console.log(`scan ${path}`)
const direntry = await readdir(path,{withFileTypes:true});
const works = direntry.filter(x=>x.isFile()&&x.name.endsWith("Config.ts")).map(x=>{
const name = x.name;
const m = /(.+)\.ts/.exec(name);
if(m !== null){
const typename = m[1];
return genSchema(join(path,typename),capitalize(typename));
}
})
await Promise.all(works);
const subdir = direntry.filter(x=>x.isDirectory()).map(x=>x.name);
for(const x of subdir){
await setToALL(join(path,x));
}
}
setToALL("src")

View File

@ -1,32 +0,0 @@
<!DOCTYPE html>
<html lang="ko"><head>
<meta charset="UTF-8">
<title>loading</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'
fonts.googleapis.com; font-src 'self' fonts.gstatic.com">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<style>
body { margin-top: 100px; background-color: #3f51b5; color: #fff; text-align:center; }
h1 {
font: 2em 'Roboto', sans-serif;
margin-bottom: 40px;
}
#loading {
display: inline-block;
width: 50px;
height: 50px;
border: 3px solid rgba(255,255,255,.3);
border-radius: 50%;
border-top-color: #fff;
animation: spin 1s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg);}
}
</style>
<body>
<h1>Loading...</h1>
<div id="loading"></div>
</body>
</html>

View File

@ -14,7 +14,8 @@ export async function up(knex:Knex) {
b.string("filename",256).notNullable().comment("filename"); b.string("filename",256).notNullable().comment("filename");
b.string("content_hash").nullable(); b.string("content_hash").nullable();
b.json("additional").nullable(); b.json("additional").nullable();
b.timestamps(); b.integer("created_at").notNullable();
b.integer("deleted_at");
b.index("content_type","content_type_index"); b.index("content_type","content_type_index");
}); });
await knex.schema.createTable("tags", (b)=>{ await knex.schema.createTable("tags", (b)=>{

View File

@ -18,7 +18,8 @@
"files": [ "files": [
"build/**/*", "build/**/*",
"node_modules/**/*", "node_modules/**/*",
"package.json" "package.json",
"!node_modules/@material-ui/**/*"
], ],
"appId": "com.prelude.ionian.app", "appId": "com.prelude.ionian.app",
"productName": "Ionian", "productName": "Ionian",
@ -82,6 +83,7 @@
"eslint-plugin-node": "^11.1.0", "eslint-plugin-node": "^11.1.0",
"mini-css-extract-plugin": "^1.3.3", "mini-css-extract-plugin": "^1.3.3",
"style-loader": "^2.0.0", "style-loader": "^2.0.0",
"ts-json-schema-generator": "^0.82.0",
"ts-node": "^9.1.1", "ts-node": "^9.1.1",
"typescript": "^4.1.3", "typescript": "^4.1.3",
"webpack": "^5.11.0", "webpack": "^5.11.0",

1
preload.ts Normal file
View File

@ -0,0 +1 @@
import {} from 'electron';

View File

@ -0,0 +1,66 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$ref": "#/definitions/SettingConfig",
"definitions": {
"SettingConfig": {
"type": "object",
"properties": {
"localmode": {
"type": "boolean",
"description": "if true, server will bind on '127.0.0.1' rather than '0.0.0.0'"
},
"guest": {
"type": "array",
"items": {
"$ref": "#/definitions/Permission"
},
"description": "guest permission"
},
"jwt_secretkey": {
"type": "string",
"description": "JWT secret key. if you change its value, all access tokens are invalidated."
},
"port": {
"type": "number",
"description": "the port which running server is binding on."
},
"mode": {
"type": "string",
"enum": [
"development",
"production"
]
},
"cli": {
"type": "boolean",
"description": "if true, do not show 'electron' window and show terminal only."
},
"forbid_remote_admin_login": {
"type": "boolean",
"description": "forbid to login admin from remote client. but, it do not invalidate access token. \r if you want to invalidate access token, change 'jwt_secretkey'."
},
"$schema": {
"type": "string"
}
},
"required": [
"localmode",
"guest",
"jwt_secretkey",
"port",
"mode",
"cli",
"forbid_remote_admin_login"
],
"additionalProperties": false
},
"Permission": {
"type": "string",
"enum": [
"ModifyTag",
"QueryContent",
"ModifyTagDesc"
]
}
}
}

View File

@ -1,9 +1,8 @@
import { Settings } from '@material-ui/icons';
import { randomBytes } from 'crypto'; import { randomBytes } from 'crypto';
import { existsSync, readFileSync, writeFileSync } from 'fs'; import { existsSync, readFileSync, writeFileSync } from 'fs';
import { Permission } from './permission/permission'; import { Permission } from './permission/permission';
export type Setting = { export interface SettingConfig {
/** /**
* if true, server will bind on '127.0.0.1' rather than '0.0.0.0' * if true, server will bind on '127.0.0.1' rather than '0.0.0.0'
*/ */
@ -31,7 +30,7 @@ export type Setting = {
* if you want to invalidate access token, change 'jwt_secretkey'.*/ * if you want to invalidate access token, change 'jwt_secretkey'.*/
forbid_remote_admin_login:boolean, forbid_remote_admin_login:boolean,
} }
const default_setting:Setting = { const default_setting:SettingConfig = {
localmode: true, localmode: true,
guest:[], guest:[],
@ -41,15 +40,15 @@ const default_setting:Setting = {
cli:false, cli:false,
forbid_remote_admin_login:true, forbid_remote_admin_login:true,
} }
let setting: null|Setting = null; let setting: null|SettingConfig = null;
const setEmptyToDefault = (target:any,default_table:Setting)=>{ const setEmptyToDefault = (target:any,default_table:SettingConfig)=>{
let diff_occur = false; let diff_occur = false;
for(const key in default_table){ for(const key in default_table){
if(key === undefined || key in target){ if(key === undefined || key in target){
continue; continue;
} }
target[key] = default_table[key as keyof Setting]; target[key] = default_table[key as keyof SettingConfig];
diff_occur = true; diff_occur = true;
} }
return diff_occur; return diff_occur;
@ -61,9 +60,9 @@ export const read_setting_from_file = ()=>{
if(partial_occur){ if(partial_occur){
writeFileSync("settings.json",JSON.stringify(ret)); writeFileSync("settings.json",JSON.stringify(ret));
} }
return ret as Setting; return ret as SettingConfig;
} }
export function get_setting():Setting{ export function get_setting():SettingConfig{
if(setting === null){ if(setting === null){
setting = read_setting_from_file(); setting = read_setting_from_file();
const env = process.env.NODE_ENV || 'development'; const env = process.env.NODE_ENV || 'development';

View File

@ -44,6 +44,7 @@ export class ClientDocumentAccessor implements DocumentAccessor{
return ret; return ret;
} }
async add(c: DocumentBody): Promise<number>{ async add(c: DocumentBody): Promise<number>{
throw new Error("not allow");
const res = await fetch(`${baseurl}`,{ const res = await fetch(`${baseurl}`,{
method: "POST", method: "POST",
body: JSON.stringify(c) body: JSON.stringify(c)

View File

@ -1,11 +1,51 @@
import React from 'react'; import React, { useContext, useEffect, useState } from 'react';
import { CommonMenuList, Headline } from "../component/mod"; import { CommonMenuList, Headline } from "../component/mod";
import { UserContext } from "../state"; import { UserContext } from "../state";
import { Grid, Typography } from "@material-ui/core"; import { Grid, Paper, Typography } from "@material-ui/core";
export function DifferencePage(){ export function DifferencePage(){
const ctx = useContext(UserContext);
const [diffList,setDiffList] = useState<
{type:string,value:{path:string,type:string}[]}[]
>([]);
const doLoad = async ()=>{
const list = await fetch('/api/diff/list');
if(list.ok){
const inner = await list.json();
setDiffList(inner);
}
else{
//setDiffList([]);
}
};
useEffect(
()=>{
doLoad();
const i = setInterval(doLoad,5000);
return ()=>{
clearInterval(i);
}
},[]
)
const Commit = async(x:{type:string,path:string})=>{
const res = await fetch('/api/diff/commit',{
method:'POST',
body: JSON.stringify([{...x}]),
headers:{
'content-type':'application/json'
}
});
const bb = await res.json();
if(bb.ok){
doLoad();
}
}
const menu = CommonMenuList(); const menu = CommonMenuList();
return (<Headline menu={menu}> return (<Headline menu={menu}>
<div>Not implemented</div> {diffList.map(x=><Paper key={x.type}>
<Typography variant='h3'>{x.type}</Typography>
{x.value.map(y=><Typography variant='h5' onClick={()=>Commit(y)}>{y.path}</Typography>)}
</Paper>)}
</Headline>) </Headline>)
} }

View File

@ -3,37 +3,65 @@ import Router from 'koa-router';
import {createHash} from 'crypto'; import {createHash} from 'crypto';
import {promises} from 'fs' import {promises} from 'fs'
import {extname} from 'path'; import {extname} from 'path';
import { DocumentBody } from '../model/mod';
import path from 'path';
/** /**
* content file or directory referrer * content file or directory referrer
*/ */
export interface ContentFile{ export interface ContentFile{
getHash():Promise<string>; getHash():Promise<string>;
getDesc():Promise<object|null>; createDocumentBody():Promise<DocumentBody>;
readonly path: string; readonly path: string;
readonly type: string; readonly type: string;
} }
type ContentFileConstructor = (new (path:string,desc?:object) => ContentFile)&{content_type:string}; export type ContentConstructOption = {
hash: string,
tags: string[],
title: string,
additional: JSONMap
}
type ContentFileConstructor = (new (path:string,option?:ContentConstructOption) => ContentFile)&{content_type:string};
export const createDefaultClass = (type:string):ContentFileConstructor=>{ export const createDefaultClass = (type:string):ContentFileConstructor=>{
let cons = class implements ContentFile{ let cons = class implements ContentFile{
readonly path: string; readonly path: string;
type = type; //type = type;
static content_type = type; static content_type = type;
protected hash: string| undefined;
constructor(path:string,option?:object){ constructor(path:string,option?:ContentConstructOption){
this.path = path; this.path = path;
this.hash = option?.hash;
}
async createDocumentBody(): Promise<DocumentBody> {
const {base,dir, name} = path.parse(this.path);
const ret = {
title : name,
basepath : dir,
additional: {},
content_type: cons.content_type,
filename: base,
tags: [],
content_hash: await this.getHash(),
} as DocumentBody;
return ret;
}
get type():string{
return cons.content_type;
} }
async getDesc(): Promise<object|null> { async getDesc(): Promise<object|null> {
return null; return null;
} }
async getHash():Promise<string>{ async getHash():Promise<string>{
if(this.hash !== undefined) return this.hash;
const stat = await promises.stat(this.path); const stat = await promises.stat(this.path);
const hash = createHash("sha512"); const hash = createHash("sha512");
hash.update(extname(this.type)); hash.update(extname(this.path));
hash.update(stat.mode.toString()); hash.update(stat.mode.toString());
//if(this.desc !== undefined) //if(this.desc !== undefined)
// hash.update(JSON.stringify(this.desc)); // hash.update(JSON.stringify(this.desc));
hash.update(stat.size.toString()); hash.update(stat.size.toString());
return hash.digest("base64"); this.hash = hash.digest("base64");
return this.hash;
} }
}; };
return cons; return cons;
@ -43,11 +71,11 @@ export function registerContentReferrer(s: ContentFileConstructor){
console.log(`registered content type: ${s.content_type}`) console.log(`registered content type: ${s.content_type}`)
ContstructorTable[s.content_type] = s; ContstructorTable[s.content_type] = s;
} }
export function createContentFile(type:string,path:string,option?:object){ export function createContentFile(type:string,path:string,option?:ContentConstructOption){
const constructorMethod = ContstructorTable[type]; const constructorMethod = ContstructorTable[type];
if(constructorMethod === undefined){ if(constructorMethod === undefined){
console.log(type); console.log(`${type} are not in ${JSON.stringify(ContstructorTable)}`);
throw new Error("undefined"); throw new Error("construction method of the content type is undefined");
} }
return new constructorMethod(path,option); return new constructorMethod(path,option);
} }

View File

@ -1,10 +1,11 @@
import {ContentFile} from './file'; import {ContentFile, createDefaultClass,registerContentReferrer, ContentConstructOption} from './file';
import {createDefaultClass,registerContentReferrer} from './file';
import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap'; import {readZip,createReadStreamFromZip, readAllFromZip} from '../util/zipwrap';
export class MangaReferrer extends createDefaultClass("manga"){ export class MangaReferrer extends createDefaultClass("manga"){
desc: object|null|undefined; desc: object|null|undefined;
constructor(path:string,option?:object|undefined){ additional: object| undefined;
constructor(path:string,option?:ContentConstructOption){
super(path); super(path);
this.additional = option;
} }
async getDesc(){ async getDesc(){
if(this.desc !== undefined){ if(this.desc !== undefined){

View File

@ -1,8 +1,8 @@
import {ContentFile, registerContentReferrer} from './file'; import {ContentFile, registerContentReferrer, ContentConstructOption} from './file';
import {createDefaultClass} from './file'; import {createDefaultClass} from './file';
export class VideoReferrer extends createDefaultClass("video"){ export class VideoReferrer extends createDefaultClass("video"){
constructor(path:string,desc?:object){ constructor(path:string,desc?:ContentConstructOption){
super(path,desc); super(path,desc);
} }
} }

View File

@ -1,7 +1,7 @@
import { existsSync } from 'fs'; import { existsSync } from 'fs';
import Knex from 'knex'; import Knex from 'knex';
import {Knex as KnexConfig} from './config'; import {Knex as KnexConfig} from './config';
import { get_setting } from './setting'; import { get_setting } from './SettingConfig';
export async function connectDB(){ export async function connectDB(){
const config = KnexConfig.config; const config = KnexConfig.config;

View File

@ -8,7 +8,7 @@ type DBTagContentRelation = {
tag_name:string tag_name:string
} }
class KnexContentsAccessor implements DocumentAccessor{ class KnexDocumentAccessor implements DocumentAccessor{
knex : Knex; knex : Knex;
tagController: TagAccessor; tagController: TagAccessor;
constructor(knex : Knex){ constructor(knex : Knex){
@ -19,6 +19,7 @@ class KnexContentsAccessor implements DocumentAccessor{
const {tags,additional, ...rest} = c; const {tags,additional, ...rest} = c;
const id_lst = await this.knex.insert({ const id_lst = await this.knex.insert({
additional:JSON.stringify(additional), additional:JSON.stringify(additional),
created_at:Date.now(),
...rest ...rest
}).into('document'); }).into('document');
const id = id_lst[0]; const id = id_lst[0];
@ -53,9 +54,20 @@ class KnexContentsAccessor implements DocumentAccessor{
return { return {
...first, ...first,
tags:ret_tags, tags:ret_tags,
additional: JSON.parse(first.additional || "{}"), additional: first.additional !== null ? JSON.parse(first.additional) : {},
}; };
}; };
async findDeleted(content_type:string){
const s = await this.knex.select("*")
.where({content_type:content_type})
.whereNotNull("update_at")
.from("document");
return s.map(x=>({
...x,
tags:[],
additional:{}
}));
}
async findList(option?:QueryListOption){ async findList(option?:QueryListOption){
option = option || {}; option = option || {};
const allow_tag = option.allow_tag || []; const allow_tag = option.allow_tag || [];
@ -94,6 +106,7 @@ class KnexContentsAccessor implements DocumentAccessor{
} }
} }
query = query.limit(limit); query = query.limit(limit);
query = query.orderBy('id',"desc");
return query; return query;
} }
let query = buildquery(); let query = buildquery();
@ -119,13 +132,14 @@ class KnexContentsAccessor implements DocumentAccessor{
} }
return result; return result;
}; };
async findListByBasePath(path:string):Promise<Document[]>{ async findByPath(path:string,filename?:string):Promise<Document[]>{
let results = await this.knex.select("*").from("document").where({basepath:path}); const e = filename == undefined ? {} : {filename:filename}
const results = await this.knex.select("*").from("document").where({basepath:path,...e});
return results.map(x=>({ return results.map(x=>({
...x, ...x,
tags:[], tags:[],
additional:JSON.parse(x.additional || "{}"), additional:{}
})); }))
} }
async update(c:Partial<Document> & { id:number }){ async update(c:Partial<Document> & { id:number }){
const {id,tags,...rest} = c; const {id,tags,...rest} = c;
@ -150,6 +164,6 @@ class KnexContentsAccessor implements DocumentAccessor{
return true; return true;
} }
} }
export const createKnexContentsAccessor = (knex:Knex): DocumentAccessor=>{ export const createKnexDocumentAccessor = (knex:Knex): DocumentAccessor=>{
return new KnexContentsAccessor(knex); return new KnexDocumentAccessor(knex);
} }

View File

@ -0,0 +1 @@
{"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/MangaConfig","definitions":{"MangaConfig":{"type":"object","properties":{"watch":{"type":"array","items":{"type":"string"}},"$schema":{"type":"string"}},"required":["watch"],"additionalProperties":false}}}

6
src/diff/MangaConfig.ts Normal file
View File

@ -0,0 +1,6 @@
import Schema from './MangaConfig.schema.json';
export interface MangaConfig{
watch:string[]
}

View File

@ -0,0 +1,74 @@
import {join as pathjoin} from 'path';
import {Document, DocumentAccessor} from '../model/mod';
import { ContentFile, createContentFile } from '../content/mod';
import {IDiffWatcher} from './watcher';
import {ContentList} from './content_list';
//refactoring needed.
export class ContentDiffHandler{
waiting_list:ContentList;
tombstone: Map<string,Document>;//hash, contentfile
doc_cntr: DocumentAccessor;
content_type: string;
constructor(cntr: DocumentAccessor,content_type:string){
this.waiting_list = new ContentList();
this.tombstone = new Map<string,Document>();
this.doc_cntr = cntr;
this.content_type = content_type;
}
async setup(){
const deleted = await this.doc_cntr.findDeleted(this.content_type);
for (const it of deleted) {
this.tombstone.set(it.content_hash,it);
}
}
register(diff:IDiffWatcher){
diff.on('create',(filename)=>this.OnCreated(diff.path,filename))
.on('delete',(filename)=>this.OnDeleted(diff.path,filename))
.on('change',(prev_filename,cur_filename)=>this.OnChanged(diff.path,prev_filename,cur_filename));
}
private async OnDeleted(basepath:string,filename:string){
const cpath = pathjoin(basepath,filename);
if(this.waiting_list.hasPath(cpath)){
this.waiting_list.deletePath(cpath);
return;
}
const dbc = await this.doc_cntr.findByPath(basepath,filename);
if(dbc.length === 0) return; //ignore
if(this.waiting_list.hasHash(dbc[0].content_hash)){
//if path changed, update changed path.
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
//db invalidate
await this.doc_cntr.update({
id:dbc[0].id,
deleted_at: Date.now(),
});
this.tombstone.set(dbc[0].content_hash, dbc[0]);
}
private async OnCreated(basepath:string,filename:string){
const content = createContentFile(this.content_type,pathjoin(basepath,filename));
const hash = await content.getHash();
const c = this.tombstone.get(hash);
if(c !== undefined){
this.doc_cntr.update({
id: c.id,
deleted_at: null,
filename:filename,
basepath:basepath
});
return;
}
this.waiting_list.set(content);
}
private async OnChanged(basepath:string,prev_filename:string,cur_filename:string){
const doc = await this.doc_cntr.findByPath(basepath,prev_filename);
await this.doc_cntr.update({...doc[0],filename:cur_filename});
}
}

62
src/diff/content_list.ts Normal file
View File

@ -0,0 +1,62 @@
import { ContentFile } from '../content/mod';
import event from 'events';
interface ContentListEvent{
'set':(c:ContentFile)=>void,
'delete':(c:ContentFile)=>void,
}
export class ContentList extends event.EventEmitter{
cl:Map<string,ContentFile>;
hl:Map<string,ContentFile>;
on<U extends keyof ContentListEvent>(event:U,listener:ContentListEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof ContentListEvent>(event:U,...arg:Parameters<ContentListEvent[U]>): boolean{
return super.emit(event,...arg);
}
constructor(){
super();
this.cl = new Map;
this.hl = new Map;
}
hasHash(s:string){
return this.hl.has(s);
}
hasPath(p:string){
return this.cl.has(p);
}
getHash(s:string){
return this.hl.get(s)
}
getPath(p:string){
return this.cl.get(p);
}
async set(c:ContentFile){
const path = c.path;
const hash = await c.getHash();
this.cl.set(path,c);
this.hl.set(hash,c);
this.emit('set',c);
}
async delete(c:ContentFile){
let r = true;
r &&= this.cl.delete(c.path);
r &&= this.hl.delete(await c.getHash());
this.emit('delete',c);
return r;
}
async deletePath(p:string){
const o = this.getPath(p);
if(o === undefined) return false;
return this.delete(o);
}
async deleteHash(s:string){
const o = this.getHash(s);
if(o === undefined) return false;
return this.delete(o);
}
getAll(){
return [...this.cl.values()];
}
}

View File

@ -1,82 +1,39 @@
import { watch } from 'fs'; import { DocumentAccessor } from '../model/doc';
import { promises } from 'fs'; import {ContentDiffHandler} from './content_handler';
import { ContentFile, createContentReferrer, getContentRefererConstructor } from '../content/referrer' import { CommonDiffWatcher } from './watcher';
import path from 'path'; //import {join as pathjoin} from 'path';
export class DiffManager{
const readdir = promises.readdir; watching: {[content_type:string]:ContentDiffHandler};
doc_cntr: DocumentAccessor;
constructor(contorller: DocumentAccessor){
export class Watcher{ this.watching = {};
private _type: string; this.doc_cntr = contorller;
private _path:string;
/**
* @todo : alter type Map<string,ContentReferrer>
*/
private _added: ContentFile[];
private _deleted: ContentFile[];
constructor(path:string,type:string){
this._path = path;
this._added =[];
this._deleted =[];
this._type = type;
} }
public get added() : ContentFile[] { async register(content_type:string,path:string){
return this._added; if(this.watching[content_type] === undefined){
this.watching[content_type] = new ContentDiffHandler(this.doc_cntr,content_type);
} }
/*public set added(diff : FileDiff[]) { const watcher = new CommonDiffWatcher(path);
this._added = diff; this.watching[content_type].register(watcher);
}*/ const initial_doc = await this.doc_cntr.findByPath(path);
public get deleted(): ContentFile[]{ await watcher.setup(initial_doc.map(x=>x.filename));
return this._deleted; watcher.watch();
} }
/*public set deleted(diff : FileDiff[]){ async commit(type:string,path:string){
this._deleted = diff; const list = this.watching[type].waiting_list;
}*/ const c = list.getPath(path);
public get path(){ if(c===undefined){
return this._path; throw new Error("path is not exist");
} }
public get type(){ await list.delete(c);
return this._type; const body = await c.createDocumentBody();
const id = await this.doc_cntr.add(body);
return id;
} }
private createCR(filename: string){ getAdded(){
return createContentReferrer(`${this.path}/${filename}`,this.type); return Object.keys(this.watching).map(x=>({
} type:x,
/** value:this.watching[x].waiting_list.getAll(),
* }));
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
this._added = added.map(x=>this.createCR(x));
this._deleted = deleted.map(x=>this.createCR(x));
watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this._added.push(this.createCR(filename));
}
else{
//added has one
if(this._added.map(x=>x.path).includes(path.join(this.path,filename))){
this._added = this._added.filter(x=> x.path !== path.join(this.path,filename));
}
else {
this._deleted.push(this.createCR(filename));
}
}
}
});
}
}
export class DiffWatcher{
Watchers: {[basepath:string]:Watcher} = {};
} }
};

2
src/diff/mod.ts Normal file
View File

@ -0,0 +1,2 @@
export * from './router';
export * from './diff';

61
src/diff/router.ts Normal file
View File

@ -0,0 +1,61 @@
import Koa from 'koa';
import Router from 'koa-router';
import { ContentFile } from '../content/mod';
import { sendError } from '../route/error_handler';
import {DiffManager} from './diff';
function content_file_to_return(x:ContentFile){
return {path:x.path,type:x.type};
}
export const getAdded = (diffmgr:DiffManager)=> (ctx:Koa.Context,next:Koa.Next)=>{
const ret = diffmgr.getAdded();
ctx.body = ret.map(x=>({
type:x.type,
value:x.value.map(x=>({path:x.path,type:x.type})),
}));
ctx.type = 'json';
}
type PostAddedBody = {
type:string,
path:string,
}[];
function checkPostAddedBody(body: any): body is PostAddedBody{
if(body instanceof Array){
return body.map(x=> 'type' in x && 'path' in x).every(x=>x);
}
return false;
}
export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next)=>{
const reqbody = ctx.request.body;
console.log(reqbody);
if(!checkPostAddedBody(reqbody)){
sendError(400,"format exception");
return;
}
const allWork = reqbody.map(op=>diffmgr.commit(op.type,op.path));
const results = await Promise.all(allWork);
ctx.body = {
ok:true,
docs:results,
}
ctx.type = 'json';
}
/*
export const getNotWatched = (diffmgr : DiffManager)=> (ctx:Router.IRouterContext,next:Koa.Next)=>{
ctx.body = {
added: diffmgr.added.map(content_file_to_return),
deleted: diffmgr.deleted.map(content_file_to_return),
};
ctx.type = 'json';
}*/
export function createDiffRouter(diffmgr: DiffManager){
const ret = new Router();
ret.get("/list",getAdded(diffmgr));
ret.post("/commit",postAdded(diffmgr));
return ret;
}

78
src/diff/watcher.ts Normal file
View File

@ -0,0 +1,78 @@
import { FSWatcher, watch } from 'fs';
import { promises } from 'fs';
import event from 'events';
const readdir = promises.readdir;
interface DiffWatcherEvent{
'create':(filename:string)=>void,
'delete':(filename:string)=>void,
'change':(prev_filename:string,cur_filename:string)=>void,
}
export interface IDiffWatcher extends event.EventEmitter {
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this;
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean;
readonly path: string;
}
export class CommonDiffWatcher extends event.EventEmitter implements IDiffWatcher{
on<U extends keyof DiffWatcherEvent>(event:U,listener:DiffWatcherEvent[U]): this{
return super.on(event,listener);
}
emit<U extends keyof DiffWatcherEvent>(event:U,...arg:Parameters<DiffWatcherEvent[U]>): boolean{
return super.emit(event,...arg);
}
private _path:string;
private _watcher: FSWatcher|null;
constructor(path:string){
super();
this._path = path;
this._watcher = null;
}
public get path(){
return this._path;
}
/**
* setup
* @argument initial_filenames filename in path
*/
async setup(initial_filenames:string[]){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//Todo : reduce O(nm) to O(n+m) using hash map.
let added = cur.filter(x => !initial_filenames.includes(x));
let deleted = initial_filenames.filter(x=>!cur.includes(x));
for (const iterator of added) {
this.emit('create',iterator);
}
for (const iterator of deleted){
this.emit('delete',iterator);
}
}
watch():FSWatcher{
this._watcher = watch(this._path,{persistent: true, recursive:false},async (eventType,filename)=>{
if(eventType === "rename"){
const cur = (await readdir(this._path,{
encoding:"utf8",
withFileTypes: true,
})).filter(x=>x.isFile).map(x=>x.name);
//add
if(cur.includes(filename)){
this.emit('create',filename);
}
else{
this.emit('delete',filename)
}
}
});
return this._watcher;
}
watchClose(){
this._watcher?.close()
}
}

View File

@ -5,7 +5,7 @@ import { sendError } from "./route/error_handler";
import Knex from "knex"; import Knex from "knex";
import { createKnexUserController } from "./db/mod"; import { createKnexUserController } from "./db/mod";
import { request } from "http"; import { request } from "http";
import { get_setting } from "./setting"; import { get_setting } from "./SettingConfig";
import { IUser, UserAccessor } from "./model/mod"; import { IUser, UserAccessor } from "./model/mod";
type PayloadInfo = { type PayloadInfo = {

View File

@ -1,17 +1,12 @@
import {TagAccessor} from './tag'; import {TagAccessor} from './tag';
import {check_type} from '../util/type_check' import {check_type} from '../util/type_check'
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;
export interface DocumentBody{ export interface DocumentBody{
title : string, title : string,
content_type : string, content_type : string,
basepath : string, basepath : string,
filename : string, filename : string,
content_hash? : string, content_hash : string,
additional : JSONMap, additional : JSONMap,
tags : string[],//eager loading tags : string[],//eager loading
} }
@ -32,6 +27,8 @@ export const isDocBody = (c : any):c is DocumentBody =>{
export interface Document extends DocumentBody{ export interface Document extends DocumentBody{
readonly id: number; readonly id: number;
readonly created_at:number;
readonly deleted_at:number|null;
}; };
export const isDoc = (c: any):c is Document =>{ export const isDoc = (c: any):c is Document =>{
@ -88,9 +85,14 @@ export interface DocumentAccessor{
*/ */
findById: (id:number,tagload?:boolean)=> Promise<Document| undefined>, findById: (id:number,tagload?:boolean)=> Promise<Document| undefined>,
/** /**
* * find by base path and filename.
* if you call this function with filename, its return array length is 0 or 1.
*/ */
findListByBasePath:(basepath: string)=>Promise<Document[]>; findByPath:(basepath: string,filename?:string)=>Promise<Document[]>;
/**
* find deleted content
*/
findDeleted:(content_type:string)=>Promise<Document[]>;
/** /**
* update document except tag. * update document except tag.
*/ */

View File

@ -71,7 +71,7 @@ const UpdateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
ctx.body = JSON.stringify(success); ctx.body = JSON.stringify(success);
ctx.type = 'json'; ctx.type = 'json';
} }
const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => { /*const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => {
const content_desc = ctx.request.body; const content_desc = ctx.request.body;
if(!isDocBody(content_desc)){ if(!isDocBody(content_desc)){
return sendError(400,"it is not a valid format"); return sendError(400,"it is not a valid format");
@ -79,7 +79,7 @@ const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
const id = await controller.add(content_desc); const id = await controller.add(content_desc);
ctx.body = JSON.stringify(id); ctx.body = JSON.stringify(id);
ctx.type = 'json'; ctx.type = 'json';
}; };*/
const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{ const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{
let tag_name = ctx.params['tag']; let tag_name = ctx.params['tag'];
const num = Number.parseInt(ctx.params['num']); const num = Number.parseInt(ctx.params['num']);
@ -137,7 +137,7 @@ export const getContentRouter = (controller: DocumentAccessor)=>{
ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller)); ret.get("/:num(\\d+)",PerCheck(Per.QueryContent),ContentIDHandler(controller));
ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller)); ret.post("/:num(\\d+)",AdminOnly,UpdateContentHandler(controller));
//ret.use("/:num(\\d+)/:content_type"); //ret.use("/:num(\\d+)/:content_type");
ret.post("/",AdminOnly,CreateContentHandler(controller)); //ret.post("/",AdminOnly,CreateContentHandler(controller));
ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller)); ret.get("/:num(\\d+)/tags",PerCheck(Per.QueryContent),ContentTagIDHandler(controller));
ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller)); ret.post("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),AddTagHandler(controller));
ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller)); ret.del("/:num(\\d+)/tags/:tag",PerCheck(Per.ModifyTag),DelTagHandler(controller));

View File

@ -1,5 +1,3 @@
import {ContentReferrer} from '../content/mod';
export type ContentLocation = { export type ContentLocation = {
path:string, path:string,
type:string, type:string,

View File

@ -1,25 +1,30 @@
import Koa from 'koa'; import Koa from 'koa';
import Router from 'koa-router'; import Router from 'koa-router';
import {get_setting} from './setting'; import {get_setting} from './SettingConfig';
import {connectDB} from './database'; import {connectDB} from './database';
import {Watcher} from './diff/diff' import {DiffManager, createDiffRouter} from './diff/mod';
import { createReadStream, readFileSync } from 'fs'; import { createReadStream, readFileSync } from 'fs';
import getContentRouter from './route/contents'; import getContentRouter from './route/contents';
import { createKnexContentsAccessor } from './db/doc'; import { createKnexDocumentAccessor } from './db/mod';
import bodyparser from 'koa-bodyparser'; import bodyparser from 'koa-bodyparser';
import {error_handler} from './route/error_handler'; import {error_handler} from './route/error_handler';
import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login'; import {createUserMiddleWare,createLoginMiddleware, isAdminFirst, getAdmin, LogoutMiddleware, createRefreshTokenMiddleware} from './login';
import {createInterface as createReadlineInterface} from 'readline'; import {createInterface as createReadlineInterface} from 'readline';
//let Koa = require("koa"); //let Koa = require("koa");
export async function create_server(){ export async function create_server(){
let setting = get_setting(); const setting = get_setting();
let db = await connectDB(); let db = await connectDB();
let diffmgr = new DiffManager(createKnexDocumentAccessor(db));
let diff_router = createDiffRouter(diffmgr);
diffmgr.register("manga","testdata");
if(setting.cli){ if(setting.cli){
const userAdmin = await getAdmin(db); const userAdmin = await getAdmin(db);
if(await isAdminFirst(userAdmin)){ if(await isAdminFirst(userAdmin)){
@ -38,9 +43,12 @@ export async function create_server(){
app.use(createUserMiddleWare(db)); app.use(createUserMiddleWare(db));
//app.use(ctx=>{ctx.state['setting'] = settings}); //app.use(ctx=>{ctx.state['setting'] = settings});
const index_html = readFileSync("index.html"); const index_html = readFileSync("index.html");
let router = new Router(); let router = new Router();
router.use('/api/diff',diff_router.routes());
router.use('/api/diff',diff_router.allowedMethods());
//let watcher = new Watcher(setting.path[0]); //let watcher = new Watcher(setting.path[0]);
//await watcher.setup([]); //await watcher.setup([]);
@ -63,7 +71,7 @@ export async function create_server(){
if(setting.mode === "development") if(setting.mode === "development")
static_file_server('dist/js/bundle.js.map','text'); static_file_server('dist/js/bundle.js.map','text');
const content_router = getContentRouter(createKnexContentsAccessor(db)); const content_router = getContentRouter(createKnexDocumentAccessor(db));
router.use('/api/doc',content_router.routes()); router.use('/api/doc',content_router.routes());
router.use('/api/doc',content_router.allowedMethods()); router.use('/api/doc',content_router.allowedMethods());

6
src/types/db.d.ts vendored
View File

@ -17,8 +17,10 @@ declare module "knex" {
content_type: string; content_type: string;
basepath: string; basepath: string;
filename: string; filename: string;
content_hash?: string; created_at: number;
additional?: string; deleted_at: number|null;
content_hash: string;
additional: string|null;
}; };
doc_tag_relation: { doc_tag_relation: {
doc_id: number; doc_id: number;

5
src/types/json.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
type JSONPrimitive = null|boolean|number|string;
interface JSONMap extends Record<string, JSONType>{}
interface JSONArray extends Array<JSONType>{};
type JSONType = JSONMap|JSONPrimitive|JSONArray;

10
test.ts Normal file
View File

@ -0,0 +1,10 @@
import Knex from 'knex';
import {connectDB} from './src/database';
async function main() {
const db = await connectDB();
const query = db.update({deleted_at: null}).from('document');
console.log(query.toSQL());
}
main()

View File

@ -49,6 +49,7 @@
// "types": [], /* Type declaration files to be included in compilation. */ // "types": [], /* Type declaration files to be included in compilation. */
"allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
"resolveJsonModule": true,
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */