add commitall api and button

app.ts

@@ -5,6 +5,7 @@ import { getAdminAccessTokenValue,getAdminRefreshTokenValue, accessTokenName, re
 import { join } from "path";
 import { ipcMain } from 'electron';
 import { UserAccessor } from "./src/model/mod";
+
 function registerChannel(cntr: UserAccessor){
   ipcMain.handle('reset_password', async(event,username:string,password:string)=>{
     const user = await cntr.findUser(username);

migrations/initial.ts

@@ -1,4 +1,4 @@
-import Knex from 'knex';
+import {Knex} from 'knex';
 
 export async function up(knex:Knex) {
     await knex.schema.createTable("users",(b)=>{

package.json

@@ -6,11 +6,11 @@
   "scripts": {
     "compile": "tsc",
     "compile:watch": "tsc -w",
-    "build":"cd src/client && npm run build:prod",
+    "build": "cd src/client && npm run build:prod",
-    "build:watch":"cd src/client && npm run build:watch",
+    "build:watch": "cd src/client && npm run build:watch",
     "app": "electron build/app.js",
-    "app:build":"electron-builder",
+    "app:build": "electron-builder",
-    "app:pack":"electron-builder --dir",
+    "app:pack": "electron-builder --dir",
     "app:build:win64": "electron-builder --win --x64",
     "app:pack:win64": "electron-builder --win --x64 --dir"
   },
@@ -21,11 +21,14 @@
       "node_modules/**/*",
       "package.json"
     ],
-    "extraFiles":[
+    "extraFiles": [
       {
-        "from":"dist/",
+        "from": "dist/",
-        "to":"dist/",
+        "to": "dist/",
-        "filter":["**/*","!**/*.map"]
+        "filter": [
+          "**/*",
+          "!**/*.map"
+        ]
       },
       "index.html"
     ],
@@ -49,24 +52,26 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@louislam/sqlite3": "^6.0.0",
     "chokidar": "^3.5.1",
     "jsonschema": "^1.4.0",
     "jsonwebtoken": "^8.5.1",
-    "knex": "^0.21.16",
+    "knex": "^0.95.11",
     "koa": "^2.13.1",
     "koa-bodyparser": "^4.3.0",
     "koa-router": "^10.0.0",
     "natural-orderby": "^2.0.3",
     "node-stream-zip": "^1.12.0",
-    "sqlite3": "^5.0.1"
+    "sqlite3": "^5.0.2",
+    "tiny-async-pool": "^1.2.0"
   },
   "devDependencies": {
     "@types/jsonwebtoken": "^8.5.0",
-    "@types/knex": "^0.16.1",
     "@types/koa": "^2.11.6",
     "@types/koa-bodyparser": "^4.3.0",
     "@types/koa-router": "^7.4.1",
     "@types/node": "^14.14.22",
+    "@types/tiny-async-pool": "^1.0.0",
     "electron": "^11.2.0",
     "electron-builder": "^22.9.1",
     "eslint-plugin-node": "^11.1.0",

src/client/package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "@material-ui/core": "^4.11.2",
     "@material-ui/icons": "^4.11.2",
+    "@mui/material": "^5.0.3",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "react-router-dom": "^5.2.0"

src/client/page/difference.tsx

@@ -2,6 +2,7 @@ import React, { useContext, useEffect, useState } from 'react';
 import { CommonMenuList, Headline } from "../component/mod";
 import { UserContext } from "../state";
 import { Box, Grid, Paper, Typography,Button, makeStyles, Theme } from "@material-ui/core";
+import {Stack} from '@mui/material';
 
 const useStyles = makeStyles((theme:Theme)=>({
     paper:{
@@ -26,17 +27,25 @@ type FileDifference = {
 
 function TypeDifference(prop:{
     content:FileDifference,
-    onCommit:(v:{type:string,path:string})=>void
+    onCommit:(v:{type:string,path:string})=>void,
+    onCommitAll:(type:string) => void
 }){
     const classes = useStyles();
     const x = prop.content;
     const [button_disable,set_disable] = useState(false);
 
     return (<Paper className={classes.paper}>
-                    <Typography variant='h3' className={classes.contentTitle}>{x.type}</Typography>
+                    <Box className={classes.contentTitle}>
+                    <Typography variant='h3' >{x.type}</Typography>
+                    <Button variant="contained" key={x.type} onClick={()=>{
+                        set_disable(true);
+                        prop.onCommitAll(x.type);
+                        set_disable(false);
+                    }}>Commit all</Button>
+                    </Box>
                         {x.value.map(y=>(
                             <Box className={classes.commitable} key={y.path}>
-                            <Button onClick={()=>{
+                            <Button variant="contained" onClick={()=>{
                                 set_disable(true);
                                 prop.onCommit(y);
                                 set_disable(false);
@@ -76,7 +85,26 @@ export function DifferencePage(){
         if(bb.ok){
             doLoad();
         }
-    } 
+        else{
+            console.error("fail to add document");
+        }
+    }
+    const CommitAll = async (type :string)=>{
+        const res = await fetch("/api/diff/commitall",{
+            method:"POST",
+            body: JSON.stringify({type:type}),
+            headers:{
+                'content-type':'application/json'
+            }
+        });
+        const bb = await res.json();
+        if(bb.ok){
+            doLoad();
+        }
+        else{
+            console.error("fail to add document");
+        }
+    }
     useEffect(
         ()=>{
             doLoad();
@@ -90,7 +118,7 @@ export function DifferencePage(){
     return (<Headline menu={menu}>
         {(ctx.username == "admin") ? (<div>
             {(diffList.map(x=>
-                <TypeDifference key={x.type} content={x} onCommit={Commit}/>))}
+                <TypeDifference key={x.type} content={x} onCommit={Commit} onCommitAll={CommitAll}/>))}
         </div>)
         :(<Typography variant='h2'>Not Allowed : please login as an admin</Typography>)
         }

src/db/doc.ts

@@ -1,5 +1,5 @@
 import { Document, DocumentBody, DocumentAccessor, QueryListOption } from '../model/doc';
-import Knex from 'knex';
+import {Knex} from 'knex';
 import {createKnexTagController} from './tag';
 import { TagAccessor } from '../model/tag';
 
@@ -15,6 +15,43 @@ class KnexDocumentAccessor implements DocumentAccessor{
         this.knex = knex;
         this.tagController = createKnexTagController(knex);
     }
+    async addList(content_list: DocumentBody[]):Promise<number[]>{
+        return await this.knex.transaction(async (trx)=>{
+            //add tags
+            const tagCollected = new Set<string>();
+            content_list.map(x=>x.tags).forEach((x)=>{
+                x.forEach(x=>{
+                    tagCollected.add(x);
+                });
+            });
+            const tagCollectPromiseList = [];
+            const tagController = createKnexTagController(trx);
+            for (const it of tagCollected){
+                const p = tagController.addTag({name:it});
+                tagCollectPromiseList.push(p);
+            }
+            await Promise.all(tagCollectPromiseList);
+            //add for each contents
+            const ret = [];
+            for (const content of content_list) {
+                const {tags,additional, ...rest} = content;
+                const id_lst = await trx.insert({
+                        additional:JSON.stringify(additional),
+                        created_at:Date.now(),
+                        ...rest
+                }).into("document");
+                const id = id_lst[0];
+                if(tags.length > 0){
+                    await trx.insert(tags.map(y=>({
+                        doc_id:id,
+                        tag_name:y
+                    }))).into('doc_tag_relation');
+                }
+                ret.push(id);
+            }
+            return ret;
+        });
+    }
     async add(c: DocumentBody){
         const {tags,additional, ...rest} = c;
         const id_lst = await this.knex.insert({

src/db/tag.ts

@@ -1,5 +1,5 @@
 import {Tag, TagAccessor} from '../model/tag';
-import Knex from 'knex';
+import {Knex} from 'knex';
 
 type DBTags = {
     name: string,

src/db/user.ts

@@ -1,4 +1,4 @@
-import Knex from 'knex';
+import {Knex} from 'knex';
 import {IUser,UserCreateInput, UserAccessor, Password} from '../model/user';
 
 type PermissionTable = {

src/diff/content_handler.ts

@@ -6,9 +6,12 @@ import {ContentList} from './content_list';
 
 //refactoring needed.
 export class ContentDiffHandler{
+    /** content file list waiting to add */
     waiting_list:ContentList;
+    /** deleted contents */
     tombstone: Map<string,Document>;//hash, contentfile
     doc_cntr: DocumentAccessor;
+    /** content type of handle */
     content_type: string;
     constructor(cntr: DocumentAccessor,content_type:string){
         this.waiting_list = new ContentList();
@@ -30,14 +33,19 @@ export class ContentDiffHandler{
     private async OnDeleted(cpath: string){
         const basepath = dirname(cpath);
         const filename = basename(cpath);
-        if(this.waiting_list.hasPath(cpath)){
+        //if it wait to add, delete it from waiting list.
-            this.waiting_list.deletePath(cpath);
+        if(this.waiting_list.hasByPath(cpath)){
+            this.waiting_list.deleteByPath(cpath);
             return;
         }
         const dbc = await this.doc_cntr.findByPath(basepath,filename);
-        if(dbc.length === 0) return; //ignore
+        //when there is no related content in db, ignore.
-        if(this.waiting_list.hasHash(dbc[0].content_hash)){
+        if(dbc.length === 0) return;
-            //if path changed, update changed path.
+        // When a path is changed, it takes into account when the
+        // creation event occurs first and the deletion occurs, not
+        // the change event. 
+        if(this.waiting_list.hasByHash(dbc[0].content_hash)){
+            //if a path is changed, update the changed path.
             await this.doc_cntr.update({
                 id:dbc[0].id,
                 deleted_at: null,
@@ -46,7 +54,7 @@ export class ContentDiffHandler{
             });
             return;
         }
-        //db invalidate
+        //invalidate db and add it to tombstone.
         await this.doc_cntr.update({
             id:dbc[0].id,
             deleted_at: Date.now(),

src/diff/content_list.ts

@@ -1,35 +1,25 @@
 import { ContentFile } from '../content/mod';
-import event from 'events';
 
-interface ContentListEvent{
+export class ContentList{
-    'set':(c:ContentFile)=>void,
+    /** path map */
-    'delete':(c:ContentFile)=>void,
+    private cl:Map<string,ContentFile>;
-}
+    /** hash map */
-
+    private hl:Map<string,ContentFile>;
-export class ContentList extends event.EventEmitter{
+    
-    cl:Map<string,ContentFile>;
-    hl:Map<string,ContentFile>;
-    on<U extends keyof ContentListEvent>(event:U,listener:ContentListEvent[U]): this{
-        return super.on(event,listener);
-    }
-    emit<U extends keyof ContentListEvent>(event:U,...arg:Parameters<ContentListEvent[U]>): boolean{
-        return super.emit(event,...arg);
-    }
     constructor(){
-        super();
         this.cl = new Map;
         this.hl = new Map;
     }
-    hasHash(s:string){
+    hasByHash(s:string){
         return this.hl.has(s);
     }
-    hasPath(p:string){
+    hasByPath(p:string){
         return this.cl.has(p);
     }
-    getHash(s:string){
+    getByHash(s:string){
         return this.hl.get(s)
     }
-    getPath(p:string){
+    getByPath(p:string){
         return this.cl.get(p);
     }
     async set(c:ContentFile){
@@ -37,25 +27,29 @@ export class ContentList extends event.EventEmitter{
         const hash = await c.getHash();
         this.cl.set(path,c);
         this.hl.set(hash,c);
-        this.emit('set',c);
     }
+    /** delete content file */
     async delete(c:ContentFile){
+        const hash = await c.getHash();
         let r = true;
         r = this.cl.delete(c.path) && r;
-        r = this.hl.delete(await c.getHash()) && r;
+        r = this.hl.delete(hash) && r;
-        this.emit('delete',c);
         return r;
     }
-    async deletePath(p:string){
+    async deleteByPath(p:string){
-        const o = this.getPath(p);
+        const o = this.getByPath(p);
         if(o === undefined) return false;
         return this.delete(o);
     }
-    async deleteHash(s:string){
+    async deleteByHash(s:string){
-        const o = this.getHash(s);
+        const o = this.getByHash(s);
         if(o === undefined) return false;
         return this.delete(o);
     }
+    clear(){
+        this.cl.clear();
+        this.hl.clear();
+    }
     getAll(){
         return [...this.cl.values()];
     }

src/diff/diff.ts

@@ -1,8 +1,8 @@
 import { DocumentAccessor } from '../model/doc';
 import {ContentDiffHandler} from './content_handler';
 import { IDiffWatcher } from './watcher';
+import asyncPool from 'tiny-async-pool';
 
-//import {join as pathjoin} from 'path';
 export class DiffManager{
     watching: {[content_type:string]:ContentDiffHandler};
     doc_cntr: DocumentAccessor;
@@ -19,7 +19,7 @@ export class DiffManager{
     }
     async commit(type:string,path:string){
         const list = this.watching[type].waiting_list;
-        const c = list.getPath(path);
+        const c = list.getByPath(path);
         if(c===undefined){
             throw new Error("path is not exist");
         }
@@ -28,6 +28,14 @@ export class DiffManager{
         const id = await this.doc_cntr.add(body);
         return id;
     }
+    async commitAll(type:string){
+        const list = this.watching[type].waiting_list;
+        const contentFiles = list.getAll();
+        list.clear();
+        const bodies = await asyncPool(30,contentFiles,async (x)=>await x.createDocumentBody());
+        const ids = await this.doc_cntr.addList(bodies);
+        return ids;
+    }
     getAdded(){
         return Object.keys(this.watching).map(x=>({
             type:x,

src/diff/router.ts

@@ -32,7 +32,6 @@ function checkPostAddedBody(body: any): body is PostAddedBody{
 
 export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next)=>{
     const reqbody = ctx.request.body;
-    console.log(reqbody);
     if(!checkPostAddedBody(reqbody)){
         sendError(400,"format exception");
         return;
@@ -45,6 +44,27 @@ export const postAdded = (diffmgr:DiffManager) => async (ctx:Router.IRouterConte
     }
     ctx.type = 'json';
 }
+export const postAddedAll = (diffmgr: DiffManager) => async (ctx:Router.IRouterContext,next:Koa.Next) => {
+    if (!ctx.is('json')){
+        sendError(400,"format exception");
+        return;
+    }
+    const reqbody = ctx.request.body as Record<string,unknown>;
+    if(!("type" in reqbody)){
+        sendError(400,"format exception: there is no \"type\"");
+        return;
+    }
+    const t = reqbody["type"];
+    if(typeof t !== "string"){
+        sendError(400,"format exception: invalid type of \"type\"");
+        return;
+    }
+    await diffmgr.commitAll(t);
+    ctx.body = {
+        ok:true
+    };
+    ctx.type = 'json';
+}
 /*
 export const getNotWatched = (diffmgr : DiffManager)=> (ctx:Router.IRouterContext,next:Koa.Next)=>{
     ctx.body = {
@@ -58,5 +78,6 @@ export function createDiffRouter(diffmgr: DiffManager){
     const ret = new Router();
     ret.get("/list",AdminOnlyMiddleware,getAdded(diffmgr));
     ret.post("/commit",AdminOnlyMiddleware,postAdded(diffmgr));
+    ret.post("/commitall",AdminOnlyMiddleware,postAddedAll(diffmgr));
     return ret;
 }

src/model/doc.ts

@@ -102,6 +102,10 @@ export interface DocumentAccessor{
      * add document
      */
     add:(c:DocumentBody)=>Promise<number>;
+    /**
+     * add document list
+     */
+    addList:(content_list:DocumentBody[]) => Promise<number[]>;
     /**
      * delete document
      * @returns if it exists, return true.

src/route/contents.ts

@@ -71,15 +71,7 @@ const UpdateContentHandler = (controller : DocumentAccessor) => async (ctx: Cont
     ctx.body = JSON.stringify(success);
     ctx.type = 'json';
 }
-/*const CreateContentHandler = (controller : DocumentAccessor) => async (ctx: Context, next: Next) => {
+
-    const content_desc = ctx.request.body;
-    if(!isDocBody(content_desc)){
-        return sendError(400,"it is not a valid format");
-    }
-    const id = await controller.add(content_desc);
-    ctx.body = JSON.stringify(id);
-    ctx.type = 'json';
-};*/
 const AddTagHandler = (controller: DocumentAccessor)=>async (ctx: Context, next: Next)=>{
     let tag_name = ctx.params['tag'];
     const num = Number.parseInt(ctx.params['num']);

src/types/db.d.ts

@@ -1,4 +1,4 @@
-import Knex from "knex";
+import {Knex} from "knex";
 
 declare module "knex" {
   interface Tables {
@@ -31,6 +31,4 @@ declare module "knex" {
       name: string;
     };
   }
-  namespace Knex {
-  }
 }