[BREAKING!]: 서버 재작업 #16
175
deno.lock
Normal file
175
deno.lock
Normal file
@ -0,0 +1,175 @@
|
||||
{
|
||||
"version": "3",
|
||||
"packages": {
|
||||
"specifiers": {
|
||||
"jsr:@db/sqlite": "jsr:@db/sqlite@0.11.1",
|
||||
"jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.5",
|
||||
"jsr:@std/assert@^0.214.0": "jsr:@std/assert@0.214.0",
|
||||
"jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0",
|
||||
"jsr:@std/encoding@0.214": "jsr:@std/encoding@0.214.0",
|
||||
"jsr:@std/fmt@0.214": "jsr:@std/fmt@0.214.0",
|
||||
"jsr:@std/fs@0.214": "jsr:@std/fs@0.214.0",
|
||||
"jsr:@std/path": "jsr:@std/path@0.217.0",
|
||||
"jsr:@std/path@0.214": "jsr:@std/path@0.214.0",
|
||||
"jsr:@std/path@0.217": "jsr:@std/path@0.217.0",
|
||||
"jsr:@std/path@^0.214.0": "jsr:@std/path@0.214.0"
|
||||
},
|
||||
"jsr": {
|
||||
"@db/sqlite@0.11.1": {
|
||||
"integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5",
|
||||
"dependencies": [
|
||||
"jsr:@denosaurs/plug@1",
|
||||
"jsr:@std/path@0.217"
|
||||
]
|
||||
},
|
||||
"@denosaurs/plug@1.0.5": {
|
||||
"integrity": "04cd988da558adc226202d88c3a434d5fcc08146eaf4baf0cea0c2284b16d2bf",
|
||||
"dependencies": [
|
||||
"jsr:@std/encoding@0.214",
|
||||
"jsr:@std/fmt@0.214",
|
||||
"jsr:@std/fs@0.214",
|
||||
"jsr:@std/path@0.214"
|
||||
]
|
||||
},
|
||||
"@std/assert@0.214.0": {
|
||||
"integrity": "55d398de76a9828fd3b1aa653f4dba3eee4c6985d90c514865d2be9bd082b140"
|
||||
},
|
||||
"@std/assert@0.217.0": {
|
||||
"integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642"
|
||||
},
|
||||
"@std/encoding@0.214.0": {
|
||||
"integrity": "30a8713e1db22986c7e780555ffd2fefd1d4f9374d734bb41f5970f6c3352af5"
|
||||
},
|
||||
"@std/fmt@0.214.0": {
|
||||
"integrity": "40382cff88a0783b347b4d69b94cf931ab8e549a733916718cb866c08efac4d4"
|
||||
},
|
||||
"@std/fs@0.214.0": {
|
||||
"integrity": "bc880fea0be120cb1550b1ed7faf92fe071003d83f2456a1e129b39193d85bea",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.214.0",
|
||||
"jsr:@std/path@^0.214.0"
|
||||
]
|
||||
},
|
||||
"@std/path@0.214.0": {
|
||||
"integrity": "d5577c0b8d66f7e8e3586d864ebdf178bb326145a3611da5a51c961740300285",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.214.0"
|
||||
]
|
||||
},
|
||||
"@std/path@0.217.0": {
|
||||
"integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11",
|
||||
"dependencies": [
|
||||
"jsr:@std/assert@^0.217.0"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"remote": {
|
||||
"https://deno.land/x/sqlite@v3.9.1/build/sqlite.js": "2afc7875c7b9c85d89730c4a311ab3a304e5d1bf761fbadd8c07bbdf130f5f9b",
|
||||
"https://deno.land/x/sqlite@v3.9.1/build/vfs.js": "7f7778a9fe499cd10738d6e43867340b50b67d3e39142b0065acd51a84cd2e03",
|
||||
"https://deno.land/x/sqlite@v3.9.1/mod.ts": "e09fc79d8065fe222578114b109b1fd60077bff1bb75448532077f784f4d6a83",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/constants.ts": "90f3be047ec0a89bcb5d6fc30db121685fc82cb00b1c476124ff47a4b0472aa9",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/db.ts": "03d0c860957496eadedd86e51a6e650670764630e64f56df0092e86c90752401",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/function.ts": "bc778cab7a6d771f690afa27264c524d22fcb96f1bb61959ade7922c15a4ab8d",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/query.ts": "d58abda928f6582d77bad685ecf551b1be8a15e8e38403e293ec38522e030cad",
|
||||
"https://deno.land/x/sqlite@v3.9.1/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487"
|
||||
},
|
||||
"workspace": {
|
||||
"packageJson": {
|
||||
"dependencies": [
|
||||
"npm:@biomejs/biome@1.6.3"
|
||||
]
|
||||
},
|
||||
"members": {
|
||||
"packages/client": {
|
||||
"packageJson": {
|
||||
"dependencies": [
|
||||
"npm:@radix-ui/react-icons@^1.3.0",
|
||||
"npm:@radix-ui/react-label@^2.1.0",
|
||||
"npm:@radix-ui/react-popover@^1.1.2",
|
||||
"npm:@radix-ui/react-progress@^1.1.0",
|
||||
"npm:@radix-ui/react-radio-group@^1.2.1",
|
||||
"npm:@radix-ui/react-scroll-area@^1.2.0",
|
||||
"npm:@radix-ui/react-select@^2.1.2",
|
||||
"npm:@radix-ui/react-separator@^1.1.0",
|
||||
"npm:@radix-ui/react-slot@^1.1.0",
|
||||
"npm:@radix-ui/react-tabs@^1.1.1",
|
||||
"npm:@radix-ui/react-tooltip@^1.1.3",
|
||||
"npm:@tanstack/react-virtual@^3.10.8",
|
||||
"npm:@types/node@^22.7.4",
|
||||
"npm:@types/react-dom@^18.3.0",
|
||||
"npm:@types/react@^18.3.11",
|
||||
"npm:@typescript-eslint/eslint-plugin@^7.18.0",
|
||||
"npm:@typescript-eslint/parser@^7.18.0",
|
||||
"npm:@vitejs/plugin-react-swc@^3.7.1",
|
||||
"npm:autoprefixer@^10.4.20",
|
||||
"npm:class-variance-authority@^0.7.0",
|
||||
"npm:clsx@^2.1.1",
|
||||
"npm:eslint-plugin-react-hooks@^4.6.2",
|
||||
"npm:eslint-plugin-react-refresh@^0.4.12",
|
||||
"npm:eslint@^8.57.1",
|
||||
"npm:jotai@^2.10.0",
|
||||
"npm:lucide-react@^0.451.0",
|
||||
"npm:postcss@^8.4.47",
|
||||
"npm:react-dom@^18.3.1",
|
||||
"npm:react-resizable-panels@^2.1.4",
|
||||
"npm:react@^18.3.1",
|
||||
"npm:recharts@^2.12.7",
|
||||
"npm:shadcn-ui@^0.8.0",
|
||||
"npm:swr@^2.2.5",
|
||||
"npm:tailwind-merge@^2.5.3",
|
||||
"npm:tailwindcss-animate@^1.0.7",
|
||||
"npm:tailwindcss@^3.4.13",
|
||||
"npm:typescript@^5.6.2",
|
||||
"npm:usehooks-ts@^3.1.0",
|
||||
"npm:vite@^5.4.8",
|
||||
"npm:vitest@^2.1.2",
|
||||
"npm:wouter@^3.3.5"
|
||||
]
|
||||
}
|
||||
},
|
||||
"packages/dbtype": {
|
||||
"packageJson": {
|
||||
"dependencies": [
|
||||
"npm:@types/better-sqlite3@^7.6.9",
|
||||
"npm:better-sqlite3@^9.4.3",
|
||||
"npm:kysely-codegen@^0.14.1",
|
||||
"npm:kysely@^0.27.3",
|
||||
"npm:typescript@^5.4.3",
|
||||
"npm:zod@^3.23.8"
|
||||
]
|
||||
}
|
||||
},
|
||||
"packages/server": {
|
||||
"packageJson": {
|
||||
"dependencies": [
|
||||
"npm:@types/better-sqlite3@^7.6.11",
|
||||
"npm:@types/jsonwebtoken@^8.5.9",
|
||||
"npm:@types/koa-bodyparser@^4.3.12",
|
||||
"npm:@types/koa-compose@^3.2.8",
|
||||
"npm:@types/koa-router@^7.4.8",
|
||||
"npm:@types/koa@^2.15.0",
|
||||
"npm:@types/node@^22.7.4",
|
||||
"npm:@types/tiny-async-pool@^1.0.5",
|
||||
"npm:@zip.js/zip.js@^2.7.52",
|
||||
"npm:better-sqlite3@^9.6.0",
|
||||
"npm:chokidar@^3.6.0",
|
||||
"npm:dotenv@^16.4.5",
|
||||
"npm:jose@^5.9.3",
|
||||
"npm:koa-bodyparser@^4.4.1",
|
||||
"npm:koa-compose@^4.1.0",
|
||||
"npm:koa-router@^12.0.1",
|
||||
"npm:koa@^2.15.3",
|
||||
"npm:kysely@^0.27.4",
|
||||
"npm:natural-orderby@^2.0.3",
|
||||
"npm:nodemon@^3.1.7",
|
||||
"npm:tiny-async-pool@^1.3.0",
|
||||
"npm:tsx@^4.19.1",
|
||||
"npm:typescript@^5.6.2"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -16,6 +16,7 @@ import SettingPage from "@/page/settingPage.tsx";
|
||||
import ComicPage from "@/page/reader/comicPage.tsx";
|
||||
import DifferencePage from "./page/differencePage.tsx";
|
||||
import TagsPage from "./page/tagsPage.tsx";
|
||||
import TaskQueuePage from "./page/taskQueuePage.tsx";
|
||||
|
||||
const App = () => {
|
||||
const { isDarkMode } = useTernaryDarkMode();
|
||||
@ -42,6 +43,7 @@ const App = () => {
|
||||
<Route path="/doc/:id/reader" component={ComicPage}/>
|
||||
<Route path="/difference" component={DifferencePage}/>
|
||||
<Route path="/tags" component={TagsPage}/>
|
||||
<Route path="/queue" component={TaskQueuePage} />
|
||||
<Route component={NotFoundPage} />
|
||||
</Switch>
|
||||
</Layout>
|
||||
|
||||
206
packages/client/src/components/gallery/WorkQueue.tsx
Normal file
206
packages/client/src/components/gallery/WorkQueue.tsx
Normal file
@ -0,0 +1,206 @@
|
||||
import { useState, useEffect } from 'react'
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Progress } from "@/components/ui/progress"
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
||||
import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer } from 'recharts'
|
||||
|
||||
interface Task {
|
||||
id: string;
|
||||
status: "Processed" | "Processing" | "Queued" | "Exception";
|
||||
createdAt: Date;
|
||||
title: string;
|
||||
description: string;
|
||||
expectedProgress: number;
|
||||
currentJobDescription: string;
|
||||
}
|
||||
|
||||
const generateMockTasks = (): Task[] => {
|
||||
const statuses: Task['status'][] = ["Processed", "Processing", "Queued", "Exception"];
|
||||
return Array.from({ length: 50 }, (_, i) => ({
|
||||
id: `task-${i + 1}`,
|
||||
status: statuses[Math.floor(Math.random() * statuses.length)],
|
||||
createdAt: new Date(Date.now() - Math.random() * 10000000000),
|
||||
title: `Task ${i + 1}`,
|
||||
description: `This is a description for Task ${i + 1}`,
|
||||
expectedProgress: Math.random(),
|
||||
currentJobDescription: `Current job for Task ${i + 1}`
|
||||
}));
|
||||
};
|
||||
|
||||
export default function DynamicWorkQueue() {
|
||||
const [tasks, setTasks] = useState<Task[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
setTasks(generateMockTasks());
|
||||
|
||||
const intervalId = setInterval(() => {
|
||||
setTasks(prevTasks => {
|
||||
let newTasks: Task[] = prevTasks;
|
||||
// if processed tasks are more than 10, remove the oldest one
|
||||
if (newTasks.filter(task => task.status === "Processed").length > 10) {
|
||||
newTasks = newTasks.filter(task => task.status !== "Processed");
|
||||
}
|
||||
// update the progress of each task
|
||||
newTasks = newTasks.map(task => ({
|
||||
...task,
|
||||
expectedProgress: Math.min(1, task.expectedProgress + Math.random() * 0.2),
|
||||
status: task.expectedProgress >= 1 ? "Processed" : task.status
|
||||
}));
|
||||
// if there are no queued tasks, add a new one
|
||||
if (newTasks.filter(task => task.status === "Queued").length === 0) {
|
||||
newTasks.push({
|
||||
id: `task-${newTasks.length + 1}`,
|
||||
status: "Queued",
|
||||
createdAt: new Date(),
|
||||
title: `Task ${newTasks.length + 1}`,
|
||||
description: `This is a description for Task ${newTasks.length + 1}`,
|
||||
expectedProgress: 0,
|
||||
currentJobDescription: `Current job for Task ${newTasks.length + 1}`
|
||||
});
|
||||
}
|
||||
return newTasks;
|
||||
});
|
||||
}, 5000);
|
||||
|
||||
return () => clearInterval(intervalId);
|
||||
}, [])
|
||||
|
||||
const updateTaskStatus = (taskId: string, newStatus: Task['status']) => {
|
||||
setTasks(prevTasks =>
|
||||
prevTasks.map(task =>
|
||||
task.id === taskId ? { ...task, status: newStatus } : task
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const renderTaskList = (status: Task['status']) => {
|
||||
const filteredTasks = tasks.filter(task => task.status === status)
|
||||
|
||||
return (
|
||||
<>
|
||||
{filteredTasks.length === 0 ? (
|
||||
<p className="text-gray-500 p-4">No tasks</p>
|
||||
) : (
|
||||
<ul className="space-y-4 p-4">
|
||||
{filteredTasks.map(task => (
|
||||
<li key={task.id} className="border p-4 rounded-md shadow-sm">
|
||||
<h4 className="font-medium text-primary">{task.title}</h4>
|
||||
<p className="text-sm text-gray-600">{task.description}</p>
|
||||
<p className="text-sm text-gray-500 mt-1">Created: {task.createdAt.toLocaleString()}</p>
|
||||
<div className="mt-2">
|
||||
<Progress value={task.expectedProgress * 100} className="h-2" />
|
||||
<p className="text-xs text-right mt-1">{Math.round(task.expectedProgress * 100)}%</p>
|
||||
</div>
|
||||
<p className="text-sm mt-1 text-gray-700">{task.currentJobDescription}</p>
|
||||
{status === "Queued" && (
|
||||
<Button
|
||||
onClick={() => updateTaskStatus(task.id, "Processing")}
|
||||
className="mt-2"
|
||||
size="sm"
|
||||
>
|
||||
Start Processing
|
||||
</Button>
|
||||
)}
|
||||
{status === "Processing" && (
|
||||
<Button
|
||||
onClick={() => updateTaskStatus(task.id, "Processed")}
|
||||
className="mt-2"
|
||||
size="sm"
|
||||
>
|
||||
Mark as Processed
|
||||
</Button>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
const renderProcessedTaskSummary = () => {
|
||||
const processedTasks = tasks.filter(task => task.status === "Processed");
|
||||
const totalProcessed = processedTasks.length;
|
||||
const averageProgress = processedTasks.reduce((sum, task) => sum + task.expectedProgress, 0) / totalProcessed || 0;
|
||||
|
||||
const chartData = [
|
||||
{ name: 'Processed', value: totalProcessed },
|
||||
{ name: 'Remaining', value: tasks.length - totalProcessed },
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Total Processed</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-4xl font-bold">{totalProcessed}</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Average Progress</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-4xl font-bold">{(averageProgress * 100).toFixed(2)}%</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Processed vs Remaining Tasks</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ResponsiveContainer width="100%" height={300}>
|
||||
<BarChart data={chartData}>
|
||||
<XAxis dataKey="name" />
|
||||
<YAxis />
|
||||
<Tooltip />
|
||||
<Bar dataKey="value" fill="#8884d8" />
|
||||
</BarChart>
|
||||
</ResponsiveContainer>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="container mx-auto p-4 max-w-4xl">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Dynamic Work Queue</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<Tabs defaultValue="Queued" className="w-full">
|
||||
<TabsList className="grid w-full grid-cols-5">
|
||||
<TabsTrigger value="Queued">Queued ({tasks.filter(t => t.status === "Queued").length})</TabsTrigger>
|
||||
<TabsTrigger value="Processing">Processing ({tasks.filter(t => t.status === "Processing").length})</TabsTrigger>
|
||||
<TabsTrigger value="Processed">Processed ({tasks.filter(t => t.status === "Processed").length})</TabsTrigger>
|
||||
<TabsTrigger value="Exception">Exception ({tasks.filter(t => t.status === "Exception").length})</TabsTrigger>
|
||||
<TabsTrigger value="Summary">Summary</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="Queued">
|
||||
{renderTaskList("Queued")}
|
||||
</TabsContent>
|
||||
<TabsContent value="Processing">
|
||||
{renderTaskList("Processing")}
|
||||
</TabsContent>
|
||||
<TabsContent value="Processed">
|
||||
{renderTaskList("Processed")}
|
||||
</TabsContent>
|
||||
<TabsContent value="Exception">
|
||||
{renderTaskList("Exception")}
|
||||
</TabsContent>
|
||||
<TabsContent value="Summary">
|
||||
{renderProcessedTaskSummary()}
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
import { Link } from "wouter"
|
||||
import { Search, Settings, Tags, ArchiveIcon, UserIcon } from "lucide-react"
|
||||
import { SearchIcon, SettingsIcon, TagsIcon, ArchiveIcon, UserIcon, LayoutListIcon } from "lucide-react"
|
||||
import { Button, buttonVariants } from "@/components/ui/button.tsx"
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip.tsx"
|
||||
import { useLogin } from "@/state/user.ts";
|
||||
@ -66,13 +66,14 @@ export function NavList() {
|
||||
return <aside className="h-dvh flex flex-col">
|
||||
<nav className="flex flex-col items-center gap-4 px-2 sm:py-5 flex-1">
|
||||
{navItems && <>{navItems} <Separator/> </>}
|
||||
<NavItem icon={<Search className="h-5 w-5" />} to="/search" name="Search" />
|
||||
<NavItem icon={<Tags className="h-5 w-5" />} to="/tags" name="Tags" />
|
||||
<NavItem icon={<SearchIcon className="h-5 w-5" />} to="/search" name="Search" />
|
||||
<NavItem icon={<TagsIcon className="h-5 w-5" />} to="/tags" name="Tags" />
|
||||
<NavItem icon={<ArchiveIcon className="h-5 w-5" />} to="/difference" name="Difference" />
|
||||
<NavItem icon={<LayoutListIcon className="h-5 w-5" />} to="/queue" name="Task Queue" />
|
||||
</nav>
|
||||
<nav className="mt-auto flex flex-col items-center gap-4 px-2 sm:py-5 flex-grow-0">
|
||||
<NavItem icon={<UserIcon className="h-5 w-5" />} to={ loginInfo ? "/profile" : "/login"} name={ loginInfo ? "Profiles" : "Login"} />
|
||||
<NavItem icon={<Settings className="h-5 w-5" />} to="/setting" name="Settings" />
|
||||
<NavItem icon={<SettingsIcon className="h-5 w-5" />} to="/setting" name="Settings" />
|
||||
</nav>
|
||||
</aside>
|
||||
}
|
||||
@ -1,10 +1,5 @@
|
||||
import * as React from "react"
|
||||
import * as RechartsPrimitive from "recharts"
|
||||
import {
|
||||
NameType,
|
||||
Payload,
|
||||
ValueType,
|
||||
} from "recharts/types/component/DefaultTooltipContent"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
|
||||
9
packages/client/src/page/taskQueuePage.tsx
Normal file
9
packages/client/src/page/taskQueuePage.tsx
Normal file
@ -0,0 +1,9 @@
|
||||
import DynamicWorkQueue from "@/components/gallery/WorkQueue";
|
||||
|
||||
export default function TaskQueuePage(){
|
||||
return (
|
||||
<div>
|
||||
<DynamicWorkQueue />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -59,16 +59,7 @@ export const SchemaMigrationSchema = z.object({
|
||||
|
||||
export type SchemaMigration = z.infer<typeof SchemaMigrationSchema>;
|
||||
|
||||
const WorkStatusEnum = z.enum(["pending", "done", "error"]);
|
||||
|
||||
export const WorkSchema = z.object({
|
||||
uuid: z.string(),
|
||||
type: z.literal("rehash"),
|
||||
status: WorkStatusEnum,
|
||||
detail: z.string(),
|
||||
});
|
||||
|
||||
export type Work = z.infer<typeof WorkSchema>;
|
||||
|
||||
export const QueryListOptionSchema = z.object({
|
||||
/**
|
||||
|
||||
60
packages/server/pt.ts
Normal file
60
packages/server/pt.ts
Normal file
@ -0,0 +1,60 @@
|
||||
import { Database } from "jsr:@db/sqlite";
|
||||
import {join} from "jsr:@std/path";
|
||||
let db = new Database("./db.sqlite3")
|
||||
const stmt = db.prepare("SELECT id, basepath, filename from document");
|
||||
let ds = [...stmt.all()];
|
||||
|
||||
async function oshash(
|
||||
path: string
|
||||
){
|
||||
const chunkSize = 4096;
|
||||
const minFileSize = chunkSize * 2;
|
||||
|
||||
const fd = await Deno.open(path);
|
||||
const st = await fd.stat();
|
||||
let hash = BigInt(st.size);
|
||||
|
||||
if (st.size < minFileSize){
|
||||
throw new Error("File is too small to hash");
|
||||
}
|
||||
|
||||
// read first and last chunk
|
||||
const firstChunk = new Uint8Array(chunkSize);
|
||||
await fd.read(firstChunk, 0, chunkSize, 0);
|
||||
const lastChunk = new Uint8Array(chunkSize);
|
||||
await fd.read(lastChunk, 0, chunkSize, st.size - chunkSize);
|
||||
// iterate over first and last chunk.
|
||||
// for each uint64_t, add it to the hash.
|
||||
const firstChunkView = new DataView(firstChunk.buffer);
|
||||
for (let i = 0; i < chunkSize; i += 8){
|
||||
hash += firstChunkView.getBigUint64(i, true);
|
||||
// prevent overflow
|
||||
hash = (hash & 0xFFFFFFFFFFFFFFFFn);
|
||||
}
|
||||
const lastChunkView = new DataView(lastChunk.buffer);
|
||||
for (let i = 0; i < chunkSize; i += 8){
|
||||
hash += lastChunkView.getBigUint64(i, true);
|
||||
// prevent overflow
|
||||
hash = (hash & 0xFFFFFFFFFFFFFFFFn);
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
async function updateHash(ds: {id: number, basepath: string, filename: string}[]) {
|
||||
const content_hashs = await Promise.all(ds.map(async (d) => {
|
||||
const p = join(d.basepath, d.filename);
|
||||
return await oshash(p);
|
||||
}));
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < ds.length; i++) {
|
||||
db.run(`UPDATE document SET content_hash = ? where id = ?`, content_hashs[i].toString(), ds[i].id)
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
for (let i = 0; i < ds.length; i += 32) {
|
||||
const d = ds.slice(i, i + 32);
|
||||
console.log(d.map(x => x.id));
|
||||
await updateHash(d);
|
||||
}
|
||||
db.close();
|
||||
@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
||||
import { promises, type Stats } from "node:fs";
|
||||
import path, { extname } from "node:path";
|
||||
import type { DocumentBody } from "dbtype";
|
||||
import { oshash } from "src/util/oshash.ts";
|
||||
/**
|
||||
* content file or directory referrer
|
||||
*/
|
||||
@ -60,14 +61,8 @@ export const createDefaultClass = (type: string): ContentFileConstructor => {
|
||||
}
|
||||
async getHash(): Promise<string> {
|
||||
if (this.hash !== undefined) return this.hash;
|
||||
this.stat = await promises.stat(this.path);
|
||||
const hash = createHash("sha512");
|
||||
hash.update(extname(this.path));
|
||||
hash.update(this.stat.mode.toString());
|
||||
// if(this.desc !== undefined)
|
||||
// hash.update(JSON.stringify(this.desc));
|
||||
hash.update(this.stat.size.toString());
|
||||
this.hash = hash.digest("base64");
|
||||
const hash = await oshash(this.path);
|
||||
this.hash = hash.toString();
|
||||
return this.hash;
|
||||
}
|
||||
async getMtime(): Promise<number> {
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
import { registerContentReferrer } from "./file.ts";
|
||||
import { createDefaultClass } from "./file.ts";
|
||||
|
||||
export class VideoReferrer extends createDefaultClass("video") {
|
||||
}
|
||||
registerContentReferrer(VideoReferrer);
|
||||
Loading…
Reference in New Issue
Block a user