monoid/ionian
1
0
Fork 0
Code Issues 9 Pull requests Projects Releases Wiki Activity

feat: file reader rework

Browse source
This commit is contained in:
monoid 2025-05-01 16:51:24 +09:00
parent f8e2930ec1
commit b3f0f6d980
4 changed files with 77 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/server/package.json
View file

@ -12,7 +12,7 @@
"license": "ISC",
"dependencies": {
"@std/async": "npm:@jsr/std__async@^1.0.12",
"@zip.js/zip.js": "^2.7.52",
"@zip.js/zip.js": "^2.7.60",
"better-sqlite3": "^9.6.0",
"chokidar": "^3.6.0",
"dbtype": "workspace:dbtype",
@ -33,7 +33,7 @@
"@types/koa-bodyparser": "^4.3.12",
"@types/koa-compose": "^3.2.8",
"@types/koa-router": "^7.4.8",
"@types/node": "^22.7.4",
"@types/node": "^22.15.3",
"@types/tiny-async-pool": "^1.0.5",
"nodemon": "^3.1.7",
"tsx": "^4.19.1",

8
packages/server/src/route/comic.ts
View file

@ -12,7 +12,6 @@ import { Readable } from "node:stream";
*/
const ZipStreamCache = new Map<string, {
reader: ZipReader<FileHandle>,
handle: FileHandle,
refCount: number,
}>();
@ -39,7 +38,6 @@ async function acquireZip(path: string, marked = false) {
// if the cache is not updated, set the new one.
ZipStreamCache.set(path, {
reader: obj.reader,
handle: obj.handle,
refCount: 1,
});
return obj.reader;
@ -57,10 +55,8 @@ function releaseZip(path: string) {
return;
}
if (obj.refCount === 1) {
const { reader, handle } = obj;
reader.close().then(() => {
handle.close();
});
const { reader } = obj;
reader.close();
ZipStreamCache.delete(path);
} else {
obj.refCount--;

63
packages/server/src/util/zipwrap.ts
View file

@ -1,42 +1,67 @@
import { type FileHandle, open } from "node:fs/promises";
import { orderBy } from "natural-orderby";
import { ZipReader, Reader, type Entry } from "@zip.js/zip.js";
import EventEmitter from "node:events";
class FileReader extends Reader<FileHandle> {
private fd: FileHandle;
constructor(fd: FileHandle) {
super(fd);
this.fd = fd;
class FileReader extends Reader<string> {
private fd?: FileHandle;
private path: string;
constructor(path: string) {
super(path);
this.path = path;
}
async init(): Promise<void> {
this.size = (await this.fd.stat()).size;
await super.init?.();
const fd = await open(this.path, "r");
const stat = await fd.stat();
this.fd = fd;
this.size = stat.size;
// not implemented yet
(this.fd as unknown as EventEmitter).on("close", () => {
console.warn(`file handle closed: ${this.path}`);
this.fd = undefined;
});
}
close(): void {
this.fd.close();
async close(): Promise<void> {
await this.fd?.close();
}
async readUint8Array(index: number, length: number): Promise<Uint8Array> {
const buffer = new Uint8Array(length);
const buf = await this.fd.read(buffer, 0, length, index);
if (buf.bytesRead !== length) {
console.error(`read error: ${buf.bytesRead} !== ${length}`);
throw new Error("read error");
try {
const buffer = new Uint8Array(length);
if (this.fd === undefined) {
console.error("file handle is undefined", this.path);
// reopen the file handle
this.fd = await open(this.path, "r");
const stat = await this.fd.stat();
if (stat.size !== this.size) {
console.error("file size changed", this.path, stat.size, this.size);
throw new Error("file size changed");
}
}
const buf = await this.fd.read(buffer, 0, length, index);
if (buf.bytesRead !== length) {
console.error(`read error: ${buf.bytesRead} !== ${length}`);
throw new Error("read error");
}
return buffer;
} catch (error) {
console.error("read error", error);
throw error;
}
return buffer;
}
}
export async function readZip(path: string): Promise<{
reader: ZipReader<FileHandle>
handle: FileHandle
}> {
const fd = await open(path, "r");
const reader = new ZipReader(new FileReader(fd), {
const reader = new ZipReader(new FileReader(path), {
useCompressionStream: true,
preventClose: false,
});
return { reader, handle: fd };
return { reader };
}
export async function entriesByNaturalOrder(zip: ZipReader<FileHandle>) {
const entries = await zip.getEntries();

46
pnpm-lock.yaml generated
View file

@ -173,8 +173,8 @@ importers:
specifier: npm:@jsr/std__async@^1.0.12
version: '@jsr/std__async@1.0.12'
'@zip.js/zip.js':
specifier: ^2.7.52
version: 2.7.52
specifier: ^2.7.60
version: 2.7.60
better-sqlite3:
specifier: ^9.6.0
version: 9.6.0
@ -231,8 +231,8 @@ importers:
specifier: ^7.4.8
version: 7.4.8
'@types/node':
specifier: ^22.7.4
version: 22.7.4
specifier: ^22.15.3
version: 22.15.3
'@types/tiny-async-pool':
specifier: ^1.0.5
version: 1.0.5
@ -1468,6 +1468,9 @@ packages:
'@types/mime@1.3.5':
resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
'@types/node@22.15.3':
resolution: {integrity: sha512-lX7HFZeHf4QG/J7tBZqrCAXwz9J5RD56Y6MpP0eJkka8p+K0RY/yBTW7CYFJ4VGCclxqOLKmiGP5juQc6MKgcw==}
'@types/node@22.7.4':
resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==}
@ -1591,8 +1594,8 @@ packages:
'@vitest/utils@2.1.2':
resolution: {integrity: sha512-zMO2KdYy6mx56btx9JvAqAZ6EyS3g49krMPPrgOp1yxGZiA93HumGk+bZ5jIZtOg5/VBYl5eBmGRQHqq4FG6uQ==}
'@zip.js/zip.js@2.7.52':
resolution: {integrity: sha512-+5g7FQswvrCHwYKNMd/KFxZSObctLSsQOgqBSi0LzwHo3li9Eh1w5cF5ndjQw9Zbr3ajVnd2+XyiX85gAetx1Q==}
'@zip.js/zip.js@2.7.60':
resolution: {integrity: sha512-vA3rLyqdxBrVo1FWSsbyoecaqWTV+vgPRf0QKeM7kVDG0r+lHUqd7zQDv1TO9k4BcAoNzNDSNrrel24Mk6addA==}
engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=16.5.0'}
accepts@1.3.8:
@ -3357,6 +3360,9 @@ packages:
undici-types@6.19.8:
resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
undici-types@6.21.0:
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
universalify@2.0.1:
resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==}
engines: {node: '>= 10.0.0'}
@ -4521,11 +4527,11 @@ snapshots:
'@types/accepts@1.3.7':
dependencies:
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/better-sqlite3@7.6.11':
dependencies:
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/better-sqlite3@7.6.9':
dependencies:
@ -4534,11 +4540,11 @@ snapshots:
'@types/body-parser@1.19.5':
dependencies:
'@types/connect': 3.4.38
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/connect@3.4.38':
dependencies:
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/content-disposition@0.5.8': {}
@ -4547,7 +4553,7 @@ snapshots:
'@types/connect': 3.4.38
'@types/express': 5.0.0
'@types/keygrip': 1.0.6
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/d3-array@3.2.1': {}
@ -4577,7 +4583,7 @@ snapshots:
'@types/express-serve-static-core@5.0.0':
dependencies:
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/qs': 6.9.16
'@types/range-parser': 1.2.7
'@types/send': 0.17.4
@ -4595,7 +4601,7 @@ snapshots:
'@types/jsonwebtoken@8.5.9':
dependencies:
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/keygrip@1.0.6': {}
@ -4620,10 +4626,14 @@ snapshots:
'@types/http-errors': 2.0.4
'@types/keygrip': 1.0.6
'@types/koa-compose': 3.2.8
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/mime@1.3.5': {}
'@types/node@22.15.3':
dependencies:
undici-types: 6.21.0
'@types/node@22.7.4':
dependencies:
undici-types: 6.19.8
@ -4646,12 +4656,12 @@ snapshots:
'@types/send@0.17.4':
dependencies:
'@types/mime': 1.3.5
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/serve-static@1.15.7':
dependencies:
'@types/http-errors': 2.0.4
'@types/node': 22.7.4
'@types/node': 22.15.3
'@types/send': 0.17.4
'@types/tiny-async-pool@1.0.5': {}
@ -4786,7 +4796,7 @@ snapshots:
loupe: 3.1.2
tinyrainbow: 1.2.0
'@zip.js/zip.js@2.7.52': {}
'@zip.js/zip.js@2.7.60': {}
accepts@1.3.8:
dependencies:
@ -6564,6 +6574,8 @@ snapshots:
undici-types@6.19.8: {}
undici-types@6.21.0: {}
universalify@2.0.1: {}
unpipe@1.0.0: {}