From b3f0f6d980e44de081066b328ec4dbbbf4b74e6f Mon Sep 17 00:00:00 2001 From: monoid Date: Thu, 1 May 2025 16:51:24 +0900 Subject: [PATCH] feat: file reader rework --- packages/server/package.json | 4 +- packages/server/src/route/comic.ts | 8 +--- packages/server/src/util/zipwrap.ts | 63 ++++++++++++++++++++--------- pnpm-lock.yaml | 46 +++++++++++++-------- 4 files changed, 77 insertions(+), 44 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 2f03207..978741a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -12,7 +12,7 @@ "license": "ISC", "dependencies": { "@std/async": "npm:@jsr/std__async@^1.0.12", - "@zip.js/zip.js": "^2.7.52", + "@zip.js/zip.js": "^2.7.60", "better-sqlite3": "^9.6.0", "chokidar": "^3.6.0", "dbtype": "workspace:dbtype", @@ -33,7 +33,7 @@ "@types/koa-bodyparser": "^4.3.12", "@types/koa-compose": "^3.2.8", "@types/koa-router": "^7.4.8", - "@types/node": "^22.7.4", + "@types/node": "^22.15.3", "@types/tiny-async-pool": "^1.0.5", "nodemon": "^3.1.7", "tsx": "^4.19.1", diff --git a/packages/server/src/route/comic.ts b/packages/server/src/route/comic.ts index a88dbe5..23ac39c 100644 --- a/packages/server/src/route/comic.ts +++ b/packages/server/src/route/comic.ts @@ -12,7 +12,6 @@ import { Readable } from "node:stream"; */ const ZipStreamCache = new Map, - handle: FileHandle, refCount: number, }>(); @@ -39,7 +38,6 @@ async function acquireZip(path: string, marked = false) { // if the cache is not updated, set the new one. ZipStreamCache.set(path, { reader: obj.reader, - handle: obj.handle, refCount: 1, }); return obj.reader; @@ -57,10 +55,8 @@ function releaseZip(path: string) { return; } if (obj.refCount === 1) { - const { reader, handle } = obj; - reader.close().then(() => { - handle.close(); - }); + const { reader } = obj; + reader.close(); ZipStreamCache.delete(path); } else { obj.refCount--; diff --git a/packages/server/src/util/zipwrap.ts b/packages/server/src/util/zipwrap.ts index f32ab4f..30facc0 100644 --- a/packages/server/src/util/zipwrap.ts +++ b/packages/server/src/util/zipwrap.ts @@ -1,42 +1,67 @@ import { type FileHandle, open } from "node:fs/promises"; import { orderBy } from "natural-orderby"; import { ZipReader, Reader, type Entry } from "@zip.js/zip.js"; +import EventEmitter from "node:events"; -class FileReader extends Reader { - private fd: FileHandle; - constructor(fd: FileHandle) { - super(fd); - this.fd = fd; +class FileReader extends Reader { + private fd?: FileHandle; + private path: string; + + constructor(path: string) { + super(path); + this.path = path; } - + async init(): Promise { - this.size = (await this.fd.stat()).size; + await super.init?.(); + const fd = await open(this.path, "r"); + const stat = await fd.stat(); + this.fd = fd; + this.size = stat.size; + // not implemented yet + (this.fd as unknown as EventEmitter).on("close", () => { + console.warn(`file handle closed: ${this.path}`); + this.fd = undefined; + }); } - close(): void { - this.fd.close(); + async close(): Promise { + await this.fd?.close(); } async readUint8Array(index: number, length: number): Promise { - const buffer = new Uint8Array(length); - const buf = await this.fd.read(buffer, 0, length, index); - if (buf.bytesRead !== length) { - console.error(`read error: ${buf.bytesRead} !== ${length}`); - throw new Error("read error"); + try { + const buffer = new Uint8Array(length); + if (this.fd === undefined) { + console.error("file handle is undefined", this.path); + // reopen the file handle + this.fd = await open(this.path, "r"); + const stat = await this.fd.stat(); + if (stat.size !== this.size) { + console.error("file size changed", this.path, stat.size, this.size); + throw new Error("file size changed"); + } + } + const buf = await this.fd.read(buffer, 0, length, index); + if (buf.bytesRead !== length) { + console.error(`read error: ${buf.bytesRead} !== ${length}`); + throw new Error("read error"); + } + return buffer; + } catch (error) { + console.error("read error", error); + throw error; } - return buffer; } } export async function readZip(path: string): Promise<{ reader: ZipReader - handle: FileHandle }> { - const fd = await open(path, "r"); - const reader = new ZipReader(new FileReader(fd), { + const reader = new ZipReader(new FileReader(path), { useCompressionStream: true, preventClose: false, }); - return { reader, handle: fd }; + return { reader }; } export async function entriesByNaturalOrder(zip: ZipReader) { const entries = await zip.getEntries(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e9305d7..1429fcc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -173,8 +173,8 @@ importers: specifier: npm:@jsr/std__async@^1.0.12 version: '@jsr/std__async@1.0.12' '@zip.js/zip.js': - specifier: ^2.7.52 - version: 2.7.52 + specifier: ^2.7.60 + version: 2.7.60 better-sqlite3: specifier: ^9.6.0 version: 9.6.0 @@ -231,8 +231,8 @@ importers: specifier: ^7.4.8 version: 7.4.8 '@types/node': - specifier: ^22.7.4 - version: 22.7.4 + specifier: ^22.15.3 + version: 22.15.3 '@types/tiny-async-pool': specifier: ^1.0.5 version: 1.0.5 @@ -1468,6 +1468,9 @@ packages: '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + '@types/node@22.15.3': + resolution: {integrity: sha512-lX7HFZeHf4QG/J7tBZqrCAXwz9J5RD56Y6MpP0eJkka8p+K0RY/yBTW7CYFJ4VGCclxqOLKmiGP5juQc6MKgcw==} + '@types/node@22.7.4': resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==} @@ -1591,8 +1594,8 @@ packages: '@vitest/utils@2.1.2': resolution: {integrity: sha512-zMO2KdYy6mx56btx9JvAqAZ6EyS3g49krMPPrgOp1yxGZiA93HumGk+bZ5jIZtOg5/VBYl5eBmGRQHqq4FG6uQ==} - '@zip.js/zip.js@2.7.52': - resolution: {integrity: sha512-+5g7FQswvrCHwYKNMd/KFxZSObctLSsQOgqBSi0LzwHo3li9Eh1w5cF5ndjQw9Zbr3ajVnd2+XyiX85gAetx1Q==} + '@zip.js/zip.js@2.7.60': + resolution: {integrity: sha512-vA3rLyqdxBrVo1FWSsbyoecaqWTV+vgPRf0QKeM7kVDG0r+lHUqd7zQDv1TO9k4BcAoNzNDSNrrel24Mk6addA==} engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=16.5.0'} accepts@1.3.8: @@ -3357,6 +3360,9 @@ packages: undici-types@6.19.8: resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} @@ -4521,11 +4527,11 @@ snapshots: '@types/accepts@1.3.7': dependencies: - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/better-sqlite3@7.6.11': dependencies: - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/better-sqlite3@7.6.9': dependencies: @@ -4534,11 +4540,11 @@ snapshots: '@types/body-parser@1.19.5': dependencies: '@types/connect': 3.4.38 - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/connect@3.4.38': dependencies: - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/content-disposition@0.5.8': {} @@ -4547,7 +4553,7 @@ snapshots: '@types/connect': 3.4.38 '@types/express': 5.0.0 '@types/keygrip': 1.0.6 - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/d3-array@3.2.1': {} @@ -4577,7 +4583,7 @@ snapshots: '@types/express-serve-static-core@5.0.0': dependencies: - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/qs': 6.9.16 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -4595,7 +4601,7 @@ snapshots: '@types/jsonwebtoken@8.5.9': dependencies: - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/keygrip@1.0.6': {} @@ -4620,10 +4626,14 @@ snapshots: '@types/http-errors': 2.0.4 '@types/keygrip': 1.0.6 '@types/koa-compose': 3.2.8 - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/mime@1.3.5': {} + '@types/node@22.15.3': + dependencies: + undici-types: 6.21.0 + '@types/node@22.7.4': dependencies: undici-types: 6.19.8 @@ -4646,12 +4656,12 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 22.7.4 + '@types/node': 22.15.3 '@types/send': 0.17.4 '@types/tiny-async-pool@1.0.5': {} @@ -4786,7 +4796,7 @@ snapshots: loupe: 3.1.2 tinyrainbow: 1.2.0 - '@zip.js/zip.js@2.7.52': {} + '@zip.js/zip.js@2.7.60': {} accepts@1.3.8: dependencies: @@ -6564,6 +6574,8 @@ snapshots: undici-types@6.19.8: {} + undici-types@6.21.0: {} + universalify@2.0.1: {} unpipe@1.0.0: {}