From 385391334a218521a5aa14e09be9534669ac7fab Mon Sep 17 00:00:00 2001 From: monoid Date: Fri, 29 Mar 2024 02:14:53 +0900 Subject: [PATCH] use zip.js --- packages/server/package.json | 2 +- packages/server/src/route/comic.ts | 68 ++++++++++++++----------- packages/server/src/util/zipwrap.ts | 78 +++++++++++++++++++---------- 3 files changed, 93 insertions(+), 55 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index d24bf18..43a615f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -5,7 +5,7 @@ "main": "build/app.js", "scripts": { "compile": "swc src --out-dir dist", - "dev": "nodemon -r @swc-node/register --exec node app.ts", + "dev": "nodemon -r @swc-node/register --enable-source-maps --exec node app.ts", "start": "node build/app.js" }, "author": "", diff --git a/packages/server/src/route/comic.ts b/packages/server/src/route/comic.ts index 8606268..bac0012 100644 --- a/packages/server/src/route/comic.ts +++ b/packages/server/src/route/comic.ts @@ -1,26 +1,33 @@ import { type Context, DefaultContext, DefaultState, Next } from "koa"; import Router from "koa-router"; -import { createReadableStreamFromZip, entriesByNaturalOrder, readZip, type ZipAsync } from "../util/zipwrap"; +import { createReadableStreamFromZip, entriesByNaturalOrder, readZip } from "../util/zipwrap"; import type { ContentContext } from "./context"; import { since_last_modified } from "./util"; +import type { ZipReader } from "@zip.js/zip.js"; +import type { FileHandle } from "node:fs/promises"; +import { Readable, Writable } from "node:stream"; /** * zip stream cache. */ - -const ZipStreamCache: { [path: string]: [ZipAsync, number] } = {}; +const ZipStreamCache: { + [path: string]: [{ + reader: ZipReader, + handle: FileHandle + }, number] +} = {}; async function acquireZip(path: string) { if (!(path in ZipStreamCache)) { - const ret = await readZip(path); - ZipStreamCache[path] = [ret, 1]; + const obj = await readZip(path); + ZipStreamCache[path] = [obj, 1]; // console.log(`acquire ${path} 1`); - return ret; + return obj.reader; } - const [ret, refCount] = ZipStreamCache[path]; - ZipStreamCache[path] = [ret, refCount + 1]; - // console.log(`acquire ${path} ${refCount + 1}`); - return ret; + const [ret, refCount] = ZipStreamCache[path]; + ZipStreamCache[path] = [ret, refCount + 1]; + // console.log(`acquire ${path} ${refCount + 1}`); + return ret.reader; } function releaseZip(path: string) { @@ -29,7 +36,10 @@ function releaseZip(path: string) { const [ref, refCount] = obj; // console.log(`release ${path} : ${refCount}`); if (refCount === 1) { - ref.close(); + const { reader, handle } = ref; + reader.close().then(() => { + handle.close(); + }); delete ZipStreamCache[path]; } else { ZipStreamCache[path] = [ref, refCount - 1]; @@ -41,33 +51,35 @@ async function renderZipImage(ctx: Context, path: string, page: number) { // console.log(`opened ${page}`); const zip = await acquireZip(path); const entries = (await entriesByNaturalOrder(zip)).filter((x) => { - const ext = x.name.split(".").pop(); + const ext = x.filename.split(".").pop(); return ext !== undefined && image_ext.includes(ext); }); if (0 <= page && page < entries.length) { const entry = entries[page]; - const last_modified = new Date(entry.time); + const last_modified = entry.lastModDate; if (since_last_modified(ctx, last_modified)) { return; } const read_stream = await createReadableStreamFromZip(zip, entry); - /** Exceptions (ECONNRESET, ECONNABORTED) may be thrown when processing this request - * for reasons such as when the browser unexpectedly closes the connection. - * Once such an exception is raised, the stream is not properly destroyed, - * so there is a problem with the zlib stream being accessed even after the stream is closed. - * So it waits for 100 ms and releases it. - * Additionaly, there is a risk of memory leak becuase zlib stream is not properly destroyed. - * @todo modify function 'stream' in 'node-stream-zip' library to prevent memory leak */ - read_stream.once("close", () => { - setTimeout(() => { - releaseZip(path); - }, 100); - }); + const nodeReadableStream = new Readable(); + nodeReadableStream._read = () => { }; - ctx.body = read_stream; - ctx.response.length = entry.size; + read_stream.pipeTo(new WritableStream({ + write(chunk) { + nodeReadableStream.push(chunk); + }, + close() { + nodeReadableStream.push(null); + setTimeout(() => { + releaseZip(path); + }, 100); + }, + })); + + ctx.body = nodeReadableStream; + ctx.response.length = entry.uncompressedSize; // console.log(`${entry.name}'s ${page}:${entry.size}`); - ctx.response.type = entry.name.split(".").pop() as string; + ctx.response.type = entry.filename.split(".").pop() as string; ctx.status = 200; ctx.set("Date", new Date().toUTCString()); ctx.set("Last-Modified", last_modified.toUTCString()); diff --git a/packages/server/src/util/zipwrap.ts b/packages/server/src/util/zipwrap.ts index d1f6164..ac6c5a4 100644 --- a/packages/server/src/util/zipwrap.ts +++ b/packages/server/src/util/zipwrap.ts @@ -1,33 +1,59 @@ -import type { ZipEntry } from "node-stream-zip"; - -import { ReadStream } from "node:fs"; +import { type FileHandle, open } from "node:fs/promises"; import { orderBy } from "natural-orderby"; -import StreamZip from "node-stream-zip"; +import { ZipReader, Reader, type Entry } from "@zip.js/zip.js"; -export type ZipAsync = InstanceType; -export async function readZip(path: string): Promise { - return new StreamZip.async({ - file: path, - storeEntries: true, - }); +class FileReader extends Reader { + private fd: FileHandle; + private offset: number; + constructor(fd: FileHandle) { + super(fd); + this.fd = fd; + this.offset = 0; + } + + async init(): Promise { + this.offset = 0; + this.size = (await this.fd.stat()).size; + } + close(): void { + this.fd.close(); + } + + async readUint8Array(index: number, length: number): Promise { + const buffer = new Uint8Array(length); + const buf = await this.fd.read(buffer, 0, length, index); + if (buf.bytesRead !== length) { + console.error(`read error: ${buf.bytesRead} !== ${length}`); + throw new Error("read error"); + } + return buffer; + } } -export async function entriesByNaturalOrder(zip: ZipAsync) { - const entries = await zip.entries(); - const ret = orderBy(Object.values(entries), (v) => v.name); + +export async function readZip(path: string): Promise<{ + reader: ZipReader + handle: FileHandle +}> { + const fd = await open(path); + const reader = new ZipReader(new FileReader(fd), { + useCompressionStream: true, + preventClose: false, + }); + return { reader, handle: fd }; +} +export async function entriesByNaturalOrder(zip: ZipReader) { + // console.log(zip); + const entries = await zip.getEntries(); + // console.log(entries.map((v) => v.filename)); + const ret = orderBy(entries, (v) => v.filename); return ret; } -export async function createReadableStreamFromZip(zip: ZipAsync, entry: ZipEntry): Promise { - return await zip.stream(entry); -} -export async function readAllFromZip(zip: ZipAsync, entry: ZipEntry): Promise { - const stream = await createReadableStreamFromZip(zip, entry); - const chunks: Uint8Array[] = []; - return new Promise((resolve, reject) => { - stream.on("data", (data) => { - chunks.push(data); - }); - stream.on("error", (err) => reject(err)); - stream.on("end", () => resolve(Buffer.concat(chunks))); - }); +export async function createReadableStreamFromZip(_zip: ZipReader, entry: Entry): Promise { + if (entry.getData === undefined) { + throw new Error("entry.getData is undefined"); + } + const stream = new TransformStream(); + entry.getData(stream.writable); + return stream.readable; }