import { Database } from "jsr:@db/sqlite"; import {join} from "jsr:@std/path"; let db = new Database("./db.sqlite3") const stmt = db.prepare("SELECT id, basepath, filename from document"); let ds = [...stmt.all()]; async function oshash( path: string ){ const chunkSize = 4096; const minFileSize = chunkSize * 2; const fd = await Deno.open(path); const st = await fd.stat(); let hash = BigInt(st.size); if (st.size < minFileSize){ throw new Error("File is too small to hash"); } // read first and last chunk const firstChunk = new Uint8Array(chunkSize); await fd.read(firstChunk, 0, chunkSize, 0); const lastChunk = new Uint8Array(chunkSize); await fd.read(lastChunk, 0, chunkSize, st.size - chunkSize); // iterate over first and last chunk. // for each uint64_t, add it to the hash. const firstChunkView = new DataView(firstChunk.buffer); for (let i = 0; i < chunkSize; i += 8){ hash += firstChunkView.getBigUint64(i, true); // prevent overflow hash = (hash & 0xFFFFFFFFFFFFFFFFn); } const lastChunkView = new DataView(lastChunk.buffer); for (let i = 0; i < chunkSize; i += 8){ hash += lastChunkView.getBigUint64(i, true); // prevent overflow hash = (hash & 0xFFFFFFFFFFFFFFFFn); } return hash; } async function updateHash(ds: {id: number, basepath: string, filename: string}[]) { const content_hashs = await Promise.all(ds.map(async (d) => { const p = join(d.basepath, d.filename); return await oshash(p); })); db.transaction(() => { for (let i = 0; i < ds.length; i++) { db.run(`UPDATE document SET content_hash = ? where id = ?`, content_hashs[i].toString(), ds[i].id) } })(); } for (let i = 0; i < ds.length; i += 32) { const d = ds.slice(i, i + 32); console.log(d.map(x => x.id)); await updateHash(d); } db.close();