feat: optimize addList method to handle large content_list

This commit is contained in:
monoid 2024-09-28 23:52:31 +09:00
parent e5d410d809
commit 2a5cb909b5
1 changed files with 14 additions and 0 deletions

View File

@ -21,6 +21,20 @@ class SqliteDocumentAccessor implements DocumentAccessor {
throw new Error("Method not implemented.");
}
async addList(content_list: DocumentBody[]): Promise<number[]> {
if (content_list.length === 0) return [];
if (content_list.length > 256) {
// if content_list is too long, split it.
const chunkLength = 256;
const chunked = [];
for (let i = 0; i < content_list.length; i += chunkLength) {
chunked.push(content_list.slice(i, i + chunkLength));
}
const ids = [];
for (const chunk of chunked) {
ids.push(await this.addList(chunk));
}
return ids.flat();
}
return await this.kysely.transaction().execute(async (trx) => {
// add tags
const tagCollected = new Set<string>();