migrate to root dir
This commit is contained in:
11
lib/pipeline/chunker.ts
Normal file
11
lib/pipeline/chunker.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
export function chunkText(text: string, chunkSize: number): string[] {
|
||||
const words = text.split(/\s+/);
|
||||
const chunks: string[] = [];
|
||||
|
||||
for (let i = 0; i < words.length; i += chunkSize) {
|
||||
const chunk = words.slice(i, i + chunkSize).join(" ");
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
Reference in New Issue
Block a user