Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/utils/chunk-find/chunk-find.util.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
title: chunkFind
category: utils
see: ["utils/iterateFind"]
---
118 changes: 118 additions & 0 deletions src/utils/chunk-find/chunk-find.util.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import type { Params } from '@feathersjs/feathers'
import { feathers } from '@feathersjs/feathers'
import { MemoryService } from '@feathersjs/memory'
import { chunkFind } from './chunk-find.util.js'

type User = {
id: number
name: string
}

const setup = async () => {
const app = feathers<{
users: MemoryService<
User,
Partial<User>,
Params<{ id: number; name: string }>
>
}>()

app.use(
'users',
new MemoryService({
id: 'id',
startId: 1,
multi: true,
paginate: {
default: 10,
max: 100,
},
}),
)

const usersService = app.service('users')

for (let i = 1; i <= 100; i++) {
await usersService.create({ name: `test${i}` })
}

return { app, usersService }
}

describe('chunkFind', function () {
it('basic usage', async function () {
const { app } = await setup()

const chunks = []

for await (const chunk of chunkFind(app, 'users')) {
chunks.push(chunk)
}

expect(chunks).toHaveLength(10)
expect(chunks[0]).toHaveLength(10)
expect(chunks[0]![0]!.name).toBe('test1')
expect(chunks[9]![9]!.name).toBe('test100')
})

it('can skip items', async function () {
const { app } = await setup()

const chunks = []

for await (const chunk of chunkFind(app, 'users', {
params: { query: { $skip: 20 } },
})) {
chunks.push(chunk)
}

expect(chunks).toHaveLength(8)
expect(chunks[0]![0]!.name).toBe('test21')
})

it('can set chunk size via $limit', async function () {
const { app } = await setup()

const chunks = []

for await (const chunk of chunkFind(app, 'users', {
params: { query: { $limit: 25 } },
})) {
chunks.push(chunk)
}

expect(chunks).toHaveLength(4)
expect(chunks[0]).toHaveLength(25)
expect(chunks[3]).toHaveLength(25)
})

it('can query for items', async function () {
const { app } = await setup()

const chunks = []

for await (const chunk of chunkFind(app, 'users', {
params: { query: { name: 'test1' } },
})) {
chunks.push(chunk)
}

expect(chunks).toHaveLength(1)
expect(chunks[0]).toEqual([expect.objectContaining({ name: 'test1' })])
})

it("ignores paginate:false and always paginates", async function () {
const { app } = await setup()

const chunks = []

for await (const chunk of chunkFind(app, 'users', {
params: { query: { name: 'test1' }, paginate: false },
})) {
chunks.push(chunk)
}

expect(chunks).toHaveLength(1)
expect(chunks[0]).toEqual([expect.objectContaining({ name: 'test1' })])
});
})
73 changes: 73 additions & 0 deletions src/utils/chunk-find/chunk-find.util.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import type { Application, Params } from '@feathersjs/feathers'
import type { KeyOf } from '../../internal.utils.js'
import type {
InferFindParams,
InferFindResultSingle,
} from '../../utility-types/infer-service-methods.js'

type ChunkFindOptions<P extends Params = Params> = {
params?: P
}

/**
* Use `for await` to iterate over chunks (pages) of results from a `find` method.
*
* This function is useful for processing large datasets in batches without loading everything into memory at once.
* It uses pagination to fetch results in chunks, yielding each page's data array.
*
* @example
* ```ts
* import { chunkFind } from 'feathers-utils/utils'
*
* const app = feathers()
*
* // Assuming 'users' service has many records
* for await (const users of chunkFind(app, 'users', {
* params: { query: { active: true }, // Custom query parameters
* } })) {
* console.log(users) // Process each chunk of user records
* }
* ```
*
* @see https://utils.feathersjs.com/utils/chunk-find.html
*/
export async function* chunkFind<
Services,
Path extends KeyOf<Services>,
Service extends Services[Path] = Services[Path],
P extends Params = InferFindParams<Service>,
Item = InferFindResultSingle<Service>,
>(
app: Application<Services>,
servicePath: Path,
options?: ChunkFindOptions<P>,
): AsyncGenerator<Item[], void, unknown> {
const service = app.service(servicePath)

if (!service || !('find' in service)) {
throw new Error(`Service '${servicePath}' does not have a 'find' method.`)
}

const params = {
...options?.params,
query: {
...(options?.params?.query ?? {}),
$limit: options?.params?.query?.$limit ?? 10,
$skip: options?.params?.query?.$skip ?? 0,
},
paginate: {
default: options?.params?.paginate?.default ?? 10,
max: options?.params?.paginate?.max ?? 100,
},
}

let result

do {
result = await (service as any).find(params)

yield result.data

params.query.$skip = (params.query.$skip ?? 0) + result.data.length
} while (result.total > params.query.$skip)
}
1 change: 1 addition & 0 deletions src/utils/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export * from './add-skip/add-skip.util.js'
export * from './chunk-find/chunk-find.util.js'
export * from './add-to-query/add-to-query.util.js'
export * from './check-context/check-context.util.js'
export * from './context-to-json/context-to-json.util.js'
Expand Down
1 change: 1 addition & 0 deletions src/utils/iterate-find/iterate-find.util.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
---
title: iterateFind
category: utils
see: ["utils/chunkFind"]
---
48 changes: 42 additions & 6 deletions src/utils/iterate-find/iterate-find.util.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ type User = {
name: string
}

const length = 1000;
const max = 100;

const setup = async () => {
const app = feathers<{
users: MemoryService<
Expand All @@ -25,16 +28,16 @@ const setup = async () => {
multi: true,
paginate: {
default: 10,
max: 100,
max,
},
}),
)

const usersService = app.service('users')

for (let i = 1; i <= 100; i++) {
await usersService.create({ name: `test${i}` })
}
const usersToCreate = Array.from({ length }).map((_, i) => ({ name: `test${i + 1}` }))

await usersService.create(usersToCreate)

return { app, usersService }
}
Expand All @@ -50,7 +53,7 @@ describe('iterateFind', function () {
}

expect(userNames).toEqual(
Array.from({ length: 100 }).map((_, i) => `test${i + 1}`),
Array.from({ length }).map((_, i) => `test${i + 1}`),
)
})

Expand All @@ -66,7 +69,7 @@ describe('iterateFind', function () {
}

expect(userNames).toEqual(
Array.from({ length: 80 }).map((_, i) => `test${i + 21}`),
Array.from({ length: length - 20 }).map((_, i) => `test${i + 21}`),
)
})

Expand All @@ -83,4 +86,37 @@ describe('iterateFind', function () {

expect(userNames).toEqual(['test1'])
})

it("ignores paginate:false and always paginates", async function () {
const { app } = await setup()

const userNames = []

for await (const user of iterateFind(app, 'users', {
params: { query: { name: 'test1' }, paginate: false },
})) {
userNames.push(user.name)
}

expect(userNames).toEqual(['test1'])
});

it("works with max", async function () {
const { app } = await setup()

expect(max + 10).toBeLessThan(length)

const userNames = []

for await (const user of iterateFind(app, 'users', {
params: { query: { $limit: max + 10 } },
})) {
userNames.push(user.name)
}

expect(userNames).toEqual(
Array.from({ length }).map((_, i) => `test${i + 1}`),
)
});

})
5 changes: 4 additions & 1 deletion src/utils/iterate-find/iterate-find.util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,12 @@ export async function* iterateFind<
...options?.params,
query: {
...(options?.params?.query ?? {}),
$limit: options?.params?.query?.$limit ?? 10,
$limit: options?.params?.query?.$limit,
$skip: options?.params?.query?.$skip ?? 0,
},
paginate: {
default: options?.params?.paginate?.default ?? 10,
},
}

let result
Expand Down
1 change: 1 addition & 0 deletions test/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const utils = [
'addSkip',
'addToQuery',
'checkContext',
'chunkFind',
'contextToJson',
'defineHooks',
'getDataIsArray',
Expand Down
Loading