Files
payload/test/plugin-search/int.spec.ts
Said Akhrarov defa13e4fe feat(plugin-search): added support for reindexing collections on demand (#9391)
### What?
This PR aims to add reindexing capabilities to `plugin-search` to allow
users to reindex entire searchable collections on demand.

### Why?
As it stands, end users must either perform document reindexing manually
one-by-one or via bulk operations. Both of these approaches are
undesirable because they result in new versions being published on
existing documents. Consider the case when `plugin-search` is only added
_after_ the project has started and documents have been added to
existing collections. It would be nice if users could simply click a
button, choose the searchable collections to reindex, and have the
custom endpoint handle the rest.

### How?
This PR adds on to the existing plugin configuration, creating a custom
endpoint and a custom `beforeListTable` component in the form of a popup
button. Upon clicking the button, a dropdown/popup is opened with
options to select which collection to reindex, as well as a useful `All
Collections` option to run reindexing on all configured search
collections. It also adds a `reindexBatchSize` option in the config to
allow users to specify in what quantity to batch documents to sync with
search.

Big shoutout to @paulpopus & @r1tsuu for the triple-A level support on
this one!

Fixes #8902 

See it in action:


https://github.com/user-attachments/assets/ee8dd68c-ea89-49cd-adc3-151973eea28b

Notes:
- Traditionally these kinds of long-running tasks would be better suited
for a job. However, given how many users enjoy deploying to serverless
environments, it would be problematic to offer this feature exclusive to
jobs queues. I thought a significant amount about this and decided it
would be best to ship the feature as-is with the intention of creating
an opt-in method to use job queues in the future if/when this gets
merged.
- In my testing, the collection description somehow started to appear in
the document views after the on-demand RSC merge. I haven't reproduced
this, but this PR has an example of that problem. Super strange.

---------

Co-authored-by: Sasha <64744993+r1tsuu@users.noreply.github.com>
Co-authored-by: Paul Popus <paul@nouance.io>
2024-11-26 23:14:31 +00:00

425 lines
9.9 KiB
TypeScript

import path from 'path'
import { NotFound, type Payload } from 'payload'
import { wait } from 'payload/shared'
import { fileURLToPath } from 'url'
import type { NextRESTClient } from '../helpers/NextRESTClient.js'
import { devUser } from '../credentials.js'
import { initPayloadInt } from '../helpers/initPayloadInt.js'
import { pagesSlug, postsSlug } from './shared.js'
let payload: Payload
let restClient: NextRESTClient
let token: string
const filename = fileURLToPath(import.meta.url)
const dirname = path.dirname(filename)
describe('@payloadcms/plugin-search', () => {
beforeAll(async () => {
;({ payload, restClient } = await initPayloadInt(dirname))
const data = await restClient
.POST('/users/login', {
body: JSON.stringify({
email: devUser.email,
password: devUser.password,
}),
})
.then((res) => res.json())
token = data.token
})
beforeEach(async () => {
await payload.delete({
collection: 'search',
depth: 0,
where: {
id: {
exists: true,
},
},
})
await Promise.all([
payload.delete({
collection: postsSlug,
depth: 0,
where: {
id: {
exists: true,
},
},
}),
payload.delete({
collection: pagesSlug,
depth: 0,
where: {
id: {
exists: true,
},
},
}),
])
})
afterAll(async () => {
if (typeof payload.db.destroy === 'function') {
await payload.db.destroy()
}
})
it('should add a search collection', async () => {
const search = await payload.find({
collection: 'search',
depth: 0,
limit: 1,
})
expect(search).toBeTruthy()
})
it('should sync published pages to the search collection', async () => {
const pageToSync = await payload.create({
collection: 'pages',
data: {
_status: 'published',
excerpt: 'This is a test page',
title: 'Hello, world!',
},
})
const { docs: results } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: pageToSync.id,
},
},
})
expect(results).toHaveLength(1)
expect(results[0].doc.value).toBe(pageToSync.id)
expect(results[0].title).toBe('Hello, world!')
expect(results[0].excerpt).toBe('This is a test page')
})
it('should not sync drafts pages to the search collection', async () => {
const draftPage = await payload.create({
collection: 'pages',
data: {
_status: 'draft',
excerpt: 'This is a test page',
title: 'Hello, world!',
},
})
// wait for the search document to be potentially created
// we do not await this within the `syncToSearch` hook
await wait(200)
const { docs: results } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: draftPage.id,
},
},
})
expect(results).toHaveLength(0)
})
it('should sync changes made to an existing search document', async () => {
const pageToReceiveUpdates = await payload.create({
collection: 'pages',
data: {
_status: 'published',
excerpt: 'This is a test page',
title: 'Hello, world!',
},
})
const { docs: results } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: pageToReceiveUpdates.id,
},
},
})
expect(results).toHaveLength(1)
expect(results[0].doc.value).toBe(pageToReceiveUpdates.id)
expect(results[0].title).toBe('Hello, world!')
expect(results[0].excerpt).toBe('This is a test page')
await payload.update({
id: pageToReceiveUpdates.id,
collection: 'pages',
data: {
excerpt: 'This is a test page (updated)',
title: 'Hello, world! (updated)',
},
})
// wait for the search document to be potentially updated
// we do not await this within the `syncToSearch` hook
await wait(200)
// Do not add `limit` to this query, this way we can test if multiple documents were created
const { docs: updatedResults } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: pageToReceiveUpdates.id,
},
},
})
expect(updatedResults).toHaveLength(1)
expect(updatedResults[0].doc.value).toBe(pageToReceiveUpdates.id)
expect(updatedResults[0].title).toBe('Hello, world! (updated)')
expect(updatedResults[0].excerpt).toBe('This is a test page (updated)')
})
it('should clear the search document when the original document is deleted', async () => {
const page = await payload.create({
collection: 'pages',
data: {
_status: 'published',
excerpt: 'This is a test page',
title: 'Hello, world!',
},
})
// wait for the search document to be created
// we do not await this within the `syncToSearch` hook
await wait(200)
const { docs: results } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: page.id,
},
},
})
expect(results).toHaveLength(1)
expect(results[0].doc.value).toBe(page.id)
await payload.delete({
id: page.id,
collection: 'pages',
})
// wait for the search document to be potentially deleted
// we do not await this within the `syncToSearch` hook
await wait(200)
const { docs: deletedResults } = await payload.find({
collection: 'search',
depth: 0,
where: {
'doc.value': {
equals: page.id,
},
},
})
expect(deletedResults).toHaveLength(0)
})
it('should sync localized data', async () => {
const createdDoc = await payload.create({
collection: 'posts',
data: {
_status: 'published',
title: 'test title',
slug: 'es',
},
locale: 'es',
})
await payload.update({
collection: 'posts',
id: createdDoc.id,
data: {
_status: 'published',
title: 'test title',
slug: 'en',
},
locale: 'en',
})
const syncedSearchData = await payload.find({
collection: 'search',
locale: 'es',
where: {
and: [
{
'doc.value': {
equals: createdDoc.id,
},
},
],
},
})
expect(syncedSearchData.docs[0].slug).toEqual('es')
})
it('should respond with 401 when invalid permissions on user before reindex', async () => {
const testCreds = {
email: 'test@payloadcms.com',
password: 'test',
}
await payload.create({
collection: 'users',
data: testCreds,
})
const testUserRes = await restClient.POST(`/users/login`, {
body: JSON.stringify(testCreds),
})
const testUser = await testUserRes.json()
const endpointRes = await restClient.POST(`/search/reindex`, {
body: JSON.stringify({
collections: [postsSlug],
}),
headers: {
Authorization: `JWT ${testUser.token}`,
},
})
expect(endpointRes.status).toEqual(401)
})
it('should respond with 400 when invalid collection args passed to reindex', async () => {
const endpointNoArgsRes = await restClient.POST(`/search/reindex`, {
body: JSON.stringify({}),
headers: {
Authorization: `JWT ${token}`,
},
})
const endpointEmptyArrRes = await restClient.POST(`/search/reindex`, {
body: JSON.stringify({
collections: [],
}),
headers: {
Authorization: `JWT ${token}`,
},
})
const endpointInvalidArrRes = await restClient.POST(`/search/reindex`, {
body: JSON.stringify({
collections: ['users'],
}),
headers: {
Authorization: `JWT ${token}`,
},
})
expect(endpointNoArgsRes.status).toBe(400)
expect(endpointEmptyArrRes.status).toBe(400)
expect(endpointInvalidArrRes.status).toBe(400)
})
it('should delete existing search indexes before reindexing', async () => {
await payload.create({
collection: postsSlug,
data: {
title: 'post_1',
_status: 'published',
},
})
await wait(200)
await payload.create({
collection: postsSlug,
data: {
title: 'post_2',
_status: 'published',
},
})
const { docs } = await payload.find({ collection: 'search' })
await wait(200)
const endpointRes = await restClient.POST('/search/reindex', {
body: JSON.stringify({
collections: [postsSlug, pagesSlug],
}),
})
expect(endpointRes.status).toBe(200)
const { docs: results } = await payload.find({
collection: 'search',
depth: 0,
where: {
id: {
in: docs.map((doc) => doc.id),
},
},
})
// Should have no docs with these ID
// after reindex since it deletes indexes and recreates them
expect(results).toHaveLength(0)
})
it('should reindex whole collections', async () => {
await payload.create({
collection: pagesSlug,
data: {
title: 'Test page title',
_status: 'published',
},
})
await payload.create({
collection: postsSlug,
data: {
title: 'Test page title',
_status: 'published',
},
})
await wait(200)
const { totalDocs: totalBeforeReindex } = await payload.count({
collection: 'search',
})
const endpointRes = await restClient.POST(`/search/reindex`, {
body: JSON.stringify({
collections: [postsSlug, pagesSlug],
}),
headers: {
Authorization: `JWT ${token}`,
},
})
expect(endpointRes.status).toBe(200)
const { totalDocs: totalAfterReindex } = await payload.count({
collection: 'search',
})
expect(totalAfterReindex).toBe(totalBeforeReindex)
})
})