From 5285518562f54e62fab65533cf30378ae7e89451 Mon Sep 17 00:00:00 2001 From: Jacob Fletcher Date: Tue, 11 Mar 2025 13:29:49 -0400 Subject: [PATCH] feat: defaults to noindex nofollow (#11623) We now have the ability to define all page metadata for the admin panel via the Payload Config as a result of #11593. This means we can now set sensible defaults for additional properties, e.g. `noindex` and `nofollow` on the `robots` property. Setting this will prevent these pages from being indexed and appearing in search results. Note that setting this property prevents _indexing_ these pages, but does not prevent them from being _crawled_. To prevent crawling as well, you must add a standalone `robots.txt` file to your root directory. --- docs/admin/metadata.mdx | 44 +++++++++++++++++++++++-- packages/payload/src/config/defaults.ts | 2 ++ test/admin/config.ts | 1 - test/admin/e2e/general/e2e.spec.ts | 10 ++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/admin/metadata.mdx b/docs/admin/metadata.mdx index e0b3f01d63..dac6b73fc2 100644 --- a/docs/admin/metadata.mdx +++ b/docs/admin/metadata.mdx @@ -53,7 +53,6 @@ The following options are available for Root Metadata: | Key | Type | Description | | --- | --- | --- | | `defaultOGImageType` | `dynamic` (default), `static`, or `off` | The type of default OG image to use. If set to `dynamic`, Payload will use Next.js image generation to create an image with the title of the page. If set to `static`, Payload will use the `defaultOGImage` URL. If set to `off`, Payload will not generate an OG image. | -| `icons` | `IconConfig[]` | An array of icon objects. [More details](#icons). | | `titleSuffix` | `string` | A suffix to append to the end of the title of every page. Defaults to "- Payload". | | `[keyof Metadata]` | `unknown` | Any other properties that Next.js supports within the `generateMetadata` function. [More details](https://nextjs.org/docs/app/api-reference/functions/generate-metadata). | @@ -68,7 +67,7 @@ The Icons Config corresponds to the `` tags that are used to specify icons The most common icon type is the favicon, which is displayed in the browser tab. This is specified by the `rel` attribute `icon`. Other common icon types include `apple-touch-icon`, which is used by Apple devices when the Admin Panel is saved to the home screen, and `mask-icon`, which is used by Safari to mask the Admin Panel icon. -To customize icons, use the `icons` key within the `admin.meta` object in your Payload Config: +To customize icons, use the `admin.meta.icons` property in your Payload Config: ```ts { @@ -100,7 +99,7 @@ For a full list of all available Icon options, see the [Next.js documentation](h Open Graph metadata is a set of tags that are used to control how URLs are displayed when shared on social media platforms. Open Graph metadata is automatically generated by Payload, but can be customized at the Root level. -To customize Open Graph metadata, use the `openGraph` key within the `admin.meta` object in your Payload Config: +To customize Open Graph metadata, use the `admin.meta.openGraph` property in your Payload Config: ```ts { @@ -128,6 +127,45 @@ To customize Open Graph metadata, use the `openGraph` key within the `admin.meta For a full list of all available Open Graph options, see the [Next.js documentation](https://nextjs.org/docs/app/api-reference/functions/generate-metadata#opengraph). +### Robots + +Setting the `robots` property will allow you to control the `robots` meta tag that is rendered within the `` of the Admin Panel. This can be used to control how search engines index pages and displays them in search results. + +By default, the Admin Panel is set to prevent search engines from indexing pages within the Admin Panel. + +To customize the Robots Config, use the `admin.meta.robots` property in your Payload Config: + +```ts +{ + // ... + admin: { + meta: { + // highlight-start + robots: 'noindex, nofollow', + // highlight-end + }, + }, +} +``` + +For a full list of all available Robots options, see the [Next.js documentation](https://nextjs.org/docs/app/api-reference/functions/generate-metadata#robots). + +##### Prevent Crawling + +While setting meta tags via `admin.meta.robots` can prevent search engines from _indexing_ web pages, it does not prevent them from being _crawled_. + +To prevent your pages from being crawled altogether, add a `robots.txt` file to your root directory. + +```txt +User-agent: * +Disallow: /admin/ +``` + + + **Note:** + If you've customized the path to your Admin Panel via `config.routes`, be sure to update the `Disallow` directive to match your custom path. + + ## Collection Metadata Collection Metadata is the metadata that is applied to all pages within any given Collection within the Admin Panel. This metadata is used to customize the title and description of all views within any given Collection, unless overridden by the view itself. diff --git a/packages/payload/src/config/defaults.ts b/packages/payload/src/config/defaults.ts index 12761de1a1..a22d7d9afe 100644 --- a/packages/payload/src/config/defaults.ts +++ b/packages/payload/src/config/defaults.ts @@ -18,6 +18,7 @@ export const defaults: Omit = { }, meta: { defaultOGImageType: 'dynamic', + robots: 'noindex, nofollow', titleSuffix: '- Payload', }, routes: { @@ -91,6 +92,7 @@ export const addDefaultsToConfig = (config: Config): Config => { }, meta: { defaultOGImageType: 'dynamic', + robots: 'noindex, nofollow', titleSuffix: '- Payload', ...(config?.admin?.meta || {}), }, diff --git a/test/admin/config.ts b/test/admin/config.ts index 15cac3d378..7821de2ba4 100644 --- a/test/admin/config.ts +++ b/test/admin/config.ts @@ -129,7 +129,6 @@ export default buildConfigWithDefaults({ description: 'This is a custom OG description', title: 'This is a custom OG title', }, - robots: 'nofollow noindex', titleSuffix: '- Custom Title Suffix', }, routes: customAdminRoutes, diff --git a/test/admin/e2e/general/e2e.spec.ts b/test/admin/e2e/general/e2e.spec.ts index 9241ef8a3b..1c54289965 100644 --- a/test/admin/e2e/general/e2e.spec.ts +++ b/test/admin/e2e/general/e2e.spec.ts @@ -158,6 +158,16 @@ describe('General', () => { }) }) + describe('robots', () => { + test('should apply default robots meta tag', async () => { + await page.goto(`${serverURL}/admin`) + await expect(page.locator('meta[name="robots"]')).toHaveAttribute( + 'content', + /noindex, nofollow/, + ) + }) + }) + describe('favicons', () => { test('should render custom favicons', async () => { await page.goto(postsUrl.admin)