feat: defaults to noindex nofollow (#11623)

We now have the ability to define all page metadata for the admin panel
via the Payload Config as a result of #11593. This means we can now set
sensible defaults for additional properties, e.g. `noindex` and
`nofollow` on the `robots` property. Setting this will prevent these
pages from being indexed and appearing in search results.

Note that setting this property prevents _indexing_ these pages, but
does not prevent them from being _crawled_. To prevent crawling as well,
you must add a standalone `robots.txt` file to your root directory.

test/admin/config.ts

@@ -129,7 +129,6 @@ export default buildConfigWithDefaults({
         description: 'This is a custom OG description',
         title: 'This is a custom OG title',
       },
-      robots: 'nofollow noindex',
       titleSuffix: '- Custom Title Suffix',
     },
     routes: customAdminRoutes,

test/admin/e2e/general/e2e.spec.ts

@@ -158,6 +158,16 @@ describe('General', () => {
       })
     })
 
+    describe('robots', () => {
+      test('should apply default robots meta tag', async () => {
+        await page.goto(`${serverURL}/admin`)
+        await expect(page.locator('meta[name="robots"]')).toHaveAttribute(
+          'content',
+          /noindex, nofollow/,
+        )
+      })
+    })
+
     describe('favicons', () => {
       test('should render custom favicons', async () => {
         await page.goto(postsUrl.admin)