From 583a73333487c5ba466e8de97a98d9eafe78dbba Mon Sep 17 00:00:00 2001 From: Alessio Gravili Date: Wed, 2 Jul 2025 09:24:53 -0700 Subject: [PATCH] feat(drizzle): support half-precision, binary, and sparse vectors column types (#12491) Adds support for `halfvec` and `sparsevec` and `bit` (binary vector) column types. This is required for supporting indexing of embeddings > 2000 dimensions on postgres using the pg-vector extension. --- .github/workflows/main.yml | 3 +- package.json | 6 +- packages/db-postgres/package.json | 4 +- packages/db-sqlite/package.json | 2 +- packages/db-vercel-postgres/package.json | 4 +- .../src/postgres/columnToCodeConverter.ts | 30 +- .../src/postgres/schema/buildDrizzleTable.ts | 26 ++ packages/drizzle/src/types.ts | 18 + pnpm-lock.yaml | 322 +++++++----------- test/database/postgres-vector.int.spec.ts | 277 +++++++++++++-- test/package.json | 4 +- 11 files changed, 463 insertions(+), 233 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 78379cc24..b4b465adb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,7 +187,8 @@ jobs: services: postgres: - image: ${{ (startsWith(matrix.database, 'postgres') ) && 'postgis/postgis:16-3.4' || '' }} + # Custom postgres 17 docker image that supports both pg-vector and postgis: https://github.com/payloadcms/postgis-vector + image: ${{ (startsWith(matrix.database, 'postgres') ) && 'ghcr.io/payloadcms/postgis-vector:latest' || '' }} env: # must specify password for PG Docker container image, see: https://registry.hub.docker.com/_/postgres?tab=description&page=1&name=10 POSTGRES_USER: ${{ env.POSTGRES_USER }} diff --git a/package.json b/package.json index a32ec8d5d..4254f8b6b 100644 --- a/package.json +++ b/package.json @@ -151,8 +151,8 @@ "create-payload-app": "workspace:*", "cross-env": "7.0.3", "dotenv": "16.4.7", - "drizzle-kit": "0.31.0", - "drizzle-orm": "0.43.1", + "drizzle-kit": "0.31.4", + "drizzle-orm": "0.44.2", "escape-html": "^1.0.3", "execa": "5.1.1", "form-data": "3.0.1", @@ -166,7 +166,7 @@ "next": "15.3.2", "open": "^10.1.0", "p-limit": "^5.0.0", - "pg": "8.11.3", + "pg": "8.16.3", "playwright": "1.50.0", "playwright-core": "1.50.0", "prettier": "3.5.3", diff --git a/packages/db-postgres/package.json b/packages/db-postgres/package.json index 35923cb48..7b32757ca 100644 --- a/packages/db-postgres/package.json +++ b/packages/db-postgres/package.json @@ -78,9 +78,9 @@ "@payloadcms/drizzle": "workspace:*", "@types/pg": "8.10.2", "console-table-printer": "2.12.1", - "drizzle-kit": "0.31.1", + "drizzle-kit": "0.31.4", "drizzle-orm": "0.44.2", - "pg": "8.11.3", + "pg": "8.16.3", "prompts": "2.4.2", "to-snake-case": "1.0.0", "uuid": "10.0.0" diff --git a/packages/db-sqlite/package.json b/packages/db-sqlite/package.json index 18b7065ad..76e09fb8b 100644 --- a/packages/db-sqlite/package.json +++ b/packages/db-sqlite/package.json @@ -76,7 +76,7 @@ "@libsql/client": "0.14.0", "@payloadcms/drizzle": "workspace:*", "console-table-printer": "2.12.1", - "drizzle-kit": "0.31.1", + "drizzle-kit": "0.31.4", "drizzle-orm": "0.44.2", "prompts": "2.4.2", "to-snake-case": "1.0.0", diff --git a/packages/db-vercel-postgres/package.json b/packages/db-vercel-postgres/package.json index 919e73360..5723d362a 100644 --- a/packages/db-vercel-postgres/package.json +++ b/packages/db-vercel-postgres/package.json @@ -78,9 +78,9 @@ "@payloadcms/drizzle": "workspace:*", "@vercel/postgres": "^0.9.0", "console-table-printer": "2.12.1", - "drizzle-kit": "0.31.1", + "drizzle-kit": "0.31.4", "drizzle-orm": "0.44.2", - "pg": "8.11.3", + "pg": "8.16.3", "prompts": "2.4.2", "to-snake-case": "1.0.0", "uuid": "10.0.0" diff --git a/packages/drizzle/src/postgres/columnToCodeConverter.ts b/packages/drizzle/src/postgres/columnToCodeConverter.ts index 2519c9ae5..5beafdced 100644 --- a/packages/drizzle/src/postgres/columnToCodeConverter.ts +++ b/packages/drizzle/src/postgres/columnToCodeConverter.ts @@ -24,20 +24,26 @@ export const columnToCodeConverter: ColumnToCodeConverter = ({ const columnBuilderArgsArray: string[] = [] - if (column.type === 'timestamp') { - columnBuilderArgsArray.push(`mode: '${column.mode}'`) - if (column.withTimezone) { - columnBuilderArgsArray.push('withTimezone: true') + switch (column.type) { + case 'bit': + case 'halfvec': + case 'sparsevec': + case 'vector': { + if (column.dimensions) { + columnBuilderArgsArray.push(`dimensions: ${column.dimensions}`) + } + break } + case 'timestamp': { + columnBuilderArgsArray.push(`mode: '${column.mode}'`) + if (column.withTimezone) { + columnBuilderArgsArray.push('withTimezone: true') + } - if (typeof column.precision === 'number') { - columnBuilderArgsArray.push(`precision: ${column.precision}`) - } - } - - if (column.type === 'vector') { - if (column.dimensions) { - columnBuilderArgsArray.push(`dimensions: ${column.dimensions}`) + if (typeof column.precision === 'number') { + columnBuilderArgsArray.push(`precision: ${column.precision}`) + } + break } } diff --git a/packages/drizzle/src/postgres/schema/buildDrizzleTable.ts b/packages/drizzle/src/postgres/schema/buildDrizzleTable.ts index 62599ddf3..97d15d98f 100644 --- a/packages/drizzle/src/postgres/schema/buildDrizzleTable.ts +++ b/packages/drizzle/src/postgres/schema/buildDrizzleTable.ts @@ -1,13 +1,16 @@ import type { ForeignKeyBuilder, IndexBuilder } from 'drizzle-orm/pg-core' import { + bit, boolean, foreignKey, + halfvec, index, integer, jsonb, numeric, serial, + sparsevec, text, timestamp, uniqueIndex, @@ -44,6 +47,14 @@ export const buildDrizzleTable = ({ for (const [key, column] of Object.entries(rawTable.columns)) { switch (column.type) { + case 'bit': { + const builder = bit(column.name, { dimensions: column.dimensions }) + + columns[key] = builder + + break + } + case 'enum': if ('locale' in column) { columns[key] = adapter.enums.enum__locales(column.name) @@ -56,6 +67,21 @@ export const buildDrizzleTable = ({ } break + case 'halfvec': { + const builder = halfvec(column.name, { dimensions: column.dimensions }) + + columns[key] = builder + break + } + + case 'sparsevec': { + const builder = sparsevec(column.name, { dimensions: column.dimensions }) + + columns[key] = builder + + break + } + case 'timestamp': { let builder = timestamp(column.name, { mode: column.mode, diff --git a/packages/drizzle/src/types.ts b/packages/drizzle/src/types.ts index 63493e5f1..42f01b7ce 100644 --- a/packages/drizzle/src/types.ts +++ b/packages/drizzle/src/types.ts @@ -281,12 +281,30 @@ export type VectorRawColumn = { type: 'vector' } & BaseRawColumn +export type HalfVecRawColumn = { + dimensions?: number + type: 'halfvec' +} & BaseRawColumn + +export type SparseVecRawColumn = { + dimensions?: number + type: 'sparsevec' +} & BaseRawColumn + +export type BinaryVecRawColumn = { + dimensions?: number + type: 'bit' +} & BaseRawColumn + export type RawColumn = | ({ type: 'boolean' | 'geometry' | 'jsonb' | 'numeric' | 'serial' | 'text' | 'varchar' } & BaseRawColumn) + | BinaryVecRawColumn | EnumRawColumn + | HalfVecRawColumn | IntegerRawColumn + | SparseVecRawColumn | TimestampRawColumn | UUIDRawColumn | VectorRawColumn diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0f4684af..d15eb9fd9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,7 +44,7 @@ importers: version: 1.50.0 '@sentry/nextjs': specifier: ^8.33.1 - version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)) '@sentry/node': specifier: ^8.33.1 version: 8.37.1 @@ -97,11 +97,11 @@ importers: specifier: 16.4.7 version: 16.4.7 drizzle-kit: - specifier: 0.31.0 - version: 0.31.0 + specifier: 0.31.4 + version: 0.31.4 drizzle-orm: - specifier: 0.43.1 - version: 0.43.1(@libsql/client@0.14.0(bufferutil@4.0.8))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(pg@8.11.3) + specifier: 0.44.2 + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -142,8 +142,8 @@ importers: specifier: ^5.0.0 version: 5.0.0 pg: - specifier: 8.11.3 - version: 8.11.3 + specifier: 8.16.3 + version: 8.16.3 playwright: specifier: 1.50.0 version: 1.50.0 @@ -319,14 +319,14 @@ importers: specifier: 2.12.1 version: 2.12.1 drizzle-kit: - specifier: 0.31.1 - version: 0.31.1 + specifier: 0.31.4 + version: 0.31.4 drizzle-orm: specifier: 0.44.2 - version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(pg@8.11.3) + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) pg: - specifier: 8.11.3 - version: 8.11.3 + specifier: 8.16.3 + version: 8.16.3 prompts: specifier: 2.4.2 version: 2.4.2 @@ -365,11 +365,11 @@ importers: specifier: 2.12.1 version: 2.12.1 drizzle-kit: - specifier: 0.31.1 - version: 0.31.1 + specifier: 0.31.4 + version: 0.31.4 drizzle-orm: specifier: 0.44.2 - version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(pg@8.11.3) + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) prompts: specifier: 2.4.2 version: 2.4.2 @@ -408,14 +408,14 @@ importers: specifier: 2.12.1 version: 2.12.1 drizzle-kit: - specifier: 0.31.1 - version: 0.31.1 + specifier: 0.31.4 + version: 0.31.4 drizzle-orm: specifier: 0.44.2 - version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(pg@8.11.3) + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) pg: - specifier: 8.11.3 - version: 8.11.3 + specifier: 8.16.3 + version: 8.16.3 prompts: specifier: 2.4.2 version: 2.4.2 @@ -455,7 +455,7 @@ importers: version: 2.0.3 drizzle-orm: specifier: 0.44.2 - version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(pg@8.11.3) + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) prompts: specifier: 2.4.2 version: 2.4.2 @@ -1143,7 +1143,7 @@ importers: dependencies: '@sentry/nextjs': specifier: ^8.33.1 - version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)) '@sentry/types': specifier: ^8.33.1 version: 8.37.1 @@ -2045,7 +2045,7 @@ importers: version: link:../packages/ui '@sentry/nextjs': specifier: ^8.33.1 - version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + version: 8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)) '@sentry/react': specifier: ^7.77.0 version: 7.119.2(react@19.1.0) @@ -2077,11 +2077,11 @@ importers: specifier: 16.4.7 version: 16.4.7 drizzle-kit: - specifier: 0.31.1 - version: 0.31.1 + specifier: 0.31.4 + version: 0.31.4 drizzle-orm: specifier: 0.44.2 - version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(pg@8.11.3) + version: 0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3) escape-html: specifier: 1.0.3 version: 1.0.3 @@ -2116,8 +2116,8 @@ importers: specifier: workspace:* version: link:../packages/payload pg: - specifier: 8.11.3 - version: 8.11.3 + specifier: 8.16.3 + version: 8.16.3 qs-esm: specifier: 7.0.2 version: 7.0.2 @@ -5040,6 +5040,9 @@ packages: cpu: [x64] os: [win32] + '@petamoriken/float16@3.9.2': + resolution: {integrity: sha512-VgffxawQde93xKxT3qap3OH+meZf7VaSB5Sqd4Rqc+FP5alWbpOyan/7tRbOAvynjpG3GpdtAuGU/NdhQpmrog==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -7057,10 +7060,6 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - buffer-writer@2.0.0: - resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} - engines: {node: '>=4'} - buffer@4.9.2: resolution: {integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==} @@ -7651,103 +7650,10 @@ packages: resolution: {integrity: sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==} engines: {node: '>=12'} - drizzle-kit@0.31.0: - resolution: {integrity: sha512-pcKVT+GbfPA+bUovPIilgVOoq+onNBo/YQBG86sf3/GFHkN6lRJPm1l7dKN0IMAk57RQoIm4GUllRrasLlcaSg==} + drizzle-kit@0.31.4: + resolution: {integrity: sha512-tCPWVZWZqWVx2XUsVpJRnH9Mx0ClVOf5YUHerZ5so1OKSlqww4zy1R5ksEdGRcO3tM3zj0PYN6V48TbQCL1RfA==} hasBin: true - drizzle-kit@0.31.1: - resolution: {integrity: sha512-PUjYKWtzOzPtdtQlTHQG3qfv4Y0XT8+Eas6UbxCmxTj7qgMf+39dDujf1BP1I+qqZtw9uzwTh8jYtkMuCq+B0Q==} - hasBin: true - - drizzle-orm@0.43.1: - resolution: {integrity: sha512-dUcDaZtE/zN4RV/xqGrVSMpnEczxd5cIaoDeor7Zst9wOe/HzC/7eAaulywWGYXdDEc9oBPMjayVEDg0ziTLJA==} - peerDependencies: - '@aws-sdk/client-rds-data': '>=3' - '@cloudflare/workers-types': '>=4' - '@electric-sql/pglite': '>=0.2.0' - '@libsql/client': '>=0.10.0' - '@libsql/client-wasm': '>=0.10.0' - '@neondatabase/serverless': '>=0.10.0' - '@op-engineering/op-sqlite': '>=2' - '@opentelemetry/api': ^1.4.1 - '@planetscale/database': '>=1.13' - '@prisma/client': '*' - '@tidbcloud/serverless': '*' - '@types/better-sqlite3': '*' - '@types/pg': '*' - '@types/sql.js': '*' - '@vercel/postgres': '>=0.8.0' - '@xata.io/client': '*' - better-sqlite3: '>=7' - bun-types: '*' - expo-sqlite: '>=14.0.0' - gel: '>=2' - knex: '*' - kysely: '*' - mysql2: '>=2' - pg: '>=8' - postgres: '>=3' - prisma: '*' - sql.js: '>=1' - sqlite3: '>=5' - peerDependenciesMeta: - '@aws-sdk/client-rds-data': - optional: true - '@cloudflare/workers-types': - optional: true - '@electric-sql/pglite': - optional: true - '@libsql/client': - optional: true - '@libsql/client-wasm': - optional: true - '@neondatabase/serverless': - optional: true - '@op-engineering/op-sqlite': - optional: true - '@opentelemetry/api': - optional: true - '@planetscale/database': - optional: true - '@prisma/client': - optional: true - '@tidbcloud/serverless': - optional: true - '@types/better-sqlite3': - optional: true - '@types/pg': - optional: true - '@types/sql.js': - optional: true - '@vercel/postgres': - optional: true - '@xata.io/client': - optional: true - better-sqlite3: - optional: true - bun-types: - optional: true - expo-sqlite: - optional: true - gel: - optional: true - knex: - optional: true - kysely: - optional: true - mysql2: - optional: true - pg: - optional: true - postgres: - optional: true - prisma: - optional: true - sql.js: - optional: true - sqlite3: - optional: true - drizzle-orm@0.44.2: resolution: {integrity: sha512-zGAqBzWWkVSFjZpwPOrmCrgO++1kZ5H/rZ4qTGeGOe18iXGVJWf3WPfHOVwFIbmi8kHjfJstC6rJomzGx8g/dQ==} peerDependencies: @@ -7893,6 +7799,10 @@ packages: resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} engines: {node: '>=0.12'} + env-paths@3.0.0: + resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + environment@1.1.0: resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} engines: {node: '>=18'} @@ -8574,6 +8484,11 @@ packages: peerDependencies: next: '>=13.2.0' + gel@2.0.1: + resolution: {integrity: sha512-gfem3IGvqKqXwEq7XseBogyaRwGsQGuE7Cw/yQsjLGdgiyqX92G1xENPCE0ltunPGcsJIa6XBOTx/PK169mOqw==} + engines: {node: '>= 18.0.0'} + hasBin: true + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -9210,6 +9125,10 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isexe@3.1.1: + resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==} + engines: {node: '>=16'} + isomorphic-unfetch@3.1.0: resolution: {integrity: sha512-geDJjpoZ8N0kWexiwkX8F9NkTsXhetLPVbZFQ+JTW239QNOwvB0gniuR1Wc6f0AMTn7/mFGyXvHTifrCp/GH8Q==} @@ -10282,9 +10201,6 @@ packages: package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} - packet-reader@1.0.0: - resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} - parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -10357,11 +10273,11 @@ packages: perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} - pg-cloudflare@1.1.1: - resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} + pg-cloudflare@1.2.7: + resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==} - pg-connection-string@2.7.0: - resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==} + pg-connection-string@2.9.1: + resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==} pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} @@ -10371,11 +10287,17 @@ packages: resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} engines: {node: '>=4'} - pg-pool@3.7.0: - resolution: {integrity: sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==} + pg-pool@3.10.1: + resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==} peerDependencies: pg: '>=8.0' + pg-protocol@1.10.2: + resolution: {integrity: sha512-Ci7jy8PbaWxfsck2dwZdERcDG2A0MG8JoQILs+uZNjABFuBuItAZCWUNz8sXRDMoui24rJw7WlXqgpMdBSN/vQ==} + + pg-protocol@1.10.3: + resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==} + pg-protocol@1.7.0: resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} @@ -10387,9 +10309,9 @@ packages: resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} engines: {node: '>=10'} - pg@8.11.3: - resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} - engines: {node: '>= 8.0.0'} + pg@8.16.3: + resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==} + engines: {node: '>= 16.0.0'} peerDependencies: pg-native: '>=3.0.1' peerDependenciesMeta: @@ -11216,6 +11138,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} + shelljs@0.8.5: resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} engines: {node: '>=4'} @@ -12351,6 +12277,11 @@ packages: engines: {node: '>= 8'} hasBin: true + which@4.0.0: + resolution: {integrity: sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==} + engines: {node: ^16.13.0 || >=18.0.0} + hasBin: true + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} @@ -15972,6 +15903,9 @@ snapshots: '@oxc-resolver/binding-win32-x64-msvc@5.3.0': optional: true + '@petamoriken/float16@3.9.2': + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -16466,7 +16400,7 @@ snapshots: '@sentry/utils': 7.119.2 localforage: 1.10.0 - '@sentry/nextjs@8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5))': + '@sentry/nextjs@8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/instrumentation-http': 0.53.0(@opentelemetry/api@1.9.0) @@ -16480,7 +16414,7 @@ snapshots: '@sentry/types': 8.37.1 '@sentry/utils': 8.37.1 '@sentry/vercel-edge': 8.37.1 - '@sentry/webpack-plugin': 2.22.6(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + '@sentry/webpack-plugin': 2.22.6(webpack@5.96.1(@swc/core@1.11.29)) chalk: 3.0.0 next: 15.3.2(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(babel-plugin-react-compiler@19.1.0-rc.2)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4) resolve: 1.22.8 @@ -16495,7 +16429,7 @@ snapshots: - supports-color - webpack - '@sentry/nextjs@8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5))': + '@sentry/nextjs@8.37.1(@opentelemetry/core@1.27.0(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.54.2(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.27.0(@opentelemetry/api@1.9.0))(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4))(react@19.1.0)(webpack@5.96.1(@swc/core@1.11.29))': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/instrumentation-http': 0.53.0(@opentelemetry/api@1.9.0) @@ -16509,7 +16443,7 @@ snapshots: '@sentry/types': 8.37.1 '@sentry/utils': 8.37.1 '@sentry/vercel-edge': 8.37.1 - '@sentry/webpack-plugin': 2.22.6(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + '@sentry/webpack-plugin': 2.22.6(webpack@5.96.1(@swc/core@1.11.29)) chalk: 3.0.0 next: 15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4) resolve: 1.22.8 @@ -16619,12 +16553,12 @@ snapshots: '@sentry/types': 8.37.1 '@sentry/utils': 8.37.1 - '@sentry/webpack-plugin@2.22.6(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5))': + '@sentry/webpack-plugin@2.22.6(webpack@5.96.1(@swc/core@1.11.29))': dependencies: '@sentry/bundler-plugin-core': 2.22.6 unplugin: 1.0.1 uuid: 9.0.0 - webpack: 5.96.1(@swc/core@1.11.29)(esbuild@0.25.5) + webpack: 5.96.1(@swc/core@1.11.29) transitivePeerDependencies: - encoding - supports-color @@ -17481,13 +17415,13 @@ snapshots: '@types/pg@8.11.6': dependencies: '@types/node': 22.15.30 - pg-protocol: 1.7.0 + pg-protocol: 1.10.2 pg-types: 4.0.2 '@types/pg@8.6.1': dependencies: '@types/node': 22.15.30 - pg-protocol: 1.7.0 + pg-protocol: 1.10.2 pg-types: 2.2.0 '@types/pluralize@0.0.33': {} @@ -18477,8 +18411,6 @@ snapshots: buffer-from@1.1.2: {} - buffer-writer@2.0.0: {} - buffer@4.9.2: dependencies: base64-js: 1.5.1 @@ -19042,7 +18974,7 @@ snapshots: dotenv@16.4.7: {} - drizzle-kit@0.31.0: + drizzle-kit@0.31.4: dependencies: '@drizzle-team/brocli': 0.10.2 '@esbuild-kit/esm-loader': 2.6.5 @@ -19051,38 +18983,23 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-kit@0.31.1: - dependencies: - '@drizzle-team/brocli': 0.10.2 - '@esbuild-kit/esm-loader': 2.6.5 - esbuild: 0.25.5 - esbuild-register: 3.6.0(esbuild@0.25.5) - transitivePeerDependencies: - - supports-color - - drizzle-orm@0.43.1(@libsql/client@0.14.0(bufferutil@4.0.8))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(pg@8.11.3): - optionalDependencies: - '@libsql/client': 0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5) - '@opentelemetry/api': 1.9.0 - '@types/pg': 8.11.6 - '@vercel/postgres': 0.9.0 - pg: 8.11.3 - - drizzle-orm@0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(pg@8.11.3): + drizzle-orm@0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.10.2)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3): optionalDependencies: '@libsql/client': 0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5) '@opentelemetry/api': 1.9.0 '@types/pg': 8.10.2 '@vercel/postgres': 0.9.0 - pg: 8.11.3 + gel: 2.0.1 + pg: 8.16.3 - drizzle-orm@0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(pg@8.11.3): + drizzle-orm@0.44.2(@libsql/client@0.14.0(bufferutil@4.0.8))(@opentelemetry/api@1.9.0)(@types/pg@8.11.6)(@vercel/postgres@0.9.0)(gel@2.0.1)(pg@8.16.3): optionalDependencies: '@libsql/client': 0.14.0(bufferutil@4.0.8)(utf-8-validate@6.0.5) '@opentelemetry/api': 1.9.0 '@types/pg': 8.11.6 '@vercel/postgres': 0.9.0 - pg: 8.11.3 + gel: 2.0.1 + pg: 8.16.3 dunder-proto@1.0.1: dependencies: @@ -19134,6 +19051,9 @@ snapshots: entities@6.0.1: {} + env-paths@3.0.0: + optional: true + environment@1.1.0: {} error-ex@1.3.2: @@ -19318,7 +19238,7 @@ snapshots: esbuild-register@3.6.0(esbuild@0.25.5): dependencies: - debug: 4.3.7 + debug: 4.4.1 esbuild: 0.25.5 transitivePeerDependencies: - supports-color @@ -20157,6 +20077,18 @@ snapshots: dependencies: next: 15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.50.0)(babel-plugin-macros@3.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(sass@1.77.4) + gel@2.0.1: + dependencies: + '@petamoriken/float16': 3.9.2 + debug: 4.4.1 + env-paths: 3.0.0 + semver: 7.7.1 + shell-quote: 1.8.3 + which: 4.0.0 + transitivePeerDependencies: + - supports-color + optional: true + gensync@1.0.0-beta.2: {} get-caller-file@2.0.5: {} @@ -20801,6 +20733,9 @@ snapshots: isexe@2.0.0: {} + isexe@3.1.1: + optional: true + isomorphic-unfetch@3.1.0: dependencies: node-fetch: 2.7.0 @@ -22317,8 +22252,6 @@ snapshots: package-json-from-dist@1.0.1: {} - packet-reader@1.0.0: {} - parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -22383,18 +22316,22 @@ snapshots: perfect-debounce@1.0.0: {} - pg-cloudflare@1.1.1: + pg-cloudflare@1.2.7: optional: true - pg-connection-string@2.7.0: {} + pg-connection-string@2.9.1: {} pg-int8@1.0.1: {} pg-numeric@1.0.2: {} - pg-pool@3.7.0(pg@8.11.3): + pg-pool@3.10.1(pg@8.16.3): dependencies: - pg: 8.11.3 + pg: 8.16.3 + + pg-protocol@1.10.2: {} + + pg-protocol@1.10.3: {} pg-protocol@1.7.0: {} @@ -22416,17 +22353,15 @@ snapshots: postgres-interval: 3.0.0 postgres-range: 1.1.4 - pg@8.11.3: + pg@8.16.3: dependencies: - buffer-writer: 2.0.0 - packet-reader: 1.0.0 - pg-connection-string: 2.7.0 - pg-pool: 3.7.0(pg@8.11.3) - pg-protocol: 1.7.0 + pg-connection-string: 2.9.1 + pg-pool: 3.10.1(pg@8.16.3) + pg-protocol: 1.10.3 pg-types: 2.2.0 pgpass: 1.0.5 optionalDependencies: - pg-cloudflare: 1.1.1 + pg-cloudflare: 1.2.7 pgpass@1.0.5: dependencies: @@ -23326,6 +23261,9 @@ snapshots: shebang-regex@3.0.0: {} + shell-quote@1.8.3: + optional: true + shelljs@0.8.5: dependencies: glob: 7.2.3 @@ -23901,17 +23839,16 @@ snapshots: ansi-escapes: 4.3.2 supports-hyperlinks: 2.3.0 - terser-webpack-plugin@5.3.10(@swc/core@1.11.29)(esbuild@0.25.5)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)): + terser-webpack-plugin@5.3.10(@swc/core@1.11.29)(webpack@5.96.1(@swc/core@1.11.29)): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.36.0 - webpack: 5.96.1(@swc/core@1.11.29)(esbuild@0.25.5) + webpack: 5.96.1(@swc/core@1.11.29) optionalDependencies: '@swc/core': 1.11.29 - esbuild: 0.25.5 terser@5.36.0: dependencies: @@ -24618,7 +24555,7 @@ snapshots: webpack-virtual-modules@0.5.0: {} - webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5): + webpack@5.96.1(@swc/core@1.11.29): dependencies: '@types/eslint-scope': 3.7.7 '@types/estree': 1.0.7 @@ -24640,7 +24577,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(@swc/core@1.11.29)(esbuild@0.25.5)(webpack@5.96.1(@swc/core@1.11.29)(esbuild@0.25.5)) + terser-webpack-plugin: 5.3.10(@swc/core@1.11.29)(webpack@5.96.1(@swc/core@1.11.29)) watchpack: 2.4.2 webpack-sources: 3.2.3 transitivePeerDependencies: @@ -24734,6 +24671,11 @@ snapshots: dependencies: isexe: 2.0.0 + which@4.0.0: + dependencies: + isexe: 3.1.1 + optional: true + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 diff --git a/test/database/postgres-vector.int.spec.ts b/test/database/postgres-vector.int.spec.ts index 7b041f0bd..81d374a10 100644 --- a/test/database/postgres-vector.int.spec.ts +++ b/test/database/postgres-vector.int.spec.ts @@ -1,24 +1,28 @@ +/* eslint-disable jest/no-conditional-in-test */ +/* eslint-disable jest/expect-expect */ /* eslint-disable jest/require-top-level-describe */ -import type { PostgresAdapter } from '@payloadcms/db-postgres/types' +import type { PostgresAdapter } from '@payloadcms/db-postgres' +import type { PostgresDB } from '@payloadcms/drizzle' -import { cosineDistance, desc, gt, sql } from 'drizzle-orm' +import { cosineDistance, desc, gt, jaccardDistance, l2Distance, lt, sql } from 'drizzle-orm' import path from 'path' -import { buildConfig, getPayload } from 'payload' +import { BasePayload, buildConfig, type DatabaseAdapterObj } from 'payload' import { fileURLToPath } from 'url' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) -// skip on ci as there db does not have the vector extension -const describeToUse = - process.env.PAYLOAD_DATABASE.startsWith('postgres') && process.env.CI !== 'true' - ? describe - : describe.skip +const describeToUse = process.env.PAYLOAD_DATABASE?.startsWith('postgres') + ? describe + : describe.skip describeToUse('postgres vector custom column', () => { - // TODO: this test is currently not working, come back to fix in a separate PR, issue: 12907 - it.skip('should add a vector column and query it', async () => { - const { databaseAdapter } = await import(path.resolve(dirname, '../databaseAdapter.js')) + const vectorColumnQueryTest = async (vectorType: string) => { + const { + databaseAdapter, + }: { + databaseAdapter: DatabaseAdapterObj + } = await import(path.resolve(dirname, '../databaseAdapter.js')) const init = databaseAdapter.init @@ -31,10 +35,12 @@ describeToUse('postgres vector custom column', () => { } adapter.beforeSchemaInit = [ ({ schema, adapter }) => { - ;(adapter as PostgresAdapter).rawTables.posts.columns.embedding = { - type: 'vector', - dimensions: 5, - name: 'embedding', + if (adapter?.rawTables?.posts?.columns) { + adapter.rawTables.posts.columns.embedding = { + type: vectorType, + dimensions: 5, + name: 'embedding', + } } return schema }, @@ -67,7 +73,8 @@ describeToUse('postgres vector custom column', () => { ], }) - const payload = await getPayload({ config }) + // do not use getPayload to avoid caching and re-using payload instance from previous tests + const payload = await new BasePayload().init({ config }) const catEmbedding = [1.5, -0.4, 7.2, 19.6, 20.2] @@ -105,7 +112,9 @@ describeToUse('postgres vector custom column', () => { const similarity = sql`1 - (${cosineDistance(payload.db.tables.posts.embedding, catEmbedding)})` - const res = await payload.db.drizzle + const db = payload.db.drizzle as PostgresDB + + const res = await db .select() .from(payload.db.tables.posts) .where(gt(similarity, 0.9)) @@ -115,9 +124,237 @@ describeToUse('postgres vector custom column', () => { expect(res).toHaveLength(2) // similarity sort - expect(res[0].title).toBe('cat') - expect(res[1].title).toBe('dog') + expect(res?.[0]?.title).toBe('cat') + expect(res?.[1]?.title).toBe('dog') + } - payload.logger.info(res) + it('should add a vector column and query it', async () => { + await vectorColumnQueryTest('vector') + }) + + it('should add a halfvec column and query it', async () => { + await vectorColumnQueryTest('halfvec') + }) + + it('should add a sparsevec column and query it', async () => { + const { + databaseAdapter, + }: { + databaseAdapter: DatabaseAdapterObj + } = await import(path.resolve(dirname, '../databaseAdapter.js')) + + const init = databaseAdapter.init + + databaseAdapter.init = ({ payload }) => { + const adapter = init({ payload }) + + adapter.extensions = { + vector: true, + } + + adapter.beforeSchemaInit = [ + ({ schema, adapter }) => { + if (adapter?.rawTables?.posts?.columns) { + adapter.rawTables.posts.columns.embedding = { + type: 'sparsevec', + dimensions: 5, + name: 'embedding', + } + } + return schema + }, + ] + + return adapter + } + + const config = await buildConfig({ + db: databaseAdapter, + secret: 'secret', + collections: [ + { + slug: 'users', + auth: true, + fields: [], + }, + { + slug: 'posts', + fields: [ + { + name: 'embedding', + type: 'text', + }, + { + name: 'title', + type: 'text', + }, + ], + }, + ], + }) + + const payload = await new BasePayload().init({ config }) + + // sparse-vector format: '{index:value,...}/dims' + const catEmbedding = '{1:1,3:2,5:3}/5' + + await payload.create({ + collection: 'posts', + data: { + embedding: '{2:1,4:2}/5', + title: 'apple', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: catEmbedding, + title: 'cat', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: '{2:4,4:6}/5', + title: 'fruit', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: '{1:1,3:2,5:2}/5', + title: 'dog', + }, + }) + + const distance = sql`(${l2Distance(payload.db.tables.posts.embedding, catEmbedding)})` + + const db = payload.db.drizzle as PostgresDB + + const res = await db + .select() + .from(payload.db.tables.posts) + .where(lt(distance, 1.1)) + .orderBy(distance) + .execute() + + // should return cat (distance 0) then dog + expect(res).toHaveLength(2) + expect(res?.[0]?.title).toBe('cat') + expect(res?.[1]?.title).toBe('dog') + }) + + it('should add a binaryvec column and query it', async () => { + const { + databaseAdapter, + }: { + databaseAdapter: DatabaseAdapterObj + } = await import(path.resolve(dirname, '../databaseAdapter.js')) + + const init = databaseAdapter.init + + // set options + databaseAdapter.init = ({ payload }) => { + const adapter = init({ payload }) + + adapter.extensions = { + vector: true, + } + adapter.beforeSchemaInit = [ + ({ schema, adapter }) => { + if (adapter?.rawTables?.posts?.columns) { + adapter.rawTables.posts.columns.embedding = { + type: 'bit', + dimensions: 5, + name: 'embedding', + } + } + return schema + }, + ] + return adapter + } + + const config = await buildConfig({ + db: databaseAdapter, + secret: 'secret', + collections: [ + { + slug: 'users', + auth: true, + fields: [], + }, + { + slug: 'posts', + fields: [ + { + type: 'text', + name: 'embedding', + }, + { + name: 'title', + type: 'text', + }, + ], + }, + ], + }) + + // do not use getPayload to avoid caching and re-using payload instance from previous tests + const payload = await new BasePayload().init({ config }) + + const catEmbedding = '10101' + + await payload.create({ + collection: 'posts', + data: { + embedding: '01010', + title: 'apple', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: '10101', + title: 'cat', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: '11111', + title: 'fruit', + }, + }) + + await payload.create({ + collection: 'posts', + data: { + embedding: '10100', + title: 'dog', + }, + }) + + const similarity = sql`1 - (${jaccardDistance(payload.db.tables.posts.embedding, catEmbedding)})` + + const db = payload.db.drizzle as PostgresDB + + const res = await db + .select() + .from(payload.db.tables.posts) + .where(gt(similarity, 0.6)) + .orderBy(desc(similarity)) + + // Only cat and dog + expect(res).toHaveLength(2) + + // similarity sort + expect(res?.[0]?.title).toBe('cat') + expect(res?.[1]?.title).toBe('dog') }) }) diff --git a/test/package.json b/test/package.json index 4ddc19f2b..565fb5f9f 100644 --- a/test/package.json +++ b/test/package.json @@ -71,7 +71,7 @@ "csv-parse": "^5.6.0", "dequal": "2.0.3", "dotenv": "16.4.7", - "drizzle-kit": "0.31.1", + "drizzle-kit": "0.31.4", "drizzle-orm": "0.44.2", "escape-html": "1.0.3", "eslint-plugin-playwright": "2.2.0", @@ -84,7 +84,7 @@ "next": "15.3.2", "nodemailer": "6.9.16", "payload": "workspace:*", - "pg": "8.11.3", + "pg": "8.16.3", "qs-esm": "7.0.2", "react": "19.1.0", "react-dom": "19.1.0",