feat: telemetry

* feat: add telemetry to payload config

wip: more telemetry

* feat: send telemetry events

* chore: update node ci versions

* chore: cleanup console log

* chore: updates ts due to dependency update

* chore: remove unused deps

* chore: fix origin and casing

* docs: telemetry

* feat: uses oneWayHash within telemetry

* chore: sends hashed domain in telemetry

* feat: improves reliability of telemetry projectID

* chore: revises telemetry docs

Co-authored-by: Elliot DeNolf <denolfe@gmail.com>
Co-authored-by: James <james@trbl.design>
This commit is contained in:
Dan Ribbens
2022-06-24 17:32:09 -04:00
committed by GitHub
parent 7eb804daf9
commit 1c37ec3902
35 changed files with 1685 additions and 1463 deletions

View File

@@ -2,7 +2,7 @@ import React, { useEffect } from 'react';
import { FieldBase } from '../../../../fields/config/types';
import { useWatchForm } from '../Form/context';
const withCondition = <P extends unknown>(Field: React.ComponentType<P>): React.FC<P> => {
const withCondition = <P extends Record<string, unknown>>(Field: React.ComponentType<P>): React.FC<P> => {
const CheckForCondition: React.FC<P> = (props) => {
const {
admin: {

View File

@@ -3,5 +3,5 @@ import React from 'react';
export type Props = {
CustomComponent: React.ComponentType
DefaultComponent: React.ComponentType
componentProps?: unknown
componentProps?: Record<string, unknown>
}

View File

@@ -30,7 +30,7 @@ const VersionView: React.FC<Props> = ({ collection, global }) => {
const { setStepNav } = useStepNav();
const { params: { id, versionID } } = useRouteMatch<{ id?: string, versionID: string }>();
const [compareValue, setCompareValue] = useState<CompareOption>(mostRecentVersionOption);
const [localeOptions] = useState<LocaleOption[]>(() => (localization?.locales ? localization.locales.map((locale) => ({ label: locale, value: locale })) : []));
const [localeOptions] = useState<LocaleOption[]>(() => (localization ? localization.locales.map((locale) => ({ label: locale, value: locale })) : []));
const [locales, setLocales] = useState<LocaleOption[]>(localeOptions);
const { permissions } = useAuth();
const locale = useLocale();

View File

@@ -1,5 +1,6 @@
import { PayloadRequest } from '../../express/types';
import { Permissions } from '../types';
import { adminInit as adminInitTelemetry } from '../../utilities/telemetry/events/adminInit';
const allOperations = ['create', 'read', 'update', 'delete'];
@@ -18,6 +19,8 @@ async function accessOperation(args: Arguments): Promise<Permissions> {
},
} = args;
adminInitTelemetry(req);
const results = {} as Permissions;
const promises = [];

View File

@@ -1,6 +1,7 @@
import { CollectionModel } from '../../collections/config/types';
import { PayloadRequest } from '../../express/types';
async function init(args: { Model: CollectionModel }): Promise<boolean> {
async function init(args: { Model: CollectionModel, req: PayloadRequest }): Promise<boolean> {
const {
Model,
} = args;

View File

@@ -4,7 +4,7 @@ import init from '../operations/init';
export default async function initHandler(req: PayloadRequest, res: Response, next: NextFunction): Promise<any> {
try {
const initialized = await init({ Model: req.collection.Model });
const initialized = await init({ Model: req.collection.Model, req });
return res.status(200).json({ initialized });
} catch (error) {
return next(error);

View File

@@ -53,7 +53,7 @@ export default async function createLocal<T = any>(payload: Payload, options: Op
...req || {},
user,
payloadAPI: 'local',
locale: locale || req?.locale || payload?.config?.localization?.defaultLocale,
locale: locale || req?.locale || (payload?.config?.localization ? payload?.config?.localization?.defaultLocale : null),
fallbackLocale: fallbackLocale || req?.fallbackLocale || null,
payload,
files: {

View File

@@ -20,7 +20,7 @@ export default async function deleteLocal<T extends TypeWithID = any>(payload: P
collection: collectionSlug,
depth,
id,
locale = payload.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -29,7 +29,7 @@ export default async function findLocal<T extends TypeWithID = any>(payload: Pay
page,
limit,
where,
locale = payload?.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -42,7 +42,7 @@ export default async function findByIDLocal<T extends TypeWithID = any>(payload:
user: undefined,
...req || {},
payloadAPI: 'local',
locale: locale || req?.locale || payload?.config?.localization?.defaultLocale,
locale: locale || req?.locale || (payload?.config?.localization ? payload?.config?.localization?.defaultLocale : null),
fallbackLocale: fallbackLocale || req?.fallbackLocale || null,
payload,
};

View File

@@ -22,7 +22,7 @@ export default async function findVersionByIDLocal<T extends TypeWithVersion<T>
collection: collectionSlug,
depth,
id,
locale = payload.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
overrideAccess = true,
disableErrors = false,

View File

@@ -26,7 +26,7 @@ export default async function findVersionsLocal<T extends TypeWithVersion<T> = a
page,
limit,
where,
locale = payload.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -20,7 +20,7 @@ export default async function restoreVersionLocal<T extends TypeWithVersion<T> =
const {
collection: collectionSlug,
depth,
locale = payload?.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
data,
id,

View File

@@ -24,7 +24,7 @@ export default async function updateLocal<T = any>(payload: Payload, options: Op
const {
collection: collectionSlug,
depth,
locale = payload.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
data,
id,

View File

@@ -1,6 +1,7 @@
import path from 'path';
import { Config } from './types';
export const defaults = {
export const defaults: Config = {
serverURL: '',
defaultDepth: 2,
maxDepth: 10,
@@ -45,4 +46,5 @@ export const defaults = {
},
hooks: {},
localization: false,
telemetry: true,
};

View File

@@ -129,6 +129,7 @@ export default joi.object({
hooks: joi.object().keys({
afterError: joi.func(),
}),
telemetry: joi.boolean(),
plugins: joi.array().items(
joi.func(),
),

View File

@@ -95,6 +95,12 @@ export type AdminRoute = {
sensitive?: boolean
}
export type LocalizationConfig = {
locales: string[]
defaultLocale: string
fallback?: boolean
}
export type Config = {
admin?: {
user?: string;
@@ -168,11 +174,7 @@ export type Config = {
skip?: (req: PayloadRequest) => boolean;
};
upload?: Options;
localization?: {
locales: string[]
defaultLocale: string
fallback?: boolean
};
localization?: LocalizationConfig | false;
graphQL?: {
mutations?: ((graphQL: typeof GraphQL, payload: Payload) => Record<string, unknown>),
queries?: ((graphQL: typeof GraphQL, payload: Payload) => Record<string, unknown>),
@@ -185,6 +187,7 @@ export type Config = {
afterError?: AfterErrorHook;
};
plugins?: Plugin[];
telemetry?: boolean;
};
export type SanitizedConfig = Omit<DeepRequired<Config>, 'collections' | 'globals'> & {

View File

@@ -124,28 +124,30 @@ export const promise = async ({
}
// Push merge locale action if applicable
if (field.localized && req.payload.config.localization) {
if (field.localized) {
mergeLocaleActions.push(() => {
const localeData = req.payload.config.localization.locales.reduce((locales, localeID) => {
let valueToSet = siblingData[field.name];
if (req.payload.config.localization) {
const localeData = req.payload.config.localization.locales.reduce((locales, localeID) => {
let valueToSet = siblingData[field.name];
if (localeID !== req.locale) {
valueToSet = siblingDocWithLocales?.[field.name]?.[localeID];
if (localeID !== req.locale) {
valueToSet = siblingDocWithLocales?.[field.name]?.[localeID];
}
if (typeof valueToSet !== 'undefined') {
return {
...locales,
[localeID]: valueToSet,
};
}
return locales;
}, {});
// If there are locales with data, set the data
if (Object.keys(localeData).length > 0) {
siblingData[field.name] = localeData;
}
if (typeof valueToSet !== 'undefined') {
return {
...locales,
[localeID]: valueToSet,
};
}
return locales;
}, {});
// If there are locales with data, set the data
if (Object.keys(localeData).length > 0) {
siblingData[field.name] = localeData;
}
});
}

View File

@@ -19,7 +19,7 @@ export default async function findOneLocal<T extends TypeWithID = any>(payload:
const {
slug: globalSlug,
depth,
locale = payload.config.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -21,7 +21,7 @@ export default async function findVersionByIDLocal<T extends TypeWithVersion<T>
slug: globalSlug,
depth,
id,
locale = payload.config?.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -26,7 +26,7 @@ export default async function findVersionsLocal<T extends TypeWithVersion<T> = a
page,
limit,
where,
locale = payload.config.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
user,
overrideAccess = true,

View File

@@ -20,7 +20,7 @@ export default async function updateLocal<T extends TypeWithID = any>(payload: P
const {
slug: globalSlug,
depth,
locale = payload.config.localization?.defaultLocale,
locale = payload.config.localization ? payload.config.localization?.defaultLocale : null,
fallbackLocale = null,
data,
user,

View File

@@ -1,7 +1,7 @@
import { GraphQLEnumType } from 'graphql';
import { SanitizedConfig } from '../../config/types';
import { LocalizationConfig } from '../../config/types';
const buildFallbackLocaleInputType = (localization: SanitizedConfig['localization']): GraphQLEnumType => new GraphQLEnumType({
const buildFallbackLocaleInputType = (localization: LocalizationConfig): GraphQLEnumType => new GraphQLEnumType({
name: 'FallbackLocaleInputType',
values: [...localization.locales, 'none'].reduce((values, locale) => ({
...values,

View File

@@ -1,7 +1,7 @@
import { GraphQLEnumType } from 'graphql';
import { SanitizedConfig } from '../../config/types';
import { LocalizationConfig } from '../../config/types';
const buildLocaleInputType = (localization: SanitizedConfig['localization']): GraphQLEnumType => new GraphQLEnumType({
const buildLocaleInputType = (localization: LocalizationConfig): GraphQLEnumType => new GraphQLEnumType({
name: 'LocaleInputType',
values: localization.locales.reduce((values, locale) => ({
...values,

View File

@@ -62,6 +62,7 @@ import { Result as ResetPasswordResult } from './auth/operations/resetPassword';
import { Result as LoginResult } from './auth/operations/login';
import { Options as FindGlobalOptions } from './globals/operations/local/findOne';
import { Options as UpdateGlobalOptions } from './globals/operations/local/update';
import { serverInit as serverInitTelemetry } from './utilities/telemetry/events/serverInit';
require('isomorphic-fetch');
@@ -219,6 +220,8 @@ export class Payload {
}
if (typeof options.onInit === 'function') options.onInit(this);
serverInitTelemetry(this);
}
getAdminURL = (): string => `${this.config.serverURL}${this.config.routes.admin}`;

View File

@@ -32,7 +32,7 @@ const setBlockDiscriminators = (fields: Field[], schema: Schema, config: Sanitiz
const blockSchema = new Schema(blockSchemaFields, { _id: false, id: false });
if (blockFieldType.localized) {
if (blockFieldType.localized && config.localization) {
config.localization.locales.forEach((locale) => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore Possible incorrect typing in mongoose types, this works
@@ -57,10 +57,10 @@ const formatBaseSchema = (field: NonPresentationalField, buildSchemaOptions: Bui
index: field.index || field.unique || false,
});
const localizeSchema = (field: NonPresentationalField, schema, locales) => {
if (field.localized && Array.isArray(locales)) {
const localizeSchema = (field: NonPresentationalField, schema, localization) => {
if (field.localized && localization && Array.isArray(localization.locales)) {
return {
type: locales.reduce((localeSchema, locale) => ({
type: localization.locales.reduce((localeSchema, locale) => ({
...localeSchema,
[locale]: schema,
}), {
@@ -129,7 +129,7 @@ const fieldIndexMap = {
if (field.index === true || field.index === undefined) {
index = '2dsphere';
}
if (field.localized) {
if (field.localized && config.localization) {
return config.localization.locales.map((locale) => ({ [`${field.name}.${locale}`]: index }));
}
return [{ [field.name]: index }];
@@ -142,7 +142,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
text: (field: TextField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -150,7 +150,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
email: (field: EmailField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -158,7 +158,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
textarea: (field: TextareaField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -166,7 +166,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
richText: (field: RichTextField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -174,7 +174,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
code: (field: CodeField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -182,7 +182,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
point: (field: PointField, fields: SchemaDefinition, config: SanitizedConfig): SchemaDefinition => {
@@ -202,7 +202,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
radio: (field: RadioField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -217,7 +217,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
checkbox: (field: CheckboxField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -225,7 +225,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
date: (field: DateField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -233,7 +233,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
upload: (field: UploadField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -245,14 +245,14 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
relationship: (field: RelationshipField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions) => {
const hasManyRelations = Array.isArray(field.relationTo);
let schemaToReturn: { [key: string]: any } = {};
if (field.localized) {
if (field.localized && config.localization) {
schemaToReturn = {
type: config.localization.locales.reduce((locales, locale) => {
let localeSchema: { [key: string]: any } = {};
@@ -329,7 +329,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
group: (field: GroupField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -352,7 +352,7 @@ const fieldToSchemaMap = {
return {
...fields,
[field.name]: localizeSchema(field, baseSchema, config.localization.locales),
[field.name]: localizeSchema(field, baseSchema, config.localization),
};
},
select: (field: SelectField, fields: SchemaDefinition, config: SanitizedConfig, buildSchemaOptions: BuildSchemaOptions): SchemaDefinition => {
@@ -364,7 +364,7 @@ const fieldToSchemaMap = {
return option;
}),
};
const schemaToReturn = localizeSchema(field, baseSchema, config.localization.locales);
const schemaToReturn = localizeSchema(field, baseSchema, config.localization);
return {
...fields,
@@ -375,7 +375,7 @@ const fieldToSchemaMap = {
const baseSchema = [new Schema({ }, { _id: false, discriminatorKey: 'blockType' })];
let schemaToReturn;
if (field.localized) {
if (field.localized && config.localization) {
schemaToReturn = config.localization.locales.reduce((localeSchema, locale) => ({
...localeSchema,
[locale]: baseSchema,

View File

@@ -0,0 +1,34 @@
import { PayloadRequest } from '../../../express/types';
import { sendEvent } from '..';
import { oneWayHash } from '../oneWayHash';
export type AdminInitEvent = {
type: 'admin-init'
domainID?: string
userID?: string
}
export const adminInit = (req: PayloadRequest): void => {
const { user, payload } = req;
const { host } = req.headers;
let domainID: string;
let userID: string;
if (host) {
domainID = oneWayHash(host, payload.secret);
}
if (user && typeof user?.id === 'string') {
userID = oneWayHash(user.id, payload.secret);
}
sendEvent({
payload,
event: {
type: 'admin-init',
domainID,
userID,
},
});
};

View File

@@ -0,0 +1,15 @@
import { sendEvent } from '..';
import { Payload } from '../../..';
export type ServerInitEvent = {
type: 'server-init'
};
export const serverInit = (payload: Payload): void => {
sendEvent({
payload,
event: {
type: 'server-init',
},
});
};

View File

@@ -0,0 +1,105 @@
import { execSync } from 'child_process';
import Conf from 'conf';
import { randomBytes } from 'crypto';
import findUp from 'find-up';
import fs from 'fs';
import { Payload } from '../../index';
import { ServerInitEvent } from './events/serverInit';
import { AdminInitEvent } from './events/adminInit';
import { oneWayHash } from './oneWayHash';
export type BaseEvent = {
envID: string
projectID: string
nodeVersion: string
nodeEnv: string
payloadVersion: string
};
type PackageJSON = {
name: string
dependencies: Record<string, string | undefined>
}
type TelemetryEvent = ServerInitEvent | AdminInitEvent
type Args = {
payload: Payload
event: TelemetryEvent
}
export const sendEvent = async ({ payload, event } : Args): Promise<void> => {
if (payload.config.telemetry !== false) {
try {
const packageJSON = await getPackageJSON();
const baseEvent: BaseEvent = {
envID: getEnvID(),
projectID: getProjectID(payload, packageJSON),
nodeVersion: process.version,
nodeEnv: process.env.NODE_ENV || 'development',
payloadVersion: getPayloadVersion(packageJSON),
};
await fetch('https://telemetry.payloadcms.com/events', {
method: 'post',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ ...baseEvent, ...event }),
});
} catch (_) {
// Eat any errors in sending telemetry event
}
}
};
/**
* This is a quasi-persistent identifier used to dedupe recurring events. It's
* generated from random data and completely anonymous.
*/
const getEnvID = (): string => {
const conf = new Conf();
const ENV_ID = 'envID';
const val = conf.get(ENV_ID);
if (val) {
return val as string;
}
const generated = randomBytes(32).toString('hex');
conf.set(ENV_ID, generated);
return generated;
};
const getProjectID = (payload: Payload, packageJSON: PackageJSON): string => {
const projectID = getGitID(payload) || getPackageJSONID(payload, packageJSON) || payload.config.serverURL || process.cwd();
return oneWayHash(projectID, payload.secret);
};
const getGitID = (payload: Payload) => {
try {
const originBuffer = execSync('git config --local --get remote.origin.url', {
timeout: 1000,
stdio: 'pipe',
});
return oneWayHash(String(originBuffer).trim(), payload.secret);
} catch (_) {
return null;
}
};
const getPackageJSON = async (): Promise<PackageJSON> => {
const packageJsonPath = await findUp('package.json', { cwd: __dirname });
const jsonContent: PackageJSON = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
return jsonContent;
};
const getPackageJSONID = (payload: Payload, packageJSON: PackageJSON): string => {
return oneWayHash(packageJSON.name, payload.secret);
};
export const getPayloadVersion = (packageJSON: PackageJSON): string => {
return packageJSON?.dependencies?.payload ?? '';
};

View File

@@ -0,0 +1,13 @@
import { BinaryLike, createHash } from 'crypto';
export const oneWayHash = (data: BinaryLike, secret: string): string => {
const hash = createHash('sha256');
// prepend value with payload secret. This ensure one-way.
hash.update(secret);
// Update is an append operation, not a replacement. The secret from the prior
// update is still present!
hash.update(data);
return hash.digest('hex');
};