tabshift/directus-schema-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance updates

Browse Source
This commit is contained in:
Gerard Lamusse
2025-03-09 20:52:08 +01:00
parent f73c15c196
commit 18bd3d9428
5 changed files with 92 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@@ -1,4 +1,8 @@
# Version 3.0.7 # Version 3.0.8
- **Fixed** Slight performance improvement when importing data.
## Version 3.0.7
- **Fixed** SCHEMA_SYNC_DATA_ONLY being the opposite of what it should be. - **Fixed** SCHEMA_SYNC_DATA_ONLY being the opposite of what it should be.

33
src/collectionExporter.ts
View File

@@ -82,24 +82,25 @@ class CollectionExporter implements IExporter {
public export = () => this._persistQueue(); public export = () => this._persistQueue();
public async load(merge = false) { public async load(merge = false) {
if (await ExportHelper.fileExists(this.filePath)) { let json;
const json = await readFile(this.filePath, { encoding: 'utf8' }); try {
json = await readFile(this.filePath, { encoding: 'utf8' });
if (!json) { } catch (e) {
throw new Error(`Collection ${this.name} has invalid content: ${json}`); return null;
}
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
if (Array.isArray(parsedJSON)) {
return this.loadItems(parsedJSON, merge);
} else if (!parsedJSON.partial) {
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
}
return await this.loadGroupedItems(parsedJSON, merge);
} }
return null; if (!json) {
throw new Error(`Collection ${this.name} has invalid content: ${json}`);
}
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
if (Array.isArray(parsedJSON)) {
return this.loadItems(parsedJSON, merge);
} else if (!parsedJSON.partial) {
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
}
return await this.loadGroupedItems(parsedJSON, merge);
} }
protected exportCollectionToFile = async () => { protected exportCollectionToFile = async () => {

4
src/index.ts
View File

@@ -1,5 +1,5 @@
import { SchemaOverview } from '@directus/types'; import type { HookConfig } from '@directus/extensions';
import { HookConfig } from '@directus/extensions'; import type { SchemaOverview } from '@directus/types';
import { condenseAction } from './condenseAction'; import { condenseAction } from './condenseAction';
import { copyConfig } from './copyConfig'; import { copyConfig } from './copyConfig';
import { ExportManager } from './exportManager'; import { ExportManager } from './exportManager';

128
src/schemaExporter.ts
View File

@@ -29,9 +29,7 @@ export class SchemaExporter implements IExporter {
} else { } else {
// Clean up old schema files // Clean up old schema files
const files = await glob(this.schemaFilesPath('*')); const files = await glob(this.schemaFilesPath('*'));
for (const file of files) { await Promise.all(files.map(file => rm(file)));
await rm(file);
}
} }
}; };
@@ -50,71 +48,74 @@ export class SchemaExporter implements IExporter {
*/ */
public load = async () => { public load = async () => {
const svc = this._getSchemaService(); const svc = this._getSchemaService();
if (await ExportHelper.fileExists(this._filePath)) { let json;
const json = await readFile(this._filePath, { encoding: 'utf8' }); try {
if (json) { json = await readFile(this._filePath, { encoding: 'utf8' });
const schemaParsed = JSON.parse(json); } catch (e) {
// For older versions, the snapshot was stored under the key `snapshot` return;
const { partial, hash, ...snapshot } = ( }
(schemaParsed as any).snapshot if (json) {
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash }) const schemaParsed = JSON.parse(json);
: schemaParsed // For older versions, the snapshot was stored under the key `snapshot`
) as Snapshot & { partial?: boolean; hash: string }; const { partial, hash, ...snapshot } = (
(schemaParsed as any).snapshot
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash })
: schemaParsed
) as Snapshot & { partial?: boolean; hash: string };
if (partial) { if (partial) {
snapshot.collections = []; snapshot.collections = [];
snapshot.fields = []; snapshot.fields = [];
snapshot.relations = []; snapshot.relations = [];
let found = 0; let found = 0;
const files = await glob(this.schemaFilesPath('*')); const files = await glob(this.schemaFilesPath('*'));
for (const file of files) { await Promise.all(files.map(async (file) => {
const collectionJson = await readFile(file, { encoding: 'utf8' }); const collectionJson = await readFile(file, { encoding: 'utf8' });
const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & { const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & {
fields: SnapshotField[]; fields: SnapshotField[];
relations: SnapshotRelation[]; relations: SnapshotRelation[];
}; };
++found; ++found;
// Only add collection if it has a meta definition (actual table or group) // Only add collection if it has a meta definition (actual table or group)
if (collectionInfo.meta) { if (collectionInfo.meta) {
snapshot.collections.push(collectionInfo); snapshot.collections.push(collectionInfo);
}
for (const field of fields) {
snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
}
for (const relation of relations) {
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
}
} }
if (found === 0) { for (const field of fields) {
this.logger.error('No schema files found in schema directory'); snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
return;
} }
for (const relation of relations) {
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
}
}));
this.logger.info(`Stitched ${found} partial schema files`); if (found === 0) {
this.logger.error('No schema files found in schema directory');
snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
// Sort non-table collections to the start
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
// Sort fields and relations by collection
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
}
const currentSnapshot = await svc.snapshot();
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
if (currentHash === hash) {
this.logger.debug('Schema is already up-to-date');
return; return;
} }
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
if (diff !== null) { this.logger.info(`Stitched ${found} partial schema files`);
await svc.apply({ diff, hash: currentHash });
} snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
// Sort non-table collections to the start
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
// Sort fields and relations by collection
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
}
const currentSnapshot = await svc.snapshot();
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
if (currentHash === hash) {
this.logger.debug('Schema is already up-to-date');
return;
}
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
if (diff !== null) {
await svc.apply({ diff, hash: currentHash });
} }
} }
}; };
@@ -170,10 +171,11 @@ export class SchemaExporter implements IExporter {
await writeFile(this._filePath, schemaJson); await writeFile(this._filePath, schemaJson);
// Save all collections with fields as individual files // Save all collections with fields as individual files
for (const [collection, item] of Object.entries(map)) { await Promise.all(
const itemJson = JSON.stringify(item, null, 2); Object.entries(map).map(([collection, item]) =>
await writeFile(this.schemaFilesPath(collection), itemJson); writeFile(this.schemaFilesPath(collection), JSON.stringify(item, null, 2))
} )
);
} else { } else {
const schemaJson = JSON.stringify(Object.assign({ hash }, snapshot), null, 2); const schemaJson = JSON.stringify(Object.assign({ hash }, snapshot), null, 2);
await writeFile(this._filePath, schemaJson); await writeFile(this._filePath, schemaJson);

5
src/utils.ts
View File

@@ -60,7 +60,7 @@ export class ExportHelper {
} }
static async getExportMeta() { static async getExportMeta() {
if (await this.fileExists(this.hashFile)) { try {
const content = await readFile(this.hashFile, { encoding: 'utf8' }); const content = await readFile(this.hashFile, { encoding: 'utf8' });
const [hash, ts] = content.split('@'); const [hash, ts] = content.split('@');
@@ -70,8 +70,9 @@ export class ExportHelper {
ts, ts,
}; };
} }
} catch {
// ignore
} }
return null; return null;
} }
} }