tabshift/directus-schema-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance updates

Browse Source
This commit is contained in:
Gerard Lamusse
2025-03-09 20:52:08 +01:00
parent f73c15c196
commit 18bd3d9428
5 changed files with 92 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@@ -1,4 +1,8 @@
# Version 3.0.7
# Version 3.0.8
- **Fixed** Slight performance improvement when importing data.
## Version 3.0.7
- **Fixed** SCHEMA_SYNC_DATA_ONLY being the opposite of what it should be.

33
src/collectionExporter.ts
View File

@@ -82,24 +82,25 @@ class CollectionExporter implements IExporter {
public export = () => this._persistQueue();
public async load(merge = false) {
if (await ExportHelper.fileExists(this.filePath)) {
const json = await readFile(this.filePath, { encoding: 'utf8' });
if (!json) {
throw new Error(`Collection ${this.name} has invalid content: ${json}`);
}
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
if (Array.isArray(parsedJSON)) {
return this.loadItems(parsedJSON, merge);
} else if (!parsedJSON.partial) {
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
}
return await this.loadGroupedItems(parsedJSON, merge);
let json;
try {
json = await readFile(this.filePath, { encoding: 'utf8' });
} catch (e) {
return null;
}
return null;
if (!json) {
throw new Error(`Collection ${this.name} has invalid content: ${json}`);
}
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
if (Array.isArray(parsedJSON)) {
return this.loadItems(parsedJSON, merge);
} else if (!parsedJSON.partial) {
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
}
return await this.loadGroupedItems(parsedJSON, merge);
}
protected exportCollectionToFile = async () => {

4
src/index.ts
View File

@@ -1,5 +1,5 @@
import { SchemaOverview } from '@directus/types';
import { HookConfig } from '@directus/extensions';
import type { HookConfig } from '@directus/extensions';
import type { SchemaOverview } from '@directus/types';
import { condenseAction } from './condenseAction';
import { copyConfig } from './copyConfig';
import { ExportManager } from './exportManager';

128
src/schemaExporter.ts
View File

@@ -29,9 +29,7 @@ export class SchemaExporter implements IExporter {
} else {
// Clean up old schema files
const files = await glob(this.schemaFilesPath('*'));
for (const file of files) {
await rm(file);
}
await Promise.all(files.map(file => rm(file)));
}
};
@@ -50,71 +48,74 @@ export class SchemaExporter implements IExporter {
*/
public load = async () => {
const svc = this._getSchemaService();
if (await ExportHelper.fileExists(this._filePath)) {
const json = await readFile(this._filePath, { encoding: 'utf8' });
if (json) {
const schemaParsed = JSON.parse(json);
// For older versions, the snapshot was stored under the key `snapshot`
const { partial, hash, ...snapshot } = (
(schemaParsed as any).snapshot
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash })
: schemaParsed
) as Snapshot & { partial?: boolean; hash: string };
let json;
try {
json = await readFile(this._filePath, { encoding: 'utf8' });
} catch (e) {
return;
}
if (json) {
const schemaParsed = JSON.parse(json);
// For older versions, the snapshot was stored under the key `snapshot`
const { partial, hash, ...snapshot } = (
(schemaParsed as any).snapshot
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash })
: schemaParsed
) as Snapshot & { partial?: boolean; hash: string };
if (partial) {
snapshot.collections = [];
snapshot.fields = [];
snapshot.relations = [];
if (partial) {
snapshot.collections = [];
snapshot.fields = [];
snapshot.relations = [];
let found = 0;
const files = await glob(this.schemaFilesPath('*'));
for (const file of files) {
const collectionJson = await readFile(file, { encoding: 'utf8' });
const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & {
fields: SnapshotField[];
relations: SnapshotRelation[];
};
++found;
let found = 0;
const files = await glob(this.schemaFilesPath('*'));
await Promise.all(files.map(async (file) => {
const collectionJson = await readFile(file, { encoding: 'utf8' });
const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & {
fields: SnapshotField[];
relations: SnapshotRelation[];
};
++found;
// Only add collection if it has a meta definition (actual table or group)
if (collectionInfo.meta) {
snapshot.collections.push(collectionInfo);
}
for (const field of fields) {
snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
}
for (const relation of relations) {
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
}
// Only add collection if it has a meta definition (actual table or group)
if (collectionInfo.meta) {
snapshot.collections.push(collectionInfo);
}
if (found === 0) {
this.logger.error('No schema files found in schema directory');
return;
for (const field of fields) {
snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
}
for (const relation of relations) {
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
}
}));
this.logger.info(`Stitched ${found} partial schema files`);
snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
// Sort non-table collections to the start
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
// Sort fields and relations by collection
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
}
const currentSnapshot = await svc.snapshot();
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
if (currentHash === hash) {
this.logger.debug('Schema is already up-to-date');
if (found === 0) {
this.logger.error('No schema files found in schema directory');
return;
}
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
if (diff !== null) {
await svc.apply({ diff, hash: currentHash });
}
this.logger.info(`Stitched ${found} partial schema files`);
snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
// Sort non-table collections to the start
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
// Sort fields and relations by collection
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
}
const currentSnapshot = await svc.snapshot();
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
if (currentHash === hash) {
this.logger.debug('Schema is already up-to-date');
return;
}
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
if (diff !== null) {
await svc.apply({ diff, hash: currentHash });
}
}
};
@@ -170,10 +171,11 @@ export class SchemaExporter implements IExporter {
await writeFile(this._filePath, schemaJson);
// Save all collections with fields as individual files
for (const [collection, item] of Object.entries(map)) {
const itemJson = JSON.stringify(item, null, 2);
await writeFile(this.schemaFilesPath(collection), itemJson);
}
await Promise.all(
Object.entries(map).map(([collection, item]) =>
writeFile(this.schemaFilesPath(collection), JSON.stringify(item, null, 2))
)
);
} else {
const schemaJson = JSON.stringify(Object.assign({ hash }, snapshot), null, 2);
await writeFile(this._filePath, schemaJson);

5
src/utils.ts
View File

@@ -60,7 +60,7 @@ export class ExportHelper {
}
static async getExportMeta() {
if (await this.fileExists(this.hashFile)) {
try {
const content = await readFile(this.hashFile, { encoding: 'utf8' });
const [hash, ts] = content.split('@');
@@ -70,8 +70,9 @@ export class ExportHelper {
ts,
};
}
} catch {
// ignore
}
return null;
}
}