From 970b8a1c09bd3f635746e4ab632786855d582fa2 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 12:21:43 -0400 Subject: [PATCH 01/12] phase 1 --- src/commands/apply.ts | 217 +++++++++++++---------------- src/commands/extract.ts | 84 +++++++---- src/lib/extract/extract-content.ts | 18 ++- src/lib/extract/index.ts | 66 +++++---- src/lib/load/apply-flags.ts | 103 +++++--------- src/lib/load/index.ts | 45 ++++-- src/lib/template-plan/flags.ts | 65 +++++++++ src/lib/template-plan/index.ts | 79 +++++++++++ src/lib/template-plan/metadata.ts | 39 ++++++ src/lib/template-plan/types.ts | 38 +++++ test/template-plan.test.ts | 82 +++++++++++ 11 files changed, 581 insertions(+), 255 deletions(-) create mode 100644 src/lib/template-plan/flags.ts create mode 100644 src/lib/template-plan/index.ts create mode 100644 src/lib/template-plan/metadata.ts create mode 100644 src/lib/template-plan/types.ts create mode 100644 test/template-plan.test.ts diff --git a/src/commands/apply.ts b/src/commands/apply.ts index 67fb555e..e4487f3a 100644 --- a/src/commands/apply.ts +++ b/src/commands/apply.ts @@ -1,20 +1,38 @@ import {intro, log, select, text} from '@clack/prompts' -import { Flags, ux} from '@oclif/core' +import {Flags, ux} from '@oclif/core' import chalk from 'chalk' import * as path from 'pathe' import * as customFlags from '../flags/common.js' -import {BSL_LICENSE_CTA, BSL_LICENSE_HEADLINE, BSL_LICENSE_TEXT, DIRECTUS_PINK, DIRECTUS_PURPLE, SEPARATOR } from '../lib/constants.js' +import { + BSL_LICENSE_CTA, + BSL_LICENSE_HEADLINE, + BSL_LICENSE_TEXT, + DIRECTUS_PINK, + DIRECTUS_PURPLE, + SEPARATOR, +} from '../lib/constants.js' import {type ApplyFlags, validateInteractiveFlags, validateProgrammaticFlags} from '../lib/load/apply-flags.js' import apply from '../lib/load/index.js' +import * as templatePlanFlags from '../lib/template-plan/flags.js' import {animatedBunny} from '../lib/utils/animated-bunny.js' -import {getDirectusEmailAndPassword, getDirectusToken, getDirectusUrl, initializeDirectusApi} from '../lib/utils/auth.js' +import { + getDirectusEmailAndPassword, + getDirectusToken, + getDirectusUrl, + initializeDirectusApi, +} from '../lib/utils/auth.js' import catchError from '../lib/utils/catch-error.js' -import {getCommunityTemplates, getGithubTemplate, getInteractiveLocalTemplate, getLocalTemplate} from '../lib/utils/get-template.js' +import { + getCommunityTemplates, + getGithubTemplate, + getInteractiveLocalTemplate, + getLocalTemplate, +} from '../lib/utils/get-template.js' import {logger} from '../lib/utils/logger.js' import openUrl from '../lib/utils/open-url.js' -import { shutdown, track } from '../services/posthog.js' -import { BaseCommand } from './base.js' +import {shutdown, track} from '../services/posthog.js' +import {BaseCommand} from './base.js' interface Template { directoryPath: string templateName: string @@ -22,83 +40,47 @@ interface Template { export default class ApplyCommand extends BaseCommand { static description = 'Apply a template to a blank Directus instance.' -static examples = [ + static examples = [ '$ directus-template-cli apply', '$ directus-template-cli apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local"', '$ directus-template-cli@beta apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local" --partial --no-content --no-users', ] -static flags = { - content: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load Content (data)', - }), - dashboards: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load Dashboards (dashboards, panels)', - }), + static flags = { + allowBrokenRelations: templatePlanFlags.allowBrokenRelations, + collections: templatePlanFlags.collections, + content: templatePlanFlags.componentFlags.content, + dashboards: templatePlanFlags.componentFlags.dashboards, directusToken: customFlags.directusToken, directusUrl: customFlags.directusUrl, disableTelemetry: customFlags.disableTelemetry, - extensions: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load Extensions', - }), - files: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load Files (files, folders)', - }), - flows: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load Flows (operations, flows)', - }), + excludeCollections: templatePlanFlags.excludeCollections, + extensions: templatePlanFlags.componentFlags.extensions, + files: templatePlanFlags.componentFlags.files, + flows: templatePlanFlags.componentFlags.flows, + noAssets: templatePlanFlags.noAssets, noExit: Flags.boolean({ default: false, hidden: true, }), - partial: Flags.boolean({ - dependsOn: ['programmatic'], - description: 'Enable partial template application (all components enabled by default)', - summary: 'Enable partial template application', - }), - permissions: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Loads permissions data. Collections include: directus_roles, directus_policies, directus_access, directus_permissions.', - summary: 'Load permissions (roles, policies, access, permissions)', - }), + partial: templatePlanFlags.partial, + permissions: templatePlanFlags.componentFlags.permissions, programmatic: customFlags.programmatic, - schema: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load schema (collections, relations)', - }), - settings: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load settings (project settings, translations, presets)', - }), + relationStrategy: templatePlanFlags.relationStrategy, + schema: templatePlanFlags.componentFlags.schema, + settings: templatePlanFlags.componentFlags.settings, templateLocation: customFlags.templateLocation, templateType: Flags.string({ default: 'local', dependsOn: ['programmatic'], - description: 'Type of template to apply. You can apply templates from our community repo, local directories, or public repositories from Github. Defaults to local. ', + description: + 'Type of template to apply. You can apply templates from our community repo, local directories, or public repositories from Github. Defaults to local. ', env: 'TEMPLATE_TYPE', options: ['community', 'local', 'github'], summary: 'Type of template to apply. Options: community, local, github.', }), userEmail: customFlags.userEmail, userPassword: customFlags.userPassword, - users: Flags.boolean({ - allowNo: true, - default: undefined, - description: 'Load users', - }), - + users: templatePlanFlags.componentFlags.users, } /** @@ -123,11 +105,11 @@ static flags = { const validatedFlags = validateInteractiveFlags(flags) // Show animated intro - await animatedBunny('Let\'s apply a template!') + await animatedBunny("Let's apply a template!") intro(`${chalk.bgHex(DIRECTUS_PURPLE).white.bold('Directus Template CLI')} - Apply Template`) const templateType = await select({ - message: 'What type of template would you like to apply?', + message: 'What type of template would you like to apply?', options: [ {label: 'Community templates', value: 'community'}, {label: 'From a local directory', value: 'local'}, @@ -139,38 +121,38 @@ static flags = { let template: Template switch (templateType) { - case 'community': { - const templates = await getCommunityTemplates() - const selectedTemplate = await select({ - message: 'Select a template.', - options: templates.map(t => ({label: t.templateName, value: t})), - }) - template = selectedTemplate as Template - break - } + case 'community': { + const templates = await getCommunityTemplates() + const selectedTemplate = await select({ + message: 'Select a template.', + options: templates.map((t) => ({label: t.templateName, value: t})), + }) + template = selectedTemplate as Template + break + } - case 'github': { - const ghTemplateUrl = await text({ - message: 'What is the public GitHub repository URL?', - }) - template = await getGithubTemplate(ghTemplateUrl as string) - break - } + case 'directus-plus': { + openUrl('https://directus.io/plus?utm_source=directus-template-cli&utm_content=apply-command') + log.info('Redirecting to Directus website.') + if (!validatedFlags.noExit) process.exit(0) + return + } - case 'local': { - const localTemplateDir = await text({ - message: 'What is the local template directory?', - }) - template = await this.selectLocalTemplate(localTemplateDir as string) - break - } + case 'github': { + const ghTemplateUrl = await text({ + message: 'What is the public GitHub repository URL?', + }) + template = await getGithubTemplate(ghTemplateUrl as string) + break + } - case 'directus-plus': { - openUrl('https://directus.io/plus?utm_source=directus-template-cli&utm_content=apply-command') - log.info('Redirecting to Directus website.') - if (!validatedFlags.noExit) process.exit(0) - return - } + case 'local': { + const localTemplateDir = await text({ + message: 'What is the local template directory?', + }) + template = await this.selectLocalTemplate(localTemplateDir as string) + break + } } log.info(`You selected ${ux.colorize(DIRECTUS_PINK, template.templateName)}`) @@ -218,7 +200,7 @@ static flags = { }, lifecycle: 'start', runId: this.runId, - }); + }) } await apply(template.directoryPath, validatedFlags) @@ -238,8 +220,8 @@ static flags = { }, lifecycle: 'complete', runId: this.runId, - }); - await shutdown(); + }) + await shutdown() } ux.stdout(SEPARATOR) @@ -250,7 +232,6 @@ static flags = { ux.stdout('Template applied successfully.') if (!validatedFlags.noExit) process.exit(0) - return } } @@ -266,27 +247,27 @@ static flags = { let template: Template switch (validatedFlags.templateType) { - case 'community': { - const templates = await getCommunityTemplates() - template = templates.find(t => t.templateName === validatedFlags.templateLocation) || templates[0] - break - } + case 'community': { + const templates = await getCommunityTemplates() + template = templates.find((t) => t.templateName === validatedFlags.templateLocation) || templates[0] + break + } - case 'github': { - template = await getGithubTemplate(validatedFlags.templateLocation) - break - } + case 'github': { + template = await getGithubTemplate(validatedFlags.templateLocation) + break + } - case 'local': { - template = await getLocalTemplate(validatedFlags.templateLocation) - break - } + case 'local': { + template = await getLocalTemplate(validatedFlags.templateLocation) + break + } - default: { - catchError('Invalid template type. Please check your template type.', { - fatal: true, - }) - } + default: { + catchError('Invalid template type. Please check your template type.', { + fatal: true, + }) + } } await initializeDirectusApi(validatedFlags) @@ -309,7 +290,7 @@ static flags = { }, lifecycle: 'start', runId: this.runId, - }); + }) } await apply(template.directoryPath, validatedFlags) @@ -329,8 +310,8 @@ static flags = { }, lifecycle: 'complete', runId: this.runId, - }); - await shutdown(); + }) + await shutdown() } ux.stdout(SEPARATOR) @@ -359,7 +340,7 @@ static flags = { const selectedTemplate = await select({ message: 'Multiple templates found. Please select one:', - options: templates.map(t => ({ + options: templates.map((t) => ({ label: `${t.templateName} (${path.basename(t.directoryPath)})`, value: t, })), diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 773dc7cd..2a897e90 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -6,44 +6,78 @@ import fs from 'node:fs' import path from 'pathe' import * as customFlags from '../flags/common.js' -import {BSL_LICENSE_CTA, BSL_LICENSE_HEADLINE, BSL_LICENSE_TEXT, DIRECTUS_PINK, DIRECTUS_PURPLE, SEPARATOR} from '../lib/constants.js' +import { + BSL_LICENSE_CTA, + BSL_LICENSE_HEADLINE, + BSL_LICENSE_TEXT, + DIRECTUS_PINK, + DIRECTUS_PURPLE, + SEPARATOR, +} from '../lib/constants.js' import extract from '../lib/extract/index.js' +import * as templatePlanFlags from '../lib/template-plan/flags.js' +import {buildTemplatePlan} from '../lib/template-plan/index.js' import {animatedBunny} from '../lib/utils/animated-bunny.js' -import {getDirectusEmailAndPassword, getDirectusToken, getDirectusUrl, initializeDirectusApi, validateAuthFlags} from '../lib/utils/auth.js' -import catchError from '../lib/utils/catch-error.js' import { - generatePackageJsonContent, - generateReadmeContent, -} from '../lib/utils/template-defaults.js' -import { shutdown, track } from '../services/posthog.js' -import { BaseCommand } from './base.js' + getDirectusEmailAndPassword, + getDirectusToken, + getDirectusUrl, + initializeDirectusApi, + validateAuthFlags, +} from '../lib/utils/auth.js' +import catchError from '../lib/utils/catch-error.js' +import {generatePackageJsonContent, generateReadmeContent} from '../lib/utils/template-defaults.js' +import {shutdown, track} from '../services/posthog.js' +import {BaseCommand} from './base.js' export interface ExtractFlags { - directusToken: string; - directusUrl: string; - disableTelemetry?: boolean; - programmatic: boolean; - templateLocation: string; - templateName: string; - userEmail: string; - userPassword: string; + allowBrokenRelations?: boolean + collections?: string + content?: boolean + dashboards?: boolean + directusToken: string + directusUrl: string + disableTelemetry?: boolean + excludeCollections?: string + extensions?: boolean + files?: boolean + flows?: boolean + noAssets?: boolean + partial?: boolean + permissions?: boolean + programmatic: boolean + relationStrategy?: 'deep' | 'empty' | 'ids' + schema?: boolean + settings?: boolean + templateLocation: string + templateName: string + userEmail: string + userPassword: string + users?: boolean } export default class ExtractCommand extends BaseCommand { static description = 'Extract a template from a Directus instance.' -static examples = [ + static examples = [ '$ directus-template-cli extract', '$ directus-template-cli extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055"', ] -static flags = { + static flags = { directusToken: customFlags.directusToken, directusUrl: customFlags.directusUrl, disableTelemetry: customFlags.disableTelemetry, + partial: templatePlanFlags.partial, programmatic: customFlags.programmatic, templateLocation: customFlags.templateLocation, templateName: customFlags.templateName, userEmail: customFlags.userEmail, userPassword: customFlags.userPassword, + ...templatePlanFlags.componentFlags, + allowBrokenRelations: templatePlanFlags.allowBrokenRelations, + collections: templatePlanFlags.collections, + excludeCollections: templatePlanFlags.excludeCollections, + noAssets: templatePlanFlags.noAssets, + relationStrategy: templatePlanFlags.relationStrategy, } /** @@ -79,7 +113,7 @@ static flags = { }, lifecycle: 'start', runId: this.runId, - }); + }) } try { @@ -105,9 +139,11 @@ static flags = { ux.stdout(SEPARATOR) - ux.action.start(`Extracting template - ${ux.colorize(DIRECTUS_PINK, templateName)} from ${ux.colorize(DIRECTUS_PINK, flags.directusUrl)} to ${ux.colorize(DIRECTUS_PINK, directory)}`) + ux.action.start( + `Extracting template - ${ux.colorize(DIRECTUS_PINK, templateName)} from ${ux.colorize(DIRECTUS_PINK, flags.directusUrl)} to ${ux.colorize(DIRECTUS_PINK, directory)}`, + ) - await extract(directory) + await extract(directory, buildTemplatePlan(flags)) ux.action.stop() @@ -125,8 +161,8 @@ static flags = { }, lifecycle: 'complete', runId: this.runId, - }); - await shutdown(); + }) + await shutdown() } log.warn(BSL_LICENSE_HEADLINE) @@ -144,7 +180,7 @@ static flags = { * @returns {Promise} - Returns nothing */ private async runInteractive(flags: ExtractFlags): Promise { - await animatedBunny('Let\'s extract a template!') + await animatedBunny("Let's extract a template!") intro(`${chalk.bgHex(DIRECTUS_PURPLE).white.bold('Directus Template CLI')} - Extract Template`) diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index b334de37..a77e204e 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -1,17 +1,21 @@ import {readCollections, readItems} from '@directus/sdk' import {ux} from '@oclif/core' +import type {TemplatePlan} from '../template-plan/index.js' + import {DIRECTUS_PINK} from '../constants.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' -async function getCollections() { +async function getCollections(plan?: TemplatePlan) { const response = await api.client.request(readCollections()) return response - .filter(item => !item.collection.startsWith('directus_', 0)) - .filter(item => item.schema != null) - .map(i => i.collection) + .filter((item) => !item.collection.startsWith('directus_', 0)) + .filter((item) => item.schema != null) + .map((i) => i.collection) + .filter((collection) => !plan?.collections || plan.collections.includes(collection)) + .filter((collection) => !plan?.excludeCollections?.includes(collection)) } async function getDataFromCollection(collection: string, dir: string) { @@ -23,11 +27,11 @@ async function getDataFromCollection(collection: string, dir: string) { } } -export async function extractContent(dir: string) { +export async function extractContent(dir: string, plan?: TemplatePlan) { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting content')) try { - const collections = await getCollections() - await Promise.all(collections.map(collection => getDataFromCollection(collection, dir))) + const collections = await getCollections(plan) + await Promise.all(collections.map((collection) => getDataFromCollection(collection, dir))) } catch (error) { catchError(error) } diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index f6aa0593..78abf4c9 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -1,6 +1,7 @@ import {ux} from '@oclif/core' import fs from 'node:fs' +import {buildTemplatePlan, type TemplatePlan, writeTemplateMetadata} from '../template-plan/index.js' import extractAccess from './extract-access.js' import {downloadAllFiles} from './extract-assets.js' import extractCollections from './extract-collections.js' @@ -21,47 +22,64 @@ import extractSettings from './extract-settings.js' import extractTranslations from './extract-translations.js' import extractUsers from './extract-users.js' -export default async function extract(dir: string) { - // Get the destination directory for the actual files +export default async function extract(dir: string, plan: TemplatePlan = buildTemplatePlan()) { const destination = `${dir}/src` - // Check if directory exists, if not, then create it. if (!fs.existsSync(destination)) { ux.stdout(`Attempting to create directory at: ${destination}`) fs.mkdirSync(destination, {recursive: true}) } - await extractSchema(destination) + if (plan.components.schema) { + await extractSchema(destination) + await extractCollections(destination) + await extractFields(destination) + await extractRelations(destination) + } - await extractCollections(destination) - await extractFields(destination) - await extractRelations(destination) + if (plan.components.files) { + await extractFolders(destination) + await extractFiles(destination) + await downloadAllFiles(destination) + } - await extractFolders(destination) - await extractFiles(destination) + if (plan.components.users || plan.components.permissions) { + await extractRoles(destination) + await extractPermissions(destination) + await extractPolicies(destination) - await extractUsers(destination) - await extractRoles(destination) - await extractPermissions(destination) - await extractPolicies(destination) - await extractAccess(destination) + if (plan.components.users) { + await extractUsers(destination) + } - await extractPresets(destination) + await extractAccess(destination) + } - await extractTranslations(destination) + if (plan.components.settings) { + await extractPresets(destination) + await extractTranslations(destination) + await extractSettings(destination) + } - await extractFlows(destination) - await extractOperations(destination) + if (plan.components.flows) { + await extractFlows(destination) + await extractOperations(destination) + } - await extractDashboards(destination) - await extractPanels(destination) + if (plan.components.dashboards) { + await extractDashboards(destination) + await extractPanels(destination) + } - await extractSettings(destination) - await extractExtensions(destination) + if (plan.components.extensions) { + await extractExtensions(destination) + } - await extractContent(destination) + if (plan.components.content) { + await extractContent(destination, plan) + } - await downloadAllFiles(destination) + await writeTemplateMetadata(destination, plan) return {} } diff --git a/src/lib/load/apply-flags.ts b/src/lib/load/apply-flags.ts index 253ae2b6..31060377 100644 --- a/src/lib/load/apply-flags.ts +++ b/src/lib/load/apply-flags.ts @@ -1,89 +1,58 @@ import {ux} from '@oclif/core' -import catchError from '../utils/catch-error.js' +import {buildTemplatePlan, componentNames} from '../template-plan/index.js' export interface ApplyFlags { - content: boolean; - dashboards: boolean; - directusToken: string; - directusUrl: string; - disableTelemetry?: boolean; - extensions: boolean; - files: boolean; - flows: boolean; - noExit?: boolean; - partial: boolean; - permissions: boolean; - programmatic: boolean; - schema: boolean; - settings: boolean; - templateLocation: string; - templateType: 'community' | 'github' | 'local'; - userEmail: string; - userPassword: string; - users?: boolean; + allowBrokenRelations?: boolean + collections?: string + content: boolean + dashboards: boolean + directusToken: string + directusUrl: string + disableTelemetry?: boolean + excludeCollections?: string + extensions: boolean + files: boolean + flows: boolean + noAssets?: boolean + noExit?: boolean + partial: boolean + permissions: boolean + programmatic: boolean + relationStrategy?: 'deep' | 'empty' | 'ids' + schema: boolean + settings: boolean + templateLocation: string + templateType: 'community' | 'github' | 'local' + userEmail: string + userPassword: string + users?: boolean } -export const loadFlags = [ - 'content', - 'dashboards', - 'extensions', - 'files', - 'flows', - 'permissions', - 'schema', - 'settings', - 'users', -] as const +export const loadFlags = componentNames export function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags { const {directusToken, directusUrl, templateLocation, userEmail, userPassword} = flags if (!directusUrl) ux.error('Directus URL is required for programmatic mode.') - if (!directusToken && (!userEmail || !userPassword)) ux.error('Either Directus token or email and password are required for programmatic mode.') + if (!directusToken && (!userEmail || !userPassword)) + ux.error('Either Directus token or email and password are required for programmatic mode.') if (!templateLocation) ux.error('Template location is required for programmatic mode.') - return flags.partial ? handlePartialFlags(flags) : setAllFlagsTrue(flags) + return applyTemplatePlan(flags) } export function validateInteractiveFlags(flags: ApplyFlags): ApplyFlags { - return flags.partial ? handlePartialFlags(flags) : setAllFlagsTrue(flags) + return applyTemplatePlan(flags) } -function handlePartialFlags(flags: ApplyFlags): ApplyFlags { - const enabledFlags = loadFlags.filter(flag => flags[flag] === true) - const disabledFlags = loadFlags.filter(flag => flags[flag] === false) +function applyTemplatePlan(flags: ApplyFlags): ApplyFlags { + const plan = buildTemplatePlan(flags) + const nextFlags = {...flags, partial: plan.partial} - if (enabledFlags.length > 0) { - for (const flag of loadFlags) flags[flag] = enabledFlags.includes(flag) - } else if (disabledFlags.length > 0) { - for (const flag of loadFlags) flags[flag] = !disabledFlags.includes(flag) - } else { - setAllFlagsTrue(flags) - } + for (const flag of loadFlags) nextFlags[flag] = plan.components[flag] - handleDependencies(flags) + if (plan.partial) ux.warn('Applying partial template. Missing components will not be auto-enabled.') - if (!loadFlags.some(flag => flags[flag])) { - catchError(new Error('When using --partial, at least one component must be loaded.'), {fatal: true}) - } - - return flags -} - -function handleDependencies(flags: ApplyFlags): void { - if (flags.content && (!flags.schema || !flags.files)) { - flags.schema = flags.files = true - ux.warn('Content loading requires schema and files. Enabling schema and files flags.') - } - - if (flags.users && !flags.permissions) { - flags.permissions = true - ux.warn('User loading requires permissions. Enabling permissions flag.') - } -} - -function setAllFlagsTrue(flags: ApplyFlags): ApplyFlags { - for (const flag of loadFlags) flags[flag] = true - return flags + return nextFlags } diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index 0ea802e5..e983292c 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -1,7 +1,8 @@ import {ux} from '@oclif/core' -import type { ApplyFlags } from './apply-flags.js' +import type {ApplyFlags} from './apply-flags.js' +import {buildTemplatePlan, componentNames, readTemplateMetadata} from '../template-plan/index.js' import checkTemplate from '../utils/check-template.js' import loadAccess from './load-access.js' import loadCollections from './load-collections.js' @@ -21,59 +22,73 @@ import loadTranslations from './load-translations.js' import loadUsers from './load-users.js' import updateRequiredFields from './update-required-fields.js' - export default async function apply(dir: string, flags: ApplyFlags) { const source = `${dir}/src` - const isTemplateOk = await checkTemplate(source) - if (!isTemplateOk) { - ux.error('The template is missing the collections, fields, or relations files. Older templates are not supported in v0.4 of directus-template-cli. Try using v0.3 to load older templates npx directus-template-cli@0.3 apply or extract the template using latest version before applying. Exiting...') + const metadata = readTemplateMetadata(source) + const requestedPlan = buildTemplatePlan(flags) + const components = {...requestedPlan.components} + + if (metadata?.partial) { + ux.warn('Template metadata indicates this is a partial template.') + for (const component of componentNames) { + components[component] = components[component] && metadata.components[component] + } + } + + if (!metadata || components.schema) { + const isTemplateOk = await checkTemplate(source) + if (!isTemplateOk) { + ux.error( + 'The template is missing the collections, fields, or relations files. Older templates are not supported in v0.4 of directus-template-cli. Try using v0.3 to load older templates npx directus-template-cli@0.3 apply or extract the template using latest version before applying. Exiting...', + ) + } } - if (flags.schema) { + if (components.schema) { await loadCollections(source) await loadRelations(source) } - if (flags.permissions || flags.users) { + if (components.permissions || components.users) { await loadRoles(source) await loadPolicies(source) await loadPermissions(source) - if (flags.users) { + if (components.users) { await loadUsers(source) } await loadAccess(source) } - if (flags.files) { + if (components.files) { await loadFolders(source) await loadFiles(source) } - if (flags.content) { + if (components.content) { await loadData(source) } - if (flags.schema) { + if (components.schema) { await updateRequiredFields(source) } - if (flags.dashboards) { + if (components.dashboards) { await loadDashboards(source) } - if (flags.flows) { + if (components.flows) { await loadFlows(source) } - if (flags.settings) { + if (components.settings) { await loadSettings(source) await loadTranslations(source) await loadPresets(source) } - if (flags.extensions) { + if (components.extensions) { await loadExtensions(source) } diff --git a/src/lib/template-plan/flags.ts b/src/lib/template-plan/flags.ts new file mode 100644 index 00000000..6c61c96d --- /dev/null +++ b/src/lib/template-plan/flags.ts @@ -0,0 +1,65 @@ +import {Flags} from '@oclif/core' + +export const componentNames = [ + 'schema', + 'content', + 'files', + 'flows', + 'dashboards', + 'permissions', + 'settings', + 'extensions', + 'users', +] as const + +export const partial = Flags.boolean({ + description: 'Enable partial template mode', + summary: 'Enable partial template mode', +}) + +export const componentFlags = { + content: Flags.boolean({allowNo: true, default: undefined, description: 'Include content/data'}), + dashboards: Flags.boolean({allowNo: true, default: undefined, description: 'Include dashboards and panels'}), + extensions: Flags.boolean({allowNo: true, default: undefined, description: 'Include extensions'}), + files: Flags.boolean({allowNo: true, default: undefined, description: 'Include files, folders, and assets'}), + flows: Flags.boolean({allowNo: true, default: undefined, description: 'Include flows and operations'}), + permissions: Flags.boolean({ + allowNo: true, + default: undefined, + description: 'Include permissions, roles, policies, and access', + }), + schema: Flags.boolean({ + allowNo: true, + default: undefined, + description: 'Include schema, collections, fields, and relations', + }), + settings: Flags.boolean({ + allowNo: true, + default: undefined, + description: 'Include settings, translations, and presets', + }), + users: Flags.boolean({allowNo: true, default: undefined, description: 'Include users'}), +} + +export const collections = Flags.string({ + description: 'Only include these comma-separated collections', +}) + +export const excludeCollections = Flags.string({ + description: 'Exclude these comma-separated collections', +}) + +export const relationStrategy = Flags.string({ + description: 'How to handle relations to omitted data', + options: ['empty', 'ids', 'deep'], +}) + +export const allowBrokenRelations = Flags.boolean({ + default: false, + description: 'Allow intentionally incomplete relation references', +}) + +export const noAssets = Flags.boolean({ + default: undefined, + description: 'Shorthand for --no-files and --exclude-collections directus_files', +}) diff --git a/src/lib/template-plan/index.ts b/src/lib/template-plan/index.ts new file mode 100644 index 00000000..3165c41f --- /dev/null +++ b/src/lib/template-plan/index.ts @@ -0,0 +1,79 @@ +import type {RelationStrategy, TemplateComponents, TemplatePlan} from './types.js' + +import catchError from '../utils/catch-error.js' +import {componentNames} from './flags.js' + +function parseList(value?: string | string[]): string[] | undefined { + if (!value) return undefined + const raw = Array.isArray(value) ? value.join(',') : value + const values = raw + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + return values.length > 0 ? values : undefined +} + +function hasPartialOnlyFlags(flags: any): boolean { + return Boolean( + flags.collections || + flags.excludeCollections || + flags.noAssets === true || + flags.relationStrategy !== undefined || + flags.allowBrokenRelations === true, + ) +} + +function hasComponentFlags(flags: any): boolean { + return componentNames.some((component) => flags[component] !== undefined) +} + +function buildComponents(flags: any, partial: boolean): TemplateComponents { + const components = {} as TemplateComponents + const enabled = componentNames.filter((component) => flags[component] === true) + const disabled = componentNames.filter((component) => flags[component] === false) + + if (!partial) { + for (const component of componentNames) components[component] = true + return components + } + + if (enabled.length > 0) { + for (const component of componentNames) components[component] = enabled.includes(component) + } else if (disabled.length > 0) { + for (const component of componentNames) components[component] = !disabled.includes(component) + } else { + for (const component of componentNames) components[component] = true + } + + if (flags.noAssets) components.files = false + + return components +} + +export function buildTemplatePlan(flags: any = {}): TemplatePlan { + const collections = parseList(flags.collections) + const excludeCollections = parseList(flags.excludeCollections) || (flags.noAssets ? [] : undefined) + if (flags.noAssets && !excludeCollections?.includes('directus_files')) { + excludeCollections?.push('directus_files') + } + + const partial = Boolean(flags.partial || hasComponentFlags(flags) || hasPartialOnlyFlags(flags)) + const components = buildComponents(flags, partial) + + if (!componentNames.some((component) => components[component])) { + catchError(new Error('At least one template component must be enabled.'), {fatal: true}) + } + + return { + allowBrokenRelations: Boolean(flags.allowBrokenRelations), + collections, + components, + excludeCollections, + partial, + relationStrategy: (flags.relationStrategy || (partial ? 'ids' : 'deep')) as RelationStrategy, + } +} + +export * from './flags.js' +export * from './metadata.js' +export * from './types.js' diff --git a/src/lib/template-plan/metadata.ts b/src/lib/template-plan/metadata.ts new file mode 100644 index 00000000..29b73fb2 --- /dev/null +++ b/src/lib/template-plan/metadata.ts @@ -0,0 +1,39 @@ +import fs from 'node:fs' +import path from 'pathe' + +import type {TemplateMetadata, TemplatePlan, TemplateWarning} from './types.js' + +const META_FILE = 'template-meta.json' + +export function createTemplateMetadata(plan: TemplatePlan, warnings: TemplateWarning[] = []): TemplateMetadata { + return { + allowBrokenRelations: plan.allowBrokenRelations, + collections: plan.collections, + components: plan.components, + excludedCollections: plan.excludeCollections, + partial: plan.partial, + relationStrategy: plan.relationStrategy, + version: 2, + warnings, + } +} + +export function getTemplateMetadataPath(dir: string): string { + return path.join(dir, META_FILE) +} + +export function readTemplateMetadata(dir: string): TemplateMetadata | undefined { + const filePath = getTemplateMetadataPath(dir) + if (!fs.existsSync(filePath)) return undefined + + return JSON.parse(fs.readFileSync(filePath, 'utf8')) as TemplateMetadata +} + +export async function writeTemplateMetadata( + dir: string, + plan: TemplatePlan, + warnings: TemplateWarning[] = [], +): Promise { + const filePath = getTemplateMetadataPath(dir) + await fs.promises.writeFile(filePath, JSON.stringify(createTemplateMetadata(plan, warnings), null, 2)) +} diff --git a/src/lib/template-plan/types.ts b/src/lib/template-plan/types.ts new file mode 100644 index 00000000..7145d28c --- /dev/null +++ b/src/lib/template-plan/types.ts @@ -0,0 +1,38 @@ +export type RelationStrategy = 'deep' | 'empty' | 'ids' + +export interface TemplateComponents { + content: boolean + dashboards: boolean + extensions: boolean + files: boolean + flows: boolean + permissions: boolean + schema: boolean + settings: boolean + users: boolean +} + +export interface TemplatePlan { + allowBrokenRelations: boolean + collections?: string[] + components: TemplateComponents + excludeCollections?: string[] + partial: boolean + relationStrategy: RelationStrategy +} + +export interface TemplateWarning { + collection?: string + count?: number + field?: string + relatedCollection?: string + type: string +} + +export interface TemplateMetadata extends Omit { + excludedCollections?: string[] + version: 2 + warnings: TemplateWarning[] +} + +export type TemplateComponent = keyof TemplateComponents diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts new file mode 100644 index 00000000..72c7da83 --- /dev/null +++ b/test/template-plan.test.ts @@ -0,0 +1,82 @@ +import {expect} from 'chai' + +import {buildTemplatePlan, createTemplateMetadata} from '../src/lib/template-plan/index.js' + +describe('template plan', () => { + it('defaults to full template', () => { + const plan = buildTemplatePlan({}) + + expect(plan.partial).to.equal(false) + expect(plan.relationStrategy).to.equal('deep') + expect(Object.values(plan.components).every(Boolean)).to.equal(true) + }) + + it('enables partial mode from --partial alone', () => { + const plan = buildTemplatePlan({partial: true}) + + expect(plan.partial).to.equal(true) + expect(plan.relationStrategy).to.equal('ids') + expect(Object.values(plan.components).every(Boolean)).to.equal(true) + }) + + it('enables partial mode from component flags', () => { + const plan = buildTemplatePlan({content: true, schema: true}) + + expect(plan.partial).to.equal(true) + expect(plan.relationStrategy).to.equal('ids') + expect(plan.components.content).to.equal(true) + expect(plan.components.schema).to.equal(true) + expect(plan.components.files).to.equal(false) + }) + + it('supports negative component flags', () => { + const plan = buildTemplatePlan({files: false}) + + expect(plan.partial).to.equal(true) + expect(plan.components.files).to.equal(false) + expect(plan.components.content).to.equal(true) + }) + + it('supports explicit relation strategy', () => { + const plan = buildTemplatePlan({relationStrategy: 'empty'}) + + expect(plan.partial).to.equal(true) + expect(plan.relationStrategy).to.equal('empty') + }) + + it('does not treat default false booleans as partial-only flags', () => { + const plan = buildTemplatePlan({allowBrokenRelations: false, noAssets: false}) + + expect(plan.partial).to.equal(false) + }) + + it('supports allow broken relations', () => { + const plan = buildTemplatePlan({allowBrokenRelations: true}) + + expect(plan.partial).to.equal(true) + expect(plan.allowBrokenRelations).to.equal(true) + }) + + it('parses collection filters', () => { + const plan = buildTemplatePlan({collections: 'posts, pages', excludeCollections: 'directus_files'}) + + expect(plan.partial).to.equal(true) + expect(plan.collections).to.deep.equal(['posts', 'pages']) + expect(plan.excludeCollections).to.deep.equal(['directus_files']) + }) + + it('expands noAssets', () => { + const plan = buildTemplatePlan({noAssets: true}) + + expect(plan.partial).to.equal(true) + expect(plan.components.files).to.equal(false) + expect(plan.excludeCollections).to.deep.equal(['directus_files']) + }) + + it('writes metadata with excludedCollections only', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'})) + + expect(metadata.excludedCollections).to.deep.equal(['directus_files']) + expect(metadata).not.to.have.property('excludeCollections') + }) +}) From cc7c3e22d1a269ab9da73266dd9e25bb6ab8d7fa Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 12:37:30 -0400 Subject: [PATCH 02/12] phase 2 + 3 --- src/lib/extract/extract-collections.ts | 9 ++++--- src/lib/extract/extract-content.ts | 6 ++--- src/lib/extract/extract-fields.ts | 4 ++- src/lib/extract/extract-relations.ts | 6 ++++- src/lib/extract/index.ts | 6 ++--- src/lib/load/index.ts | 16 +++++------ src/lib/load/load-collections.ts | 5 +++- src/lib/load/load-data.ts | 37 +++++++++++++------------- src/lib/load/load-relations.ts | 6 ++++- src/lib/load/update-required-fields.ts | 4 ++- src/lib/template-plan/collections.ts | 7 +++++ src/lib/template-plan/index.ts | 2 ++ src/lib/template-plan/metadata-plan.ts | 29 ++++++++++++++++++++ test/template-plan.test.ts | 17 +++++++++++- 14 files changed, 110 insertions(+), 44 deletions(-) create mode 100644 src/lib/template-plan/collections.ts create mode 100644 src/lib/template-plan/metadata-plan.ts diff --git a/src/lib/extract/extract-collections.ts b/src/lib/extract/extract-collections.ts index 54ab6d46..bb205665 100644 --- a/src/lib/extract/extract-collections.ts +++ b/src/lib/extract/extract-collections.ts @@ -2,6 +2,7 @@ import {readCollections} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -10,13 +11,13 @@ import writeToFile from '../utils/write-to-file.js' * Extract collections from the Directus instance */ -export default async function extractCollections(dir: string) { +export default async function extractCollections(dir: string, plan?: TemplatePlan) { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting collections')) try { const response = await api.client.request(readCollections()) - const collections = response.filter( - collection => !collection.collection.startsWith('directus_'), - ) + const collections = response + .filter(collection => !collection.collection.startsWith('directus_')) + .filter(collection => includesCollection(collection.collection, plan)) await writeToFile('collections', collections, dir) } catch (error) { catchError(error) diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index a77e204e..9a01a70d 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -1,9 +1,8 @@ import {readCollections, readItems} from '@directus/sdk' import {ux} from '@oclif/core' -import type {TemplatePlan} from '../template-plan/index.js' - import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -14,8 +13,7 @@ async function getCollections(plan?: TemplatePlan) { .filter((item) => !item.collection.startsWith('directus_', 0)) .filter((item) => item.schema != null) .map((i) => i.collection) - .filter((collection) => !plan?.collections || plan.collections.includes(collection)) - .filter((collection) => !plan?.excludeCollections?.includes(collection)) + .filter((collection) => includesCollection(collection, plan)) } async function getDataFromCollection(collection: string, dir: string) { diff --git a/src/lib/extract/extract-fields.ts b/src/lib/extract/extract-fields.ts index 53022037..661ff36b 100644 --- a/src/lib/extract/extract-fields.ts +++ b/src/lib/extract/extract-fields.ts @@ -2,6 +2,7 @@ import {readFields} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -10,7 +11,7 @@ import writeToFile from '../utils/write-to-file.js' * Extract fields from the Directus instance */ -export default async function extractFields(dir: string) { +export default async function extractFields(dir: string, plan?: TemplatePlan) { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting fields')) try { const response = await api.client.request(readFields()) @@ -24,6 +25,7 @@ export default async function extractFields(dir: string) { // @ts-ignore (i: { collection: string; meta?: { system?: boolean } }) => i.meta && !i.meta.system, ) + .filter((i: { collection: string }) => includesCollection(i.collection, plan)) .map(i => { if (i.meta) { delete i.meta.id diff --git a/src/lib/extract/extract-relations.ts b/src/lib/extract/extract-relations.ts index 52cf51dd..b86a70ba 100644 --- a/src/lib/extract/extract-relations.ts +++ b/src/lib/extract/extract-relations.ts @@ -2,6 +2,7 @@ import {readFields, readRelations} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -10,7 +11,7 @@ import writeToFile from '../utils/write-to-file.js' * Extract relations from the Directus instance */ -export default async function extractRelations(dir: string) { +export default async function extractRelations(dir: string, plan?: TemplatePlan) { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting relations')) try { const response = await api.client.request(readRelations()) @@ -33,6 +34,9 @@ export default async function extractRelations(dir: string) { f.collection === i.collection && f.field === i.field, ), ) + .filter((i: any) => includesCollection(i.collection, plan)) + // Phase 4 relation strategies may keep relations to excluded collections for ids/deep behavior. + .filter((i: any) => !i.related_collection || includesCollection(i.related_collection, plan)) .map(i => { delete i.meta.id return i diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index 78abf4c9..f4ffd3be 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -32,9 +32,9 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem if (plan.components.schema) { await extractSchema(destination) - await extractCollections(destination) - await extractFields(destination) - await extractRelations(destination) + await extractCollections(destination, plan) + await extractFields(destination, plan) + await extractRelations(destination, plan) } if (plan.components.files) { diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index e983292c..6a7b6b45 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -2,7 +2,7 @@ import {ux} from '@oclif/core' import type {ApplyFlags} from './apply-flags.js' -import {buildTemplatePlan, componentNames, readTemplateMetadata} from '../template-plan/index.js' +import {applyMetadataToPlan, buildTemplatePlan, readTemplateMetadata} from '../template-plan/index.js' import checkTemplate from '../utils/check-template.js' import loadAccess from './load-access.js' import loadCollections from './load-collections.js' @@ -26,13 +26,11 @@ export default async function apply(dir: string, flags: ApplyFlags) { const source = `${dir}/src` const metadata = readTemplateMetadata(source) const requestedPlan = buildTemplatePlan(flags) - const components = {...requestedPlan.components} + const effectivePlan = applyMetadataToPlan(requestedPlan, metadata) + const components = effectivePlan.components if (metadata?.partial) { ux.warn('Template metadata indicates this is a partial template.') - for (const component of componentNames) { - components[component] = components[component] && metadata.components[component] - } } if (!metadata || components.schema) { @@ -45,8 +43,8 @@ export default async function apply(dir: string, flags: ApplyFlags) { } if (components.schema) { - await loadCollections(source) - await loadRelations(source) + await loadCollections(source, effectivePlan) + await loadRelations(source, effectivePlan) } if (components.permissions || components.users) { @@ -67,11 +65,11 @@ export default async function apply(dir: string, flags: ApplyFlags) { } if (components.content) { - await loadData(source) + await loadData(source, effectivePlan) } if (components.schema) { - await updateRequiredFields(source) + await updateRequiredFields(source, effectivePlan) } if (components.dashboards) { diff --git a/src/lib/load/load-collections.ts b/src/lib/load/load-collections.ts index f5ef0652..852581c9 100644 --- a/src/lib/load/load-collections.ts +++ b/src/lib/load/load-collections.ts @@ -6,6 +6,7 @@ import { import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' @@ -15,9 +16,11 @@ import readFile from '../utils/read-file.js' * @param dir - The directory to read the collections and fields from * @returns {Promise} - Returns nothing */ -export default async function loadCollections(dir: string) { +export default async function loadCollections(dir: string, plan?: TemplatePlan) { const collectionsToAdd = readFile('collections', dir) + .filter(collection => includesCollection(collection.collection, plan)) const fieldsToAdd = readFile('fields', dir) + .filter(field => includesCollection(field.collection, plan)) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading ${collectionsToAdd.length} collections and ${fieldsToAdd.length} fields`)) diff --git a/src/lib/load/load-data.ts b/src/lib/load/load-data.ts index 848faf27..7b794f88 100644 --- a/src/lib/load/load-data.ts +++ b/src/lib/load/load-data.ts @@ -3,6 +3,7 @@ import {ux} from '@oclif/core' import path from 'pathe' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import {chunkArray} from '../utils/chunk-array.js' @@ -10,24 +11,29 @@ import readFile from '../utils/read-file.js' const BATCH_SIZE = 50 -export default async function loadData(dir:string) { - const collections = readFile('collections', dir) +export default async function loadData(dir:string, plan?: TemplatePlan) { + const collections = getUserCollections(dir, plan) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading data for ${collections.length} collections`)) - await loadSkeletonRecords(dir) - await loadFullData(dir) - await loadSingletons(dir) + await loadSkeletonRecords(dir, plan) + await loadFullData(dir, plan) + await loadSingletons(dir, plan) ux.action.stop() } -async function loadSkeletonRecords(dir: string) { - ux.action.status = 'Loading skeleton records' +function getUserCollections(dir: string, plan?: TemplatePlan) { const collections = readFile('collections', dir) - const primaryKeyMap = await getCollectionPrimaryKeys(dir) - const userCollections = collections + return collections .filter(item => !item.collection.startsWith('directus_', 0)) .filter(item => item.schema !== null) + .filter(item => includesCollection(item.collection, plan)) +} + +async function loadSkeletonRecords(dir: string, plan?: TemplatePlan) { + ux.action.status = 'Loading skeleton records' + const primaryKeyMap = await getCollectionPrimaryKeys(dir) + const userCollections = getUserCollections(dir, plan) .filter(item => !item.meta.singleton) await Promise.all(userCollections.map(async collection => { @@ -95,12 +101,9 @@ async function uploadBatch(collection: string, batch: any[], method: Function) { } } -async function loadFullData(dir:string) { +async function loadFullData(dir:string, plan?: TemplatePlan) { ux.action.status = 'Updating records with full data' - const collections = readFile('collections', dir) - const userCollections = collections - .filter(item => !item.collection.startsWith('directus_', 0)) - .filter(item => item.schema !== null) + const userCollections = getUserCollections(dir, plan) .filter(item => !item.meta.singleton) await Promise.all(userCollections.map(async collection => { @@ -118,11 +121,9 @@ async function loadFullData(dir:string) { ux.action.status = 'Updated records with full data' } -async function loadSingletons(dir:string) { +async function loadSingletons(dir:string, plan?: TemplatePlan) { ux.action.status = 'Loading data for singleton collections' - const collections = readFile('collections', dir) - const singletonCollections = collections - .filter(item => !item.collection.startsWith('directus_', 0)) + const singletonCollections = getUserCollections(dir, plan) .filter(item => item.meta.singleton) await Promise.all(singletonCollections.map(async collection => { diff --git a/src/lib/load/load-relations.ts b/src/lib/load/load-relations.ts index 97fe40e5..8de3a1f3 100644 --- a/src/lib/load/load-relations.ts +++ b/src/lib/load/load-relations.ts @@ -2,6 +2,7 @@ import {createRelation, readRelations} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' @@ -11,8 +12,11 @@ import readFile from '../utils/read-file.js' * @param dir - The directory to read the relations from * @returns {Promise} - Returns nothing */ -export default async function loadRelations(dir: string) { +export default async function loadRelations(dir: string, plan?: TemplatePlan) { const relations = readFile('relations', dir) + .filter(relation => includesCollection(relation.collection, plan)) + // Phase 4 relation strategies may keep relations to excluded collections for ids/deep behavior. + .filter(relation => !relation.related_collection || includesCollection(relation.related_collection, plan)) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading ${relations.length} relations`)) if (relations && relations.length > 0) { diff --git a/src/lib/load/update-required-fields.ts b/src/lib/load/update-required-fields.ts index cfa1ba2a..4e46ab9d 100644 --- a/src/lib/load/update-required-fields.ts +++ b/src/lib/load/update-required-fields.ts @@ -3,12 +3,14 @@ import {updateField} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' -export default async function updateRequiredFields(dir: string) { +export default async function updateRequiredFields(dir: string, plan?: TemplatePlan) { const fieldsToUpdate = readFile('fields', dir) + .filter(field => includesCollection(field.collection, plan)) .filter(field => field.meta.required === true || field.schema?.is_nullable === false || field.schema?.is_unique === true) ux.action.start(ux.colorize(DIRECTUS_PINK, `Updating ${fieldsToUpdate.length} fields to required`)) diff --git a/src/lib/template-plan/collections.ts b/src/lib/template-plan/collections.ts new file mode 100644 index 00000000..5d4536a9 --- /dev/null +++ b/src/lib/template-plan/collections.ts @@ -0,0 +1,7 @@ +import type {TemplatePlan} from './types.js' + +export function includesCollection(collection: string, plan?: TemplatePlan): boolean { + if (plan?.collections && !plan.collections.includes(collection)) return false + if (plan?.excludeCollections?.includes(collection)) return false + return true +} diff --git a/src/lib/template-plan/index.ts b/src/lib/template-plan/index.ts index 3165c41f..691ab74a 100644 --- a/src/lib/template-plan/index.ts +++ b/src/lib/template-plan/index.ts @@ -74,6 +74,8 @@ export function buildTemplatePlan(flags: any = {}): TemplatePlan { } } +export * from './collections.js' export * from './flags.js' export * from './metadata.js' +export * from './metadata-plan.js' export * from './types.js' diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts new file mode 100644 index 00000000..3c4d2f49 --- /dev/null +++ b/src/lib/template-plan/metadata-plan.ts @@ -0,0 +1,29 @@ +import {componentNames} from './flags.js' +import type {TemplateMetadata, TemplatePlan} from './types.js' + +function intersectCollections(requested?: string[], available?: string[]): string[] | undefined { + if (requested && available) return requested.filter(collection => available.includes(collection)) + return requested || available +} + +function mergeExcludedCollections(requested?: string[], available?: string[]): string[] | undefined { + const values = [...(requested || []), ...(available || [])] + return values.length > 0 ? [...new Set(values)] : undefined +} + +export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetadata): TemplatePlan { + if (!metadata) return plan + + const components = {...plan.components} + for (const component of componentNames) { + components[component] = components[component] && metadata.components[component] + } + + return { + ...plan, + collections: intersectCollections(plan.collections, metadata.collections), + components, + excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), + partial: plan.partial || metadata.partial, + } +} diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 72c7da83..062e404b 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -1,6 +1,6 @@ import {expect} from 'chai' -import {buildTemplatePlan, createTemplateMetadata} from '../src/lib/template-plan/index.js' +import {applyMetadataToPlan, buildTemplatePlan, createTemplateMetadata} from '../src/lib/template-plan/index.js' describe('template plan', () => { it('defaults to full template', () => { @@ -79,4 +79,19 @@ describe('template plan', () => { expect(metadata.excludedCollections).to.deep.equal(['directus_files']) expect(metadata).not.to.have.property('excludeCollections') }) + + it('applies metadata as available template bounds', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({collections: 'posts,pages', files: false})) + const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'posts,authors'}), metadata) + + expect(plan.components.files).to.equal(false) + expect(plan.collections).to.deep.equal(['posts']) + }) + + it('merges requested and metadata excluded collections', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'})) + const plan = applyMetadataToPlan(buildTemplatePlan({excludeCollections: 'analytics_events'}), metadata) + + expect(plan.excludeCollections).to.deep.equal(['analytics_events', 'directus_files']) + }) }) From 93ddd304251bbb467bf2780019b7615be8375449 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 12:38:27 -0400 Subject: [PATCH 03/12] fix test imports --- test/commands/apply.test.ts | 18 +----------------- test/helpers/init.js | 3 ++- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/test/commands/apply.test.ts b/test/commands/apply.test.ts index 2f5f457f..2c4807ac 100644 --- a/test/commands/apply.test.ts +++ b/test/commands/apply.test.ts @@ -1,17 +1 @@ -import {expect, test} from '@oclif/test' - -describe('apply', () => { - // test - // .stdout() - // .command(['apply']) - // .it('runs hello', ctx => { - // expect(ctx.stdout).to.contain('hello world') - // }) - - // test - // .stdout() - // .command(['apply', '--name', 'jeff']) - // .it('runs hello --name jeff', ctx => { - // expect(ctx.stdout).to.contain('hello jeff') - // }) -}) +describe('apply', () => {}) diff --git a/test/helpers/init.js b/test/helpers/init.js index baa3d810..907392f9 100644 --- a/test/helpers/init.js +++ b/test/helpers/init.js @@ -1,4 +1,5 @@ -const path = require('node:path') +import path from 'node:path' + process.env.TS_NODE_PROJECT = path.resolve('test/tsconfig.json') process.env.NODE_ENV = 'development' From be0c7eb6078afc0800a1d187c8a30c8cefab90d5 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 12:58:03 -0400 Subject: [PATCH 04/12] phase 4 --- src/lib/extract/extract-content.ts | 83 +++++++++++++++++++++++++--- src/lib/extract/extract-relations.ts | 6 +- src/lib/extract/index.ts | 12 +++- src/lib/load/index.ts | 7 ++- src/lib/load/load-relations.ts | 10 ++-- src/lib/template-plan/collections.ts | 7 +++ test/template-plan.test.ts | 38 +++++++++++-- test/tsconfig.json | 8 +-- 8 files changed, 142 insertions(+), 29 deletions(-) diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index 9a01a70d..2f5f1074 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -1,38 +1,107 @@ -import {readCollections, readItems} from '@directus/sdk' +import {readCollections, readItems, readRelations} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesCollection, type TemplatePlan, type TemplateWarning} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' +interface RelationInfo { + collection: string + field: string + related_collection?: string +} + async function getCollections(plan?: TemplatePlan) { const response = await api.client.request(readCollections()) return response .filter((item) => !item.collection.startsWith('directus_', 0)) - .filter((item) => item.schema != null) + .filter((item) => item.schema !== null) .map((i) => i.collection) .filter((collection) => includesCollection(collection, plan)) } -async function getDataFromCollection(collection: string, dir: string) { +function getExcludedRelationFields(collection: string, relations: RelationInfo[], plan?: TemplatePlan): RelationInfo[] { + if (!plan?.partial || plan.relationStrategy === 'deep') return [] + + return relations.filter((relation) => + relation.collection === collection && + relation.related_collection && + !includesCollection(relation.related_collection, plan), + ) +} + +function hasValue(value: unknown): boolean { + if (Array.isArray(value)) return value.length > 0 + return value !== null && value !== undefined +} + +function emptyExcludedRelations(items: Record[], relations: RelationInfo[]): void { + for (const item of items) { + for (const relation of relations) { + if (!(relation.field in item)) continue + item[relation.field] = Array.isArray(item[relation.field]) ? [] : null + } + } +} + +function getBrokenRelationWarnings( + collection: string, + items: Record[], + relations: RelationInfo[], +): TemplateWarning[] { + return relations + .map((relation) => ({ + collection, + count: items.filter((item) => hasValue(item[relation.field])).length, + field: relation.field, + relatedCollection: relation.related_collection, + type: 'excluded_relation', + })) + .filter((warning) => warning.count > 0) +} + +async function getDataFromCollection( + collection: string, + dir: string, + relations: RelationInfo[], + plan?: TemplatePlan, +): Promise { try { - const response = await api.client.request(readItems(collection as never, {limit: -1})) + const response = await api.client.request(readItems(collection as never, {limit: -1})) as Record[] + const excludedRelations = getExcludedRelationFields(collection, relations, plan) + + if (plan?.relationStrategy === 'empty') { + emptyExcludedRelations(response, excludedRelations) + } + + const warnings = plan?.relationStrategy === 'ids' + ? getBrokenRelationWarnings(collection, response, excludedRelations) + : [] + await writeToFile(`${collection}`, response, `${dir}/content/`) + return warnings } catch (error) { catchError(error) + return [] } } -export async function extractContent(dir: string, plan?: TemplatePlan) { +export async function extractContent(dir: string, plan?: TemplatePlan): Promise { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting content')) try { const collections = await getCollections(plan) - await Promise.all(collections.map((collection) => getDataFromCollection(collection, dir))) + const relations = await api.client.request(readRelations()) as RelationInfo[] + const warnings = await Promise.all( + collections.map((collection) => getDataFromCollection(collection, dir, relations, plan)), + ) + ux.action.stop() + return warnings.flat() } catch (error) { catchError(error) } ux.action.stop() + return [] } diff --git a/src/lib/extract/extract-relations.ts b/src/lib/extract/extract-relations.ts index b86a70ba..da636a12 100644 --- a/src/lib/extract/extract-relations.ts +++ b/src/lib/extract/extract-relations.ts @@ -2,8 +2,8 @@ import {readFields, readRelations} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesRelation, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -34,9 +34,7 @@ export default async function extractRelations(dir: string, plan?: TemplatePlan) f.collection === i.collection && f.field === i.field, ), ) - .filter((i: any) => includesCollection(i.collection, plan)) - // Phase 4 relation strategies may keep relations to excluded collections for ids/deep behavior. - .filter((i: any) => !i.related_collection || includesCollection(i.related_collection, plan)) + .filter((i: any) => includesRelation(i.collection, i.related_collection, plan)) .map(i => { delete i.meta.id return i diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index f4ffd3be..4a0cba6c 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -1,7 +1,7 @@ import {ux} from '@oclif/core' import fs from 'node:fs' -import {buildTemplatePlan, type TemplatePlan, writeTemplateMetadata} from '../template-plan/index.js' +import {buildTemplatePlan, type TemplatePlan, type TemplateWarning, writeTemplateMetadata} from '../template-plan/index.js' import extractAccess from './extract-access.js' import {downloadAllFiles} from './extract-assets.js' import extractCollections from './extract-collections.js' @@ -75,11 +75,17 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem await extractExtensions(destination) } + const warnings: TemplateWarning[] = [] + if (plan.components.content) { - await extractContent(destination, plan) + warnings.push(...await extractContent(destination, plan)) + } + + for (const warning of warnings) { + ux.warn(`Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`) } - await writeTemplateMetadata(destination, plan) + await writeTemplateMetadata(destination, plan, warnings) return {} } diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index 6a7b6b45..f7146867 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -27,12 +27,17 @@ export default async function apply(dir: string, flags: ApplyFlags) { const metadata = readTemplateMetadata(source) const requestedPlan = buildTemplatePlan(flags) const effectivePlan = applyMetadataToPlan(requestedPlan, metadata) - const components = effectivePlan.components + const {components} = effectivePlan if (metadata?.partial) { ux.warn('Template metadata indicates this is a partial template.') } + const brokenRelationWarnings = metadata?.warnings?.filter(warning => warning.type === 'excluded_relation') || [] + if (brokenRelationWarnings.length > 0 && !effectivePlan.allowBrokenRelations) { + ux.error('This partial template contains excluded relation references. Re-run with --allow-broken-relations to apply anyway.') + } + if (!metadata || components.schema) { const isTemplateOk = await checkTemplate(source) if (!isTemplateOk) { diff --git a/src/lib/load/load-relations.ts b/src/lib/load/load-relations.ts index 8de3a1f3..dc2fced4 100644 --- a/src/lib/load/load-relations.ts +++ b/src/lib/load/load-relations.ts @@ -2,8 +2,8 @@ import {createRelation, readRelations} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesRelation, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' @@ -14,9 +14,7 @@ import readFile from '../utils/read-file.js' */ export default async function loadRelations(dir: string, plan?: TemplatePlan) { const relations = readFile('relations', dir) - .filter(relation => includesCollection(relation.collection, plan)) - // Phase 4 relation strategies may keep relations to excluded collections for ids/deep behavior. - .filter(relation => !relation.related_collection || includesCollection(relation.related_collection, plan)) + .filter(relation => includesRelation(relation.collection, relation.related_collection, plan)) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading ${relations.length} relations`)) if (relations && relations.length > 0) { @@ -45,7 +43,9 @@ export default async function loadRelations(dir: string, plan?: TemplatePlan) { ux.action.stop() } -async function addRelations(relations: any[]) { +type TemplateRelation = Parameters[0] + +async function addRelations(relations: TemplateRelation[]) { for await (const relation of relations) { try { await api.client.request(createRelation(relation)) diff --git a/src/lib/template-plan/collections.ts b/src/lib/template-plan/collections.ts index 5d4536a9..c9d63ad8 100644 --- a/src/lib/template-plan/collections.ts +++ b/src/lib/template-plan/collections.ts @@ -5,3 +5,10 @@ export function includesCollection(collection: string, plan?: TemplatePlan): boo if (plan?.excludeCollections?.includes(collection)) return false return true } + +export function includesRelation(collection: string, relatedCollection?: null | string, plan?: TemplatePlan): boolean { + if (!includesCollection(collection, plan)) return false + if (!relatedCollection) return true + if (plan?.relationStrategy === 'ids') return true + return includesCollection(relatedCollection, plan) +} diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 062e404b..21ecb63f 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -1,6 +1,6 @@ import {expect} from 'chai' -import {applyMetadataToPlan, buildTemplatePlan, createTemplateMetadata} from '../src/lib/template-plan/index.js' +import {applyMetadataToPlan, buildTemplatePlan, createTemplateMetadata, includesRelation} from '../src/lib/template-plan/index.js' describe('template plan', () => { it('defaults to full template', () => { @@ -89,9 +89,39 @@ describe('template plan', () => { }) it('merges requested and metadata excluded collections', () => { - const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'})) - const plan = applyMetadataToPlan(buildTemplatePlan({excludeCollections: 'analytics_events'}), metadata) + const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files,assets'})) + const plan = applyMetadataToPlan(buildTemplatePlan({excludeCollections: 'analytics_events,assets'}), metadata) + + expect(plan.excludeCollections).to.deep.equal(['analytics_events', 'assets', 'directus_files']) + }) + + it('keeps relations to excluded collections for ids strategy', () => { + const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'ids'}) + + expect(includesRelation('posts', 'assets', plan)).to.equal(true) + }) + + it('drops relations to excluded collections for empty strategy', () => { + const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'empty'}) + + expect(includesRelation('posts', 'assets', plan)).to.equal(false) + }) + + it('drops relations to excluded collections for deep strategy until traversal is implemented', () => { + const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'deep'}) + + expect(includesRelation('posts', 'assets', plan)).to.equal(false) + }) + + it('preserves metadata warnings', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'}), [{ + collection: 'posts', + count: 1, + field: 'image', + relatedCollection: 'directus_files', + type: 'excluded_relation', + }]) - expect(plan.excludeCollections).to.deep.equal(['analytics_events', 'directus_files']) + expect(metadata.warnings).to.have.length(1) }) }) diff --git a/test/tsconfig.json b/test/tsconfig.json index 95898fce..6d4fa942 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -1,9 +1,7 @@ { "extends": "../tsconfig", "compilerOptions": { - "noEmit": true - }, - "references": [ - {"path": ".."} - ] + "noEmit": true, + "types": ["mocha", "node"] + } } From e9c6007aeb15b56e500a40c446a6bea1895e3b79 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 13:01:55 -0400 Subject: [PATCH 05/12] phase 5 --- src/lib/extract/expand-deep-plan.ts | 52 +++++++++++++++++++++++++++++ src/lib/extract/index.ts | 30 +++++++++-------- test/template-plan.test.ts | 8 ++++- 3 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 src/lib/extract/expand-deep-plan.ts diff --git a/src/lib/extract/expand-deep-plan.ts b/src/lib/extract/expand-deep-plan.ts new file mode 100644 index 00000000..cb72a9bc --- /dev/null +++ b/src/lib/extract/expand-deep-plan.ts @@ -0,0 +1,52 @@ +import {readCollections, readRelations} from '@directus/sdk' +import {ux} from '@oclif/core' + +import {api} from '../sdk.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' + +interface RelationInfo { + collection: string + related_collection?: null | string +} + +export async function expandDeepPlan(plan: TemplatePlan): Promise { + if (!plan.partial || plan.relationStrategy !== 'deep' || !plan.collections) return plan + + const collections = await api.client.request(readCollections()) + const availableCollections = collections + .filter((collection) => !collection.collection.startsWith('directus_', 0)) + .filter((collection) => collection.schema !== null) + .map((collection) => collection.collection) + .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) + + const available = new Set(availableCollections) + const selected = new Set(plan.collections.filter((collection) => available.has(collection))) + const relations = await api.client.request(readRelations()) as RelationInfo[] + + let changed = true + while (changed) { + changed = false + + for (const relation of relations) { + if (!relation.related_collection) continue + if (!selected.has(relation.collection)) continue + if (!available.has(relation.related_collection)) continue + if (selected.has(relation.related_collection)) continue + + selected.add(relation.related_collection) + changed = true + } + } + + const expandedCollections = [...selected] + const addedCollections = expandedCollections.filter((collection) => !plan.collections?.includes(collection)) + + if (addedCollections.length > 0) { + ux.warn(`Deep relation strategy expanded collections: ${addedCollections.join(', ')}`) + } + + return { + ...plan, + collections: expandedCollections, + } +} diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index 4a0cba6c..347c1a55 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -2,6 +2,7 @@ import {ux} from '@oclif/core' import fs from 'node:fs' import {buildTemplatePlan, type TemplatePlan, type TemplateWarning, writeTemplateMetadata} from '../template-plan/index.js' +import {expandDeepPlan} from './expand-deep-plan.js' import extractAccess from './extract-access.js' import {downloadAllFiles} from './extract-assets.js' import extractCollections from './extract-collections.js' @@ -24,68 +25,69 @@ import extractUsers from './extract-users.js' export default async function extract(dir: string, plan: TemplatePlan = buildTemplatePlan()) { const destination = `${dir}/src` + const effectivePlan = await expandDeepPlan(plan) if (!fs.existsSync(destination)) { ux.stdout(`Attempting to create directory at: ${destination}`) fs.mkdirSync(destination, {recursive: true}) } - if (plan.components.schema) { + if (effectivePlan.components.schema) { await extractSchema(destination) - await extractCollections(destination, plan) - await extractFields(destination, plan) - await extractRelations(destination, plan) + await extractCollections(destination, effectivePlan) + await extractFields(destination, effectivePlan) + await extractRelations(destination, effectivePlan) } - if (plan.components.files) { + if (effectivePlan.components.files) { await extractFolders(destination) await extractFiles(destination) await downloadAllFiles(destination) } - if (plan.components.users || plan.components.permissions) { + if (effectivePlan.components.users || effectivePlan.components.permissions) { await extractRoles(destination) await extractPermissions(destination) await extractPolicies(destination) - if (plan.components.users) { + if (effectivePlan.components.users) { await extractUsers(destination) } await extractAccess(destination) } - if (plan.components.settings) { + if (effectivePlan.components.settings) { await extractPresets(destination) await extractTranslations(destination) await extractSettings(destination) } - if (plan.components.flows) { + if (effectivePlan.components.flows) { await extractFlows(destination) await extractOperations(destination) } - if (plan.components.dashboards) { + if (effectivePlan.components.dashboards) { await extractDashboards(destination) await extractPanels(destination) } - if (plan.components.extensions) { + if (effectivePlan.components.extensions) { await extractExtensions(destination) } const warnings: TemplateWarning[] = [] - if (plan.components.content) { - warnings.push(...await extractContent(destination, plan)) + if (effectivePlan.components.content) { + warnings.push(...await extractContent(destination, effectivePlan)) } for (const warning of warnings) { ux.warn(`Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`) } - await writeTemplateMetadata(destination, plan, warnings) + await writeTemplateMetadata(destination, effectivePlan, warnings) return {} } diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 21ecb63f..5ae31d0a 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -107,7 +107,13 @@ describe('template plan', () => { expect(includesRelation('posts', 'assets', plan)).to.equal(false) }) - it('drops relations to excluded collections for deep strategy until traversal is implemented', () => { + it('keeps relations to included collections for deep strategy', () => { + const plan = buildTemplatePlan({collections: 'posts,assets', relationStrategy: 'deep'}) + + expect(includesRelation('posts', 'assets', plan)).to.equal(true) + }) + + it('drops relations to excluded collections for deep strategy', () => { const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'deep'}) expect(includesRelation('posts', 'assets', plan)).to.equal(false) From 2232764a21881df31e8592da944d3b1f1079fdbb Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 13:11:59 -0400 Subject: [PATCH 06/12] Cleanup --- README.md | 74 +++++++++++++++++++++--------- src/lib/extract/extract-assets.ts | 29 +++++++++--- src/lib/extract/extract-content.ts | 36 +++++++++++---- 3 files changed, 103 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2c222a49..f39a2931 100644 --- a/README.md +++ b/README.md @@ -151,11 +151,10 @@ Using email/password: npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --userEmail="admin@example.com" --userPassword="admin" --templateLocation="./my-template" --templateType="local" ``` -Partial apply (apply only some of the parts of a template to the instance): +Partial apply (apply only some parts of a template): ``` -npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --userEmail="admin@example.com" --userPassword="your-password" --templateLocation="./my-template" --templateType="local" --partial --schema --permissions --no-content - +npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --userEmail="admin@example.com" --userPassword="your-password" --templateLocation="./my-template" --templateType="local" --schema --permissions --no-content ``` Available flags: @@ -166,7 +165,7 @@ Available flags: - `--userPassword`: Password for Directus authentication (required if not using token) - `--templateLocation`: Location of the template to apply (required) - `--templateType`: Type of template to apply. Options: community, local, github. Defaults to `local`. -- `--partial`: Enable partial template application +- `--partial`: Enable partial template mode explicitly. Component and collection flags also imply partial mode. - `--content`: Load Content (data) - `--dashboards`: Load Dashboards - `--extensions`: Load Extensions @@ -176,6 +175,11 @@ Available flags: - `--schema`: Load Schema - `--settings`: Load Settings - `--users`: Load Users +- `--collections`: Only apply these comma-separated collections +- `--excludeCollections`: Exclude these comma-separated collections +- `--relationStrategy`: How to handle omitted relation targets. Options: `empty`, `ids`, `deep`. +- `--allowBrokenRelations`: Apply templates that intentionally preserve references to omitted records +- `--noAssets`: Shorthand for `--no-files` and excluding `directus_files` - `--disableTelemetry`: Disable telemetry collection When using `--partial`, you can also use `--no` flags to exclude specific components from being applied. For example: @@ -197,29 +201,22 @@ This command will apply the template but exclude content and users. Available `- - `--no-users`: Skip loading Users -#### Template Component Dependencies +#### Partial Templates and Relations + +Partial templates can intentionally omit components or collections. The CLI writes and reads `src/template-meta.json` so apply can understand what was extracted. -When applying templates, certain components have dependencies on others. Here are the key relationships to be aware of: +Relation strategies: -- `--users`: Depends on `--permissions`. If you include users, permissions will automatically be included. -- `--permissions`: Depends on `--schema`. If you include permissions, the schema will automatically be included. -- `--content`: Depends on `--schema`. If you include content, the schema will automatically be included. -- `--files`: No direct dependencies, but often related to content. Consider including `--content` if you're including files. -- `--flows`: No direct dependencies, but may interact with other components. Consider your specific use case. -- `--dashboards`: No direct dependencies, but often rely on data from other components. -- `--extensions`: No direct dependencies, but may interact with other components. -- `--settings`: No direct dependencies, but affects the overall system configuration. +- `empty`: Relations to omitted collections are exported as `null` or `[]`. +- `ids`: Relation IDs are preserved, but omitted related records are not exported. Apply requires `--allowBrokenRelations` when metadata reports these references. +- `deep`: Related collections are added recursively where possible. Explicit exclusions still win. -When using the `--partial` flag, keep these dependencies in mind. For example: +Skip assets safely: ``` -npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local" --partial --users +npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local" --content --collections posts,pages --noAssets --relationStrategy empty ``` -This command will automatically include `--permissions` and `--schema` along with `--users`, even if not explicitly specified. - -If you use `--no-` flags, be cautious about excluding dependencies. For instance, using `--no-schema` while including `--content` may lead to errors or incomplete application of the template. - #### Using Environment Variables You can also pass flags as environment variables. This can be useful for CI/CD pipelines or when you want to avoid exposing sensitive information in command-line arguments. Here are the available environment variables: @@ -254,7 +251,7 @@ For data in your own user-created collections, if an item has the same primary k The CLI can also extract a template from a Directus instance so that it can be applied to other instances. -Note: We do not currently support partial extraction. The entire template will be extracted. We thought it better to have the data and not need it, than need it and not have it. +Full extraction remains the default. Partial extraction is available with component flags, collection filters, and relation strategies. 1. Make sure you remove any sensitive data from the Directus instance you don't want to include in the template. 2. Login and create a Static Access Token for the admin user. @@ -289,8 +286,43 @@ Available flags: - `--userPassword`: Password for Directus authentication (required if not using token) - `--templateLocation`: Directory to extract the template to (required) - `--templateName`: Name of the template (required) +- `--partial`: Enable partial template mode explicitly. Component and collection flags also imply partial mode. +- `--schema` / `--no-schema`: Include or skip schema +- `--content` / `--no-content`: Include or skip content +- `--files` / `--no-files`: Include or skip files and assets +- `--flows` / `--no-flows`: Include or skip flows +- `--dashboards` / `--no-dashboards`: Include or skip dashboards +- `--permissions` / `--no-permissions`: Include or skip permissions +- `--settings` / `--no-settings`: Include or skip settings +- `--extensions` / `--no-extensions`: Include or skip extensions +- `--users` / `--no-users`: Include or skip users +- `--collections`: Only extract these comma-separated collections +- `--excludeCollections`: Exclude these comma-separated collections +- `--relationStrategy`: How to handle omitted relation targets. Options: `empty`, `ids`, `deep`. +- `--allowBrokenRelations`: Mark intentionally incomplete relation references as allowed in metadata +- `--noAssets`: Shorthand for `--no-files` and excluding `directus_files` - `--disableTelemetry`: Disable telemetry collection +Examples: + +Skip assets safely: + +``` +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --noAssets --relationStrategy empty +``` + +Preserve asset IDs but do not export assets: + +``` +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --noAssets --relationStrategy ids --allowBrokenRelations +``` + +Portable partial snapshot: + +``` +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --relationStrategy deep +``` + #### Using Environment Variables Similar to the Apply command, you can use environment variables for the Extract command as well: diff --git a/src/lib/extract/extract-assets.ts b/src/lib/extract/extract-assets.ts index 06897a4e..5d5beb7d 100644 --- a/src/lib/extract/extract-assets.ts +++ b/src/lib/extract/extract-assets.ts @@ -7,11 +7,19 @@ import {DIRECTUS_PINK} from '../constants.js' import {api} from '../sdk.js' import catchError from '../utils/catch-error.js' -async function getAssetList() { - return api.client.request(readFiles({limit: -1})) +// Keep asset pages conservative because each item may trigger a binary download. +const PAGE_SIZE = 100 + +interface DirectusFile { + filename_disk: string + id: string +} + +async function getAssetPage(page: number): Promise { + return api.client.request(readFiles({limit: PAGE_SIZE, page})) as unknown as DirectusFile[] } -async function downloadFile(file: any, dir: string) { +async function downloadFile(file: DirectusFile, dir: string) { const response: Response | string = await api.client.request(() => ({ method: 'GET', path: `/assets/${file.id}`, @@ -34,10 +42,17 @@ export async function downloadAllFiles(dir: string) { fs.mkdirSync(fullPath, {recursive: true}) } - const fileList = await getAssetList() - await Promise.all(fileList.map(file => downloadFile(file, dir).catch(error => { - catchError(`Error downloading ${file.filename_disk}: ${error.message}`) - }))) + let page = 1 + while (true) { + ux.action.status = `Downloading assets page ${page}` + const fileList = await getAssetPage(page) + await Promise.all(fileList.map(file => downloadFile(file, dir).catch(error => { + catchError(`Error downloading ${file.filename_disk}: ${error.message}`) + }))) + + if (fileList.length < PAGE_SIZE) break + page++ + } } catch (error) { catchError(error) } diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index 2f5f1074..17892f26 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -7,10 +7,13 @@ import {includesCollection, type TemplatePlan, type TemplateWarning} from '../te import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' +// Content items are JSON-only, so pages can be larger than asset download pages. +const PAGE_SIZE = 500 + interface RelationInfo { collection: string field: string - related_collection?: string + related_collection?: null | string } async function getCollections(plan?: TemplatePlan) { @@ -22,6 +25,21 @@ async function getCollections(plan?: TemplatePlan) { .filter((collection) => includesCollection(collection, plan)) } +async function getCollectionItems(collection: string): Promise[]> { + const items: Record[] = [] + let page = 1 + + while (true) { + const response = await api.client.request(readItems(collection as never, {limit: PAGE_SIZE, page})) as Record[] + items.push(...response) + + if (response.length < PAGE_SIZE) break + page++ + } + + return items +} + function getExcludedRelationFields(collection: string, relations: RelationInfo[], plan?: TemplatePlan): RelationInfo[] { if (!plan?.partial || plan.relationStrategy === 'deep') return [] @@ -69,7 +87,8 @@ async function getDataFromCollection( plan?: TemplatePlan, ): Promise { try { - const response = await api.client.request(readItems(collection as never, {limit: -1})) as Record[] + ux.action.status = `Extracting content: ${collection}` + const response = await getCollectionItems(collection) const excludedRelations = getExcludedRelationFields(collection, relations, plan) if (plan?.relationStrategy === 'empty') { @@ -90,18 +109,19 @@ async function getDataFromCollection( export async function extractContent(dir: string, plan?: TemplatePlan): Promise { ux.action.start(ux.colorize(DIRECTUS_PINK, 'Extracting content')) + const warnings: TemplateWarning[] = [] + try { const collections = await getCollections(plan) const relations = await api.client.request(readRelations()) as RelationInfo[] - const warnings = await Promise.all( - collections.map((collection) => getDataFromCollection(collection, dir, relations, plan)), - ) - ux.action.stop() - return warnings.flat() + + for (const collection of collections) { + warnings.push(...await getDataFromCollection(collection, dir, relations, plan)) + } } catch (error) { catchError(error) } ux.action.stop() - return [] + return warnings } From 4241a86bd8d6b895a9d1f17e85d73be928f928c4 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Tue, 12 May 2026 13:57:00 -0400 Subject: [PATCH 07/12] moar clenaup --- src/lib/extract/expand-deep-plan.ts | 74 +++++++++++++++----------- src/lib/extract/extract-content.ts | 18 +++++-- src/lib/extract/index.ts | 19 +++++-- src/lib/load/apply-flags.ts | 29 +++++----- src/lib/load/index.ts | 6 ++- src/lib/load/load-relations.ts | 3 +- src/lib/template-plan/index.ts | 6 ++- src/lib/template-plan/metadata-plan.ts | 13 +++-- src/lib/template-plan/metadata.ts | 13 ++++- src/lib/template-plan/types.ts | 12 ++--- test/commands/apply.test.ts | 1 - test/template-plan.test.ts | 29 ++++++++++ 12 files changed, 153 insertions(+), 70 deletions(-) delete mode 100644 test/commands/apply.test.ts diff --git a/src/lib/extract/expand-deep-plan.ts b/src/lib/extract/expand-deep-plan.ts index cb72a9bc..681bd7e2 100644 --- a/src/lib/extract/expand-deep-plan.ts +++ b/src/lib/extract/expand-deep-plan.ts @@ -3,6 +3,7 @@ import {ux} from '@oclif/core' import {api} from '../sdk.js' import {includesCollection, type TemplatePlan} from '../template-plan/index.js' +import catchError from '../utils/catch-error.js' interface RelationInfo { collection: string @@ -12,41 +13,50 @@ interface RelationInfo { export async function expandDeepPlan(plan: TemplatePlan): Promise { if (!plan.partial || plan.relationStrategy !== 'deep' || !plan.collections) return plan - const collections = await api.client.request(readCollections()) - const availableCollections = collections - .filter((collection) => !collection.collection.startsWith('directus_', 0)) - .filter((collection) => collection.schema !== null) - .map((collection) => collection.collection) - .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) - - const available = new Set(availableCollections) - const selected = new Set(plan.collections.filter((collection) => available.has(collection))) - const relations = await api.client.request(readRelations()) as RelationInfo[] - - let changed = true - while (changed) { - changed = false - - for (const relation of relations) { - if (!relation.related_collection) continue - if (!selected.has(relation.collection)) continue - if (!available.has(relation.related_collection)) continue - if (selected.has(relation.related_collection)) continue - - selected.add(relation.related_collection) - changed = true + try { + const collections = await api.client.request(readCollections()) + const availableCollections = collections + .filter((collection) => !collection.collection.startsWith('directus_', 0)) + .filter((collection) => collection.schema !== null) + .map((collection) => collection.collection) + .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) + + const available = new Set(availableCollections) + const missingCollections = plan.collections.filter((collection) => !available.has(collection)) + if (missingCollections.length > 0) { + ux.warn(`Requested collections not found or excluded: ${missingCollections.join(', ')}`) } - } - const expandedCollections = [...selected] - const addedCollections = expandedCollections.filter((collection) => !plan.collections?.includes(collection)) + const selected = new Set(plan.collections.filter((collection) => available.has(collection))) + const relations = await api.client.request(readRelations()) as RelationInfo[] - if (addedCollections.length > 0) { - ux.warn(`Deep relation strategy expanded collections: ${addedCollections.join(', ')}`) - } + let changed = true + while (changed) { + changed = false + + for (const relation of relations) { + if (!relation.related_collection) continue + if (!selected.has(relation.collection)) continue + if (!available.has(relation.related_collection)) continue + if (selected.has(relation.related_collection)) continue + + selected.add(relation.related_collection) + changed = true + } + } - return { - ...plan, - collections: expandedCollections, + const expandedCollections = [...selected] + const addedCollections = expandedCollections.filter((collection) => !plan.collections?.includes(collection)) + + if (addedCollections.length > 0) { + ux.warn(`Deep relation strategy expanded collections: ${addedCollections.join(', ')}`) + } + + return { + ...plan, + collections: expandedCollections, + } + } catch (error) { + catchError(error, {fatal: true}) } } diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index 17892f26..d46cca7b 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -30,6 +30,7 @@ async function getCollectionItems(collection: string): Promise[] items.push(...response) @@ -70,7 +71,8 @@ function getBrokenRelationWarnings( relations: RelationInfo[], ): TemplateWarning[] { return relations - .map((relation) => ({ + .filter((relation): relation is RelationInfo & {related_collection: string} => Boolean(relation.related_collection)) + .map((relation): TemplateWarning => ({ collection, count: items.filter((item) => hasValue(item[relation.field])).length, field: relation.field, @@ -102,7 +104,10 @@ async function getDataFromCollection( await writeToFile(`${collection}`, response, `${dir}/content/`) return warnings } catch (error) { - catchError(error) + catchError(error, { + context: {collection, function: 'getDataFromCollection'}, + fatal: true, + }) return [] } } @@ -116,10 +121,15 @@ export async function extractContent(dir: string, plan?: TemplatePlan): Promise< const relations = await api.client.request(readRelations()) as RelationInfo[] for (const collection of collections) { - warnings.push(...await getDataFromCollection(collection, dir, relations, plan)) + // eslint-disable-next-line no-await-in-loop + const collectionWarnings = await getDataFromCollection(collection, dir, relations, plan) + warnings.push(...collectionWarnings) } } catch (error) { - catchError(error) + catchError(error, { + context: {function: 'extractContent'}, + fatal: true, + }) } ux.action.stop() diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index 347c1a55..94b7ff33 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -2,6 +2,7 @@ import {ux} from '@oclif/core' import fs from 'node:fs' import {buildTemplatePlan, type TemplatePlan, type TemplateWarning, writeTemplateMetadata} from '../template-plan/index.js' +import catchError from '../utils/catch-error.js' import {expandDeepPlan} from './expand-deep-plan.js' import extractAccess from './extract-access.js' import {downloadAllFiles} from './extract-assets.js' @@ -29,7 +30,11 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem if (!fs.existsSync(destination)) { ux.stdout(`Attempting to create directory at: ${destination}`) - fs.mkdirSync(destination, {recursive: true}) + try { + fs.mkdirSync(destination, {recursive: true}) + } catch (error) { + catchError(error, {context: {destination}, fatal: true}) + } } if (effectivePlan.components.schema) { @@ -80,14 +85,22 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem const warnings: TemplateWarning[] = [] if (effectivePlan.components.content) { - warnings.push(...await extractContent(destination, effectivePlan)) + const contentWarnings = await extractContent(destination, effectivePlan) + warnings.push(...contentWarnings) } for (const warning of warnings) { ux.warn(`Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`) } - await writeTemplateMetadata(destination, effectivePlan, warnings) + try { + await writeTemplateMetadata(destination, effectivePlan, warnings) + } catch (error) { + catchError(error, { + context: {function: 'writeTemplateMetadata'}, + fatal: true, + }) + } return {} } diff --git a/src/lib/load/apply-flags.ts b/src/lib/load/apply-flags.ts index 31060377..7f010222 100644 --- a/src/lib/load/apply-flags.ts +++ b/src/lib/load/apply-flags.ts @@ -1,7 +1,5 @@ import {ux} from '@oclif/core' -import {buildTemplatePlan, componentNames} from '../template-plan/index.js' - export interface ApplyFlags { allowBrokenRelations?: boolean collections?: string @@ -29,7 +27,17 @@ export interface ApplyFlags { users?: boolean } -export const loadFlags = componentNames +export const loadFlags = [ + 'content', + 'dashboards', + 'extensions', + 'files', + 'flows', + 'permissions', + 'schema', + 'settings', + 'users', +] as const export function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags { const {directusToken, directusUrl, templateLocation, userEmail, userPassword} = flags @@ -39,20 +47,9 @@ export function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags { ux.error('Either Directus token or email and password are required for programmatic mode.') if (!templateLocation) ux.error('Template location is required for programmatic mode.') - return applyTemplatePlan(flags) + return flags } export function validateInteractiveFlags(flags: ApplyFlags): ApplyFlags { - return applyTemplatePlan(flags) -} - -function applyTemplatePlan(flags: ApplyFlags): ApplyFlags { - const plan = buildTemplatePlan(flags) - const nextFlags = {...flags, partial: plan.partial} - - for (const flag of loadFlags) nextFlags[flag] = plan.components[flag] - - if (plan.partial) ux.warn('Applying partial template. Missing components will not be auto-enabled.') - - return nextFlags + return flags } diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index f7146867..4061c9c1 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -29,12 +29,14 @@ export default async function apply(dir: string, flags: ApplyFlags) { const effectivePlan = applyMetadataToPlan(requestedPlan, metadata) const {components} = effectivePlan - if (metadata?.partial) { + if (!metadata) { + ux.warn('No template-meta.json found. Treating as a full template — relation integrity check skipped.') + } else if (metadata.partial) { ux.warn('Template metadata indicates this is a partial template.') } const brokenRelationWarnings = metadata?.warnings?.filter(warning => warning.type === 'excluded_relation') || [] - if (brokenRelationWarnings.length > 0 && !effectivePlan.allowBrokenRelations) { + if (components.content && brokenRelationWarnings.length > 0 && !effectivePlan.allowBrokenRelations) { ux.error('This partial template contains excluded relation references. Re-run with --allow-broken-relations to apply anyway.') } diff --git a/src/lib/load/load-relations.ts b/src/lib/load/load-relations.ts index dc2fced4..79a6da3d 100644 --- a/src/lib/load/load-relations.ts +++ b/src/lib/load/load-relations.ts @@ -46,8 +46,9 @@ export default async function loadRelations(dir: string, plan?: TemplatePlan) { type TemplateRelation = Parameters[0] async function addRelations(relations: TemplateRelation[]) { - for await (const relation of relations) { + for (const relation of relations) { try { + // eslint-disable-next-line no-await-in-loop await api.client.request(createRelation(relation)) } catch (error) { catchError(error) diff --git a/src/lib/template-plan/index.ts b/src/lib/template-plan/index.ts index 691ab74a..136a5e2f 100644 --- a/src/lib/template-plan/index.ts +++ b/src/lib/template-plan/index.ts @@ -13,6 +13,7 @@ function parseList(value?: string | string[]): string[] | undefined { return values.length > 0 ? values : undefined } +// eslint-disable-next-line @typescript-eslint/no-explicit-any function hasPartialOnlyFlags(flags: any): boolean { return Boolean( flags.collections || @@ -23,10 +24,12 @@ function hasPartialOnlyFlags(flags: any): boolean { ) } +// eslint-disable-next-line @typescript-eslint/no-explicit-any function hasComponentFlags(flags: any): boolean { return componentNames.some((component) => flags[component] !== undefined) } +// eslint-disable-next-line @typescript-eslint/no-explicit-any function buildComponents(flags: any, partial: boolean): TemplateComponents { const components = {} as TemplateComponents const enabled = componentNames.filter((component) => flags[component] === true) @@ -50,6 +53,7 @@ function buildComponents(flags: any, partial: boolean): TemplateComponents { return components } +// eslint-disable-next-line @typescript-eslint/no-explicit-any export function buildTemplatePlan(flags: any = {}): TemplatePlan { const collections = parseList(flags.collections) const excludeCollections = parseList(flags.excludeCollections) || (flags.noAssets ? [] : undefined) @@ -76,6 +80,6 @@ export function buildTemplatePlan(flags: any = {}): TemplatePlan { export * from './collections.js' export * from './flags.js' -export * from './metadata.js' export * from './metadata-plan.js' +export * from './metadata.js' export * from './types.js' diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts index 3c4d2f49..8a245086 100644 --- a/src/lib/template-plan/metadata-plan.ts +++ b/src/lib/template-plan/metadata-plan.ts @@ -1,8 +1,13 @@ -import {componentNames} from './flags.js' import type {TemplateMetadata, TemplatePlan} from './types.js' +import {componentNames} from './flags.js' + function intersectCollections(requested?: string[], available?: string[]): string[] | undefined { - if (requested && available) return requested.filter(collection => available.includes(collection)) + if (requested && available) { + const collections = requested.filter(collection => available.includes(collection)) + return collections.length > 0 ? collections : undefined + } + return requested || available } @@ -19,11 +24,13 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad components[component] = components[component] && metadata.components[component] } + const partial = metadata.partial || componentNames.some(component => components[component] !== plan.components[component]) + return { ...plan, collections: intersectCollections(plan.collections, metadata.collections), components, excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), - partial: plan.partial || metadata.partial, + partial, } } diff --git a/src/lib/template-plan/metadata.ts b/src/lib/template-plan/metadata.ts index 29b73fb2..c227de9f 100644 --- a/src/lib/template-plan/metadata.ts +++ b/src/lib/template-plan/metadata.ts @@ -3,6 +3,8 @@ import path from 'pathe' import type {TemplateMetadata, TemplatePlan, TemplateWarning} from './types.js' +import catchError from '../utils/catch-error.js' + const META_FILE = 'template-meta.json' export function createTemplateMetadata(plan: TemplatePlan, warnings: TemplateWarning[] = []): TemplateMetadata { @@ -26,7 +28,16 @@ export function readTemplateMetadata(dir: string): TemplateMetadata | undefined const filePath = getTemplateMetadataPath(dir) if (!fs.existsSync(filePath)) return undefined - return JSON.parse(fs.readFileSync(filePath, 'utf8')) as TemplateMetadata + try { + const metadata = JSON.parse(fs.readFileSync(filePath, 'utf8')) as TemplateMetadata + if (metadata.version !== 2) { + catchError(new Error(`Unsupported template metadata version: ${metadata.version}`), {fatal: true}) + } + + return metadata + } catch (error) { + catchError(error, {fatal: true}) + } } export async function writeTemplateMetadata( diff --git a/src/lib/template-plan/types.ts b/src/lib/template-plan/types.ts index 7145d28c..b2a1d866 100644 --- a/src/lib/template-plan/types.ts +++ b/src/lib/template-plan/types.ts @@ -21,12 +21,12 @@ export interface TemplatePlan { relationStrategy: RelationStrategy } -export interface TemplateWarning { - collection?: string - count?: number - field?: string - relatedCollection?: string - type: string +export type TemplateWarning = { + collection: string + count: number + field: string + relatedCollection: string + type: 'excluded_relation' } export interface TemplateMetadata extends Omit { diff --git a/test/commands/apply.test.ts b/test/commands/apply.test.ts deleted file mode 100644 index 2c4807ac..00000000 --- a/test/commands/apply.test.ts +++ /dev/null @@ -1 +0,0 @@ -describe('apply', () => {}) diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 5ae31d0a..fdc8c92d 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -130,4 +130,33 @@ describe('template plan', () => { expect(metadata.warnings).to.have.length(1) }) + + it('returns plan unchanged when no metadata (legacy fallback)', () => { + const plan = buildTemplatePlan({content: true, schema: true}) + const result = applyMetadataToPlan(plan) + + expect(result).to.equal(plan) + }) + + it('re-derives partial when metadata disables a component', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({files: false})) + const plan = applyMetadataToPlan(buildTemplatePlan({}), metadata) + + expect(plan.partial).to.equal(true) + expect(plan.components.files).to.equal(false) + }) + + it('does not apply requested collections outside metadata bounds', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({collections: 'posts,pages'})) + const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'posts,authors'}), metadata) + + expect(plan.collections).to.deep.equal(['posts']) + }) + + it('returns undefined collections when requested has no overlap with metadata', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({collections: 'posts'})) + const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'authors'}), metadata) + + expect(plan.collections).to.equal(undefined) + }) }) From b3fa0d5cf177261e196f945282653effd5c349a5 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Wed, 13 May 2026 11:07:31 -0400 Subject: [PATCH 08/12] fix relations --- README.md | 137 ++++++++++++++------ src/commands/extract.ts | 2 +- src/lib/extract/expand-deep-plan.ts | 27 +++- src/lib/extract/expand-schema-plan.ts | 86 +++++++++++++ src/lib/extract/extract-assets.ts | 11 +- src/lib/extract/extract-collections.ts | 6 +- src/lib/extract/extract-content.ts | 110 ++++++++++++---- src/lib/extract/extract-fields.ts | 23 ++-- src/lib/extract/extract-relations.ts | 31 ++--- src/lib/extract/index.ts | 15 ++- src/lib/load/apply-flags.ts | 2 +- src/lib/load/finalize-collections.ts | 34 +++++ src/lib/load/finalize-fields.ts | 29 +++++ src/lib/load/index.ts | 15 ++- src/lib/load/load-collections.ts | 64 ++++------ src/lib/load/load-data.ts | 170 ++++++++++++++++--------- src/lib/load/load-relations.ts | 39 +++--- src/lib/load/update-required-fields.ts | 14 +- src/lib/template-plan/collections.ts | 14 +- src/lib/template-plan/flags.ts | 6 +- src/lib/template-plan/index.ts | 30 +++-- src/lib/template-plan/metadata-plan.ts | 6 +- src/lib/template-plan/metadata.ts | 1 + src/lib/template-plan/types.ts | 3 +- test/template-plan.test.ts | 49 +++++-- 25 files changed, 658 insertions(+), 266 deletions(-) create mode 100644 src/lib/extract/expand-schema-plan.ts create mode 100644 src/lib/load/finalize-collections.ts create mode 100644 src/lib/load/finalize-fields.ts diff --git a/README.md b/README.md index f39a2931..0227cebd 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ A streamlined CLI tool for creating new Directus projects and managing Directus templates - making it easy to apply and extract template configurations across instances. This tool is best suited for: + - Proof of Concept (POC) projects - Demo environments - New project setups @@ -10,8 +11,9 @@ This tool is best suited for: ⚠️ We strongly recommend against using this tool in existing production environments or as a critical part of your CI/CD pipeline without thorough testing. Always create backups before applying templates. **Important Notes:** + - **Primary Purpose**: Built to deploy templates created by the Directus Core Team. While community templates are supported, the unlimited possible configurations make comprehensive support challenging. -- **Database Compatibility**: PostgreSQL is recommended. Applying templates that are extracted and applied between different databases (Extract from SQLite -> Apply to Postgres) can caused issues and is not recommended. MySQL users may encounter known issues. +- **Database Compatibility**: PostgreSQL is recommended. Applying templates that are extracted and applied between different databases (Extract from SQLite -> Apply to Postgres) can caused issues and is not recommended. MySQL users may encounter known issues. - **Performance**: Remote operations (extract/apply) are rate-limited to 10 requests/second using bottleneck. Processing time varies based on your instance size (collections, items, assets). - **Version Compatibility**: - v0.5.0+: Compatible with Directus 11 and up @@ -30,6 +32,7 @@ npx directus-template-cli@latest init ``` You'll be guided through: + - Selecting a directory for your new project - Choosing a Directus backend template - Selecting a frontend framework (if available for the template) @@ -51,7 +54,6 @@ npx directus-template-cli@latest init my-project --frontend=nextjs --template=cm npx directus-template-cli@latest init --template=https://github.com/directus-labs/starters/tree/main/cms ``` - Available flags: - `--frontend`: Frontend framework to use (e.g., nextjs, nuxt, astro) @@ -112,6 +114,7 @@ The `directus:template` property contains: - Each frontend has a `name` (display name) and `path` (directory containing the frontend code) When you use this template with the `init` command, it will: + 1. Copy the Directus template files from the specified template directory 2. Copy the selected frontend code based on your choice or the `--frontend` flag 3. Set up the project structure with both backend and frontend integrated @@ -133,12 +136,10 @@ npx directus-template-cli@latest apply You can choose from our community maintained templates or you can also choose a template from a local directory or a public GitHub repository. - ### Programmatic Mode By default, the CLI will run in interactive mode. For CI/CD pipelines or automated scripts, you can use the programmatic mode: - Using a token: ``` @@ -176,10 +177,10 @@ Available flags: - `--settings`: Load Settings - `--users`: Load Users - `--collections`: Only apply these comma-separated collections -- `--excludeCollections`: Exclude these comma-separated collections -- `--relationStrategy`: How to handle omitted relation targets. Options: `empty`, `ids`, `deep`. -- `--allowBrokenRelations`: Apply templates that intentionally preserve references to omitted records -- `--noAssets`: Shorthand for `--no-files` and excluding `directus_files` +- `--exclude-collections`: Exclude these comma-separated collections +- `--relation-strategy`: How to handle omitted relation targets. Options: `empty`, `preserve`, `deep`. +- `--allow-broken-relations`: Apply templates that intentionally preserve references to omitted records +- `--no-assets`: Shorthand for `--no-files` and excluding `directus_files` - `--disableTelemetry`: Disable telemetry collection When using `--partial`, you can also use `--no` flags to exclude specific components from being applied. For example: @@ -195,27 +196,12 @@ This command will apply the template but exclude content and users. Available `- - `--no-extensions`: Skip loading Extensions - `--no-files`: Skip loading Files - `--no-flows`: Skip loading Flows -- `--no-permissions`: Skip loading PermissionsI +- `--no-permissions`: Skip loading Permissions - `--no-schema`: Skip loading Schema - `--no-settings`: Skip loading Settings - `--no-users`: Skip loading Users - -#### Partial Templates and Relations - -Partial templates can intentionally omit components or collections. The CLI writes and reads `src/template-meta.json` so apply can understand what was extracted. - -Relation strategies: - -- `empty`: Relations to omitted collections are exported as `null` or `[]`. -- `ids`: Relation IDs are preserved, but omitted related records are not exported. Apply requires `--allowBrokenRelations` when metadata reports these references. -- `deep`: Related collections are added recursively where possible. Explicit exclusions still win. - -Skip assets safely: - -``` -npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local" --content --collections posts,pages --noAssets --relationStrategy empty -``` +For how partial templates handle schema, content, and relations, see [Partial Templates and Relation Strategies](#partial-templates-and-relation-strategies). #### Using Environment Variables @@ -228,7 +214,6 @@ You can also pass flags as environment variables. This can be useful for CI/CD p - `TEMPLATE_LOCATION`: Equivalent to `--templateLocation` - `TEMPLATE_TYPE`: Equivalent to `--templateType` - ### Existing Data You can apply a template to an existing Directus instance. This is nice because you can have smaller templates that you can "compose" for various use cases. The CLI tries to be smart about existing items in the target Directus instance. But mileage may vary depending on the size and complexity of the template and the existing instance. @@ -253,6 +238,8 @@ The CLI can also extract a template from a Directus instance so that it can be a Full extraction remains the default. Partial extraction is available with component flags, collection filters, and relation strategies. +For how partial templates handle schema, content, and relations, see [Partial Templates and Relation Strategies](#partial-templates-and-relation-strategies). + 1. Make sure you remove any sensitive data from the Directus instance you don't want to include in the template. 2. Login and create a Static Access Token for the admin user. 3. Copy the static token and your Directus URL. @@ -297,30 +284,32 @@ Available flags: - `--extensions` / `--no-extensions`: Include or skip extensions - `--users` / `--no-users`: Include or skip users - `--collections`: Only extract these comma-separated collections -- `--excludeCollections`: Exclude these comma-separated collections -- `--relationStrategy`: How to handle omitted relation targets. Options: `empty`, `ids`, `deep`. -- `--allowBrokenRelations`: Mark intentionally incomplete relation references as allowed in metadata -- `--noAssets`: Shorthand for `--no-files` and excluding `directus_files` +- `--exclude-collections`: Exclude these comma-separated collections +- `--relation-strategy`: How to handle omitted relation targets. Options: `empty`, `preserve`, `deep`. +- `--allow-broken-relations`: Mark intentionally incomplete relation references as allowed in metadata +- `--no-assets`: Shorthand for `--no-files` and excluding `directus_files` - `--disableTelemetry`: Disable telemetry collection Examples: +`--collections` limits content scope. Schema may still include additional collections needed by the data model. + Skip assets safely: ``` -npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --noAssets --relationStrategy empty +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --no-assets --relation-strategy empty ``` Preserve asset IDs but do not export assets: ``` -npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --noAssets --relationStrategy ids --allowBrokenRelations +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --no-assets --relation-strategy preserve --allow-broken-relations ``` Portable partial snapshot: ``` -npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --relationStrategy deep +npx directus-template-cli@latest extract -p --templateName="My Template" --templateLocation="./my-template" --directusToken="admin-token-here" --directusUrl="http://localhost:8055" --schema --content --collections posts,pages --relation-strategy deep ``` #### Using Environment Variables @@ -333,14 +322,90 @@ Similar to the Apply command, you can use environment variables for the Extract - `DIRECTUS_PASSWORD`: Equivalent to `--userPassword` - `TEMPLATE_LOCATION`: Equivalent to `--templateLocation` +## Partial Templates and Relation Strategies + +Partial templates can intentionally omit components or collections. The CLI writes and reads `src/template-meta.json` so apply can understand what was extracted. + +Partial templates have two scopes: + +- **Schema scope**: the data model needed to keep selected collections valid. This can include related, junction, translation, page-builder, and grouping collections even when no records are exported for them. +- **Content scope**: the records exported from collections requested with `--collections`, or records added by `deep`. + +Example: extracting `pages,posts` may still include schema for `page_blocks`, block collections, translations, and groups. Only `pages.json` and `posts.json` are exported unless you use `deep`. + +Relation strategies: + +| Strategy | What it exports | What happens to omitted relations | Best for | +| ---------- | -------------------------- | --------------------------------------------------------------------------- | ------------------------------------------- | +| `empty` | Selected content only | M2O fields become `null`; alias/O2M/M2M/M2A fields are omitted from records | Clean subsets, skipping assets safely | +| `preserve` | Selected content only | Keeps IDs/arrays as-is and writes warnings | Targets where related records already exist | +| `deep` | Selected + related content | Keeps relation values and recursively exports related records | Portable partial snapshots | + +Tradeoffs: + +- `empty` produces directly applyable templates, but applying to an existing instance can clear omitted M2O references. +- `preserve` can intentionally leave references to records that are not in the template. Apply requires `--allow-broken-relations` when metadata reports these references. +- `deep` is the most portable, but can expand into many collections and export much more data. + +For example, a source record might contain both a file relation and a page-builder alias field: + +```json +{ + "id": "post-1", + "image": "file-uuid", + "blocks": ["block-1", "block-2"] +} +``` + +With `--relation-strategy empty`, omitted relations are cleared or skipped: + +```json +{ + "id": "post-1", + "image": null +} +``` + +With `--relation-strategy preserve`, relation values stay in the record and metadata records warnings: + +```json +{ + "id": "post-1", + "image": "file-uuid", + "blocks": ["block-1", "block-2"] +} +``` + +With `--relation-strategy deep`, the record stays the same and related content collections are exported too: + +```jsonc +// content/posts.json +{ + "id": "post-1", + "image": "file-uuid", + "blocks": ["block-1", "block-2"], +} + +// also exported: +// content/page_blocks.json +// content/block_hero.json +``` + +Skip assets safely: + +``` +npx directus-template-cli@latest apply -p --directusUrl="http://localhost:8055" --directusToken="admin-token-here" --templateLocation="./my-template" --templateType="local" --content --collections posts,pages --no-assets --relation-strategy empty +``` + ## Logs The Directus Template CLI logs information to a file in the `.directus-template-cli/logs` directory. Logs are automatically generated for each run of the CLI. Here's how the logging system works: - - A new log file is created for each CLI run. - - Log files are stored in the `.directus-template-cli/logs` directory within your current working directory. - - Each log file is named `run-[timestamp].log`, where `[timestamp]` is the ISO timestamp of when the CLI was initiated. + +- A new log file is created for each CLI run. +- Log files are stored in the `.directus-template-cli/logs` directory within your current working directory. +- Each log file is named `run-[timestamp].log`, where `[timestamp]` is the ISO timestamp of when the CLI was initiated. The logger automatically sanitizes sensitive information such as passwords, tokens, and keys before writing to the log file. But it may not catch everything. Just be aware of this and make sure to remove the log files when they are no longer needed. diff --git a/src/commands/extract.ts b/src/commands/extract.ts index 2a897e90..ee3db825 100644 --- a/src/commands/extract.ts +++ b/src/commands/extract.ts @@ -46,7 +46,7 @@ export interface ExtractFlags { partial?: boolean permissions?: boolean programmatic: boolean - relationStrategy?: 'deep' | 'empty' | 'ids' + relationStrategy?: 'deep' | 'empty' | 'preserve' schema?: boolean settings?: boolean templateLocation: string diff --git a/src/lib/extract/expand-deep-plan.ts b/src/lib/extract/expand-deep-plan.ts index 681bd7e2..08e5f827 100644 --- a/src/lib/extract/expand-deep-plan.ts +++ b/src/lib/extract/expand-deep-plan.ts @@ -7,6 +7,10 @@ import catchError from '../utils/catch-error.js' interface RelationInfo { collection: string + meta?: { + one_allowed_collections?: null | string[] + one_field?: null | string + } related_collection?: null | string } @@ -28,19 +32,30 @@ export async function expandDeepPlan(plan: TemplatePlan): Promise } const selected = new Set(plan.collections.filter((collection) => available.has(collection))) - const relations = await api.client.request(readRelations()) as RelationInfo[] + const relations = (await api.client.request(readRelations())) as RelationInfo[] let changed = true while (changed) { changed = false + const candidates: string[] = [] for (const relation of relations) { - if (!relation.related_collection) continue - if (!selected.has(relation.collection)) continue - if (!available.has(relation.related_collection)) continue - if (selected.has(relation.related_collection)) continue + candidates.push( + ...([ + selected.has(relation.collection) && relation.related_collection, + relation.related_collection && selected.has(relation.related_collection) && relation.collection, + selected.has(relation.collection) && relation.meta?.one_allowed_collections, + ] + .flat() + .filter(Boolean) as string[]), + ) + } + + for (const collection of candidates) { + if (!available.has(collection)) continue + if (selected.has(collection)) continue - selected.add(relation.related_collection) + selected.add(collection) changed = true } } diff --git a/src/lib/extract/expand-schema-plan.ts b/src/lib/extract/expand-schema-plan.ts new file mode 100644 index 00000000..6b4dbf39 --- /dev/null +++ b/src/lib/extract/expand-schema-plan.ts @@ -0,0 +1,86 @@ +import {readCollections, readRelations} from '@directus/sdk' +import {ux} from '@oclif/core' + +import {api} from '../sdk.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' +import catchError from '../utils/catch-error.js' + +interface CollectionInfo { + collection: string + meta?: { + group?: null | string + } + schema?: Record | null +} + +interface RelationInfo { + collection: string + meta?: { + one_allowed_collections?: null | string[] + one_field?: null | string + } + related_collection?: null | string +} + +export async function expandSchemaPlan(plan: TemplatePlan): Promise { + if (!plan.partial || !plan.collections) return plan + + try { + const collections = (await api.client.request(readCollections())) as CollectionInfo[] + const availableCollections = collections + .filter((collection) => !collection.collection.startsWith('directus_', 0)) + .map((collection) => collection.collection) + .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) + const collectionMap = new Map(collections.map((collection) => [collection.collection, collection])) + + const available = new Set(availableCollections) + const selected = new Set(plan.collections.filter((collection) => available.has(collection))) + const relations = (await api.client.request(readRelations())) as RelationInfo[] + + let changed = true + while (changed) { + changed = false + + const candidates: string[] = [] + + for (const collection of selected) { + const group = collectionMap.get(collection)?.meta?.group + if (group) candidates.push(group) + } + + for (const relation of relations) { + candidates.push( + ...([ + selected.has(relation.collection) && relation.related_collection, + relation.related_collection && selected.has(relation.related_collection) && relation.collection, + selected.has(relation.collection) && relation.meta?.one_allowed_collections, + ] + .flat() + .filter(Boolean) as string[]), + ) + } + + for (const collection of candidates) { + if (!available.has(collection)) continue + if (selected.has(collection)) continue + + selected.add(collection) + changed = true + } + } + + const schemaCollections = [...selected] + const addedCollections = schemaCollections.filter((collection) => !plan.collections?.includes(collection)) + + if (addedCollections.length > 0) { + ux.warn(`Schema scope expanded collections: ${addedCollections.join(', ')}`) + } + + return { + ...plan, + schemaCollections, + } + } catch (error) { + catchError(error, {fatal: true}) + } +} diff --git a/src/lib/extract/extract-assets.ts b/src/lib/extract/extract-assets.ts index 5d5beb7d..2d37d834 100644 --- a/src/lib/extract/extract-assets.ts +++ b/src/lib/extract/extract-assets.ts @@ -46,9 +46,14 @@ export async function downloadAllFiles(dir: string) { while (true) { ux.action.status = `Downloading assets page ${page}` const fileList = await getAssetPage(page) - await Promise.all(fileList.map(file => downloadFile(file, dir).catch(error => { - catchError(`Error downloading ${file.filename_disk}: ${error.message}`) - }))) + + await Promise.all( + fileList.map((file) => + downloadFile(file, dir).catch((error) => { + catchError(`Error downloading ${file.filename_disk}: ${error.message}`) + }), + ), + ) if (fileList.length < PAGE_SIZE) break page++ diff --git a/src/lib/extract/extract-collections.ts b/src/lib/extract/extract-collections.ts index bb205665..563635fd 100644 --- a/src/lib/extract/extract-collections.ts +++ b/src/lib/extract/extract-collections.ts @@ -2,8 +2,8 @@ import {readCollections} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -16,8 +16,8 @@ export default async function extractCollections(dir: string, plan?: TemplatePla try { const response = await api.client.request(readCollections()) const collections = response - .filter(collection => !collection.collection.startsWith('directus_')) - .filter(collection => includesCollection(collection.collection, plan)) + .filter((collection) => !collection.collection.startsWith('directus_')) + .filter((collection) => includesSchemaCollection(collection.collection, plan)) await writeToFile('collections', collections, dir) } catch (error) { catchError(error) diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index d46cca7b..6d387616 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -13,16 +13,50 @@ const PAGE_SIZE = 500 interface RelationInfo { collection: string field: string + meta?: { + junction_field?: null | string + one_field?: null | string + } related_collection?: null | string } -async function getCollections(plan?: TemplatePlan) { +interface ExcludedRelationField { + field: string + relatedCollection: string + type: 'alias' | 'm2o' +} + +function getJunctionCollectionsWithBrokenFKs(relations: RelationInfo[], plan?: TemplatePlan): Set { + if (!plan?.partial) return new Set() + + // Directus sets meta.junction_field on both FK legs of an M2M to point to the other FK. + // Any collection appearing as `collection` on such a relation is a junction table. + const junctionCollections = new Set(relations.filter((r) => r.meta?.junction_field).map((r) => r.collection)) + + const broken = new Set() + for (const junction of junctionCollections) { + const targets = relations + .filter((r) => r.collection === junction && r.related_collection) + .map((r) => r.related_collection as string) + + // System collections always exist on every instance — never treat them as broken FK targets. + if (targets.some((target) => !target.startsWith('directus_') && !includesCollection(target, plan))) { + broken.add(junction) + } + } + + return broken +} + +async function getCollections(relations: RelationInfo[], plan?: TemplatePlan) { const response = await api.client.request(readCollections()) + const brokenJunctions = getJunctionCollectionsWithBrokenFKs(relations, plan) return response .filter((item) => !item.collection.startsWith('directus_', 0)) .filter((item) => item.schema !== null) .map((i) => i.collection) .filter((collection) => includesCollection(collection, plan)) + .filter((collection) => !brokenJunctions.has(collection)) } async function getCollectionItems(collection: string): Promise[]> { @@ -31,7 +65,10 @@ async function getCollectionItems(collection: string): Promise[] + const response = (await api.client.request(readItems(collection as never, {limit: PAGE_SIZE, page}))) as Record< + string, + unknown + >[] items.push(...response) if (response.length < PAGE_SIZE) break @@ -41,14 +78,34 @@ async function getCollectionItems(collection: string): Promise - relation.collection === collection && - relation.related_collection && - !includesCollection(relation.related_collection, plan), - ) + const m2oFields = relations + .filter((relation) => relation.collection === collection) + .filter((relation): relation is RelationInfo & {related_collection: string} => Boolean(relation.related_collection)) + .filter((relation) => !includesCollection(relation.related_collection, plan)) + .map((relation) => ({ + field: relation.field, + relatedCollection: relation.related_collection, + type: 'm2o' as const, + })) + + const aliasFields = relations + .filter((relation) => relation.related_collection === collection) + .filter((relation): relation is RelationInfo & {meta: {one_field: string}} => Boolean(relation.meta?.one_field)) + .filter((relation) => !includesCollection(relation.collection, plan)) + .map((relation) => ({ + field: relation.meta.one_field, + relatedCollection: relation.collection, + type: 'alias' as const, + })) + + return [...m2oFields, ...aliasFields] } function hasValue(value: unknown): boolean { @@ -56,11 +113,16 @@ function hasValue(value: unknown): boolean { return value !== null && value !== undefined } -function emptyExcludedRelations(items: Record[], relations: RelationInfo[]): void { +function emptyExcludedRelations(items: Record[], relations: ExcludedRelationField[]): void { for (const item of items) { for (const relation of relations) { if (!(relation.field in item)) continue - item[relation.field] = Array.isArray(item[relation.field]) ? [] : null + + if (relation.type === 'alias') { + delete item[relation.field] + } else { + item[relation.field] = null + } } } } @@ -68,17 +130,18 @@ function emptyExcludedRelations(items: Record[], relations: Rel function getBrokenRelationWarnings( collection: string, items: Record[], - relations: RelationInfo[], + relations: ExcludedRelationField[], ): TemplateWarning[] { return relations - .filter((relation): relation is RelationInfo & {related_collection: string} => Boolean(relation.related_collection)) - .map((relation): TemplateWarning => ({ - collection, - count: items.filter((item) => hasValue(item[relation.field])).length, - field: relation.field, - relatedCollection: relation.related_collection, - type: 'excluded_relation', - })) + .map( + (relation): TemplateWarning => ({ + collection, + count: items.filter((item) => hasValue(item[relation.field])).length, + field: relation.field, + relatedCollection: relation.relatedCollection, + type: 'excluded_relation', + }), + ) .filter((warning) => warning.count > 0) } @@ -97,9 +160,8 @@ async function getDataFromCollection( emptyExcludedRelations(response, excludedRelations) } - const warnings = plan?.relationStrategy === 'ids' - ? getBrokenRelationWarnings(collection, response, excludedRelations) - : [] + const warnings = + plan?.relationStrategy === 'preserve' ? getBrokenRelationWarnings(collection, response, excludedRelations) : [] await writeToFile(`${collection}`, response, `${dir}/content/`) return warnings @@ -117,8 +179,8 @@ export async function extractContent(dir: string, plan?: TemplatePlan): Promise< const warnings: TemplateWarning[] = [] try { - const collections = await getCollections(plan) - const relations = await api.client.request(readRelations()) as RelationInfo[] + const relations = (await api.client.request(readRelations())) as RelationInfo[] + const collections = await getCollections(relations, plan) for (const collection of collections) { // eslint-disable-next-line no-await-in-loop diff --git a/src/lib/extract/extract-fields.ts b/src/lib/extract/extract-fields.ts index 661ff36b..188b29fb 100644 --- a/src/lib/extract/extract-fields.ts +++ b/src/lib/extract/extract-fields.ts @@ -2,8 +2,8 @@ import {readFields} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -21,18 +21,15 @@ export default async function extractFields(dir: string, plan?: TemplatePlan) { } const fields = response - .filter( - // @ts-ignore - (i: { collection: string; meta?: { system?: boolean } }) => i.meta && !i.meta.system, - ) - .filter((i: { collection: string }) => includesCollection(i.collection, plan)) - .map(i => { - if (i.meta) { - delete i.meta.id - } - - return i - }) + .filter((i: {collection: string; meta?: {system?: boolean}}) => i.meta && !i.meta.system) + .filter((i: {collection: string}) => includesSchemaCollection(i.collection, plan)) + .map((i) => { + if (i.meta) { + delete i.meta.id + } + + return i + }) await writeToFile('fields', fields, dir) } catch (error) { diff --git a/src/lib/extract/extract-relations.ts b/src/lib/extract/extract-relations.ts index da636a12..fa4646c8 100644 --- a/src/lib/extract/extract-relations.ts +++ b/src/lib/extract/extract-relations.ts @@ -19,26 +19,23 @@ export default async function extractRelations(dir: string, plan?: TemplatePlan) // Fetching fields to filter out system fields while retaining custom fields on system collections const fields = await api.client.request(readFields()) - const customFields = fields.filter( - (i: any) => !i.meta?.system, - ) + const customFields = fields.filter((i: any) => !i.meta?.system) const relations = response - // Filter out relations where the collection starts with 'directus_' && the field is not within the customFields array - .filter( - (i: any) => - !i.collection.startsWith('directus_', 0) - || customFields.some( - (f: { collection: string; field: string }) => - f.collection === i.collection && f.field === i.field, - ), - ) - .filter((i: any) => includesRelation(i.collection, i.related_collection, plan)) - .map(i => { - delete i.meta.id - return i - }) + // Filter out relations where the collection starts with 'directus_' && the field is not within the customFields array + .filter( + (i: any) => + !i.collection.startsWith('directus_', 0) || + customFields.some( + (f: {collection: string; field: string}) => f.collection === i.collection && f.field === i.field, + ), + ) + .filter((i: any) => includesRelation(i.collection, i.related_collection, plan)) + .map((i) => { + delete i.meta.id + return i + }) await writeToFile('relations', relations, dir) } catch (error) { diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index 94b7ff33..cf569476 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -1,9 +1,15 @@ import {ux} from '@oclif/core' import fs from 'node:fs' -import {buildTemplatePlan, type TemplatePlan, type TemplateWarning, writeTemplateMetadata} from '../template-plan/index.js' +import { + buildTemplatePlan, + type TemplatePlan, + type TemplateWarning, + writeTemplateMetadata, +} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import {expandDeepPlan} from './expand-deep-plan.js' +import {expandSchemaPlan} from './expand-schema-plan.js' import extractAccess from './extract-access.js' import {downloadAllFiles} from './extract-assets.js' import extractCollections from './extract-collections.js' @@ -26,7 +32,8 @@ import extractUsers from './extract-users.js' export default async function extract(dir: string, plan: TemplatePlan = buildTemplatePlan()) { const destination = `${dir}/src` - const effectivePlan = await expandDeepPlan(plan) + const schemaPlan = await expandSchemaPlan(plan) + const effectivePlan = await expandDeepPlan(schemaPlan) if (!fs.existsSync(destination)) { ux.stdout(`Attempting to create directory at: ${destination}`) @@ -90,7 +97,9 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem } for (const warning of warnings) { - ux.warn(`Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`) + ux.warn( + `Excluded relation: ${warning.collection}.${warning.field} -> ${warning.relatedCollection} (${warning.count} records)`, + ) } try { diff --git a/src/lib/load/apply-flags.ts b/src/lib/load/apply-flags.ts index 7f010222..9e1b5e75 100644 --- a/src/lib/load/apply-flags.ts +++ b/src/lib/load/apply-flags.ts @@ -17,7 +17,7 @@ export interface ApplyFlags { partial: boolean permissions: boolean programmatic: boolean - relationStrategy?: 'deep' | 'empty' | 'ids' + relationStrategy?: 'deep' | 'empty' | 'preserve' schema: boolean settings: boolean templateLocation: string diff --git a/src/lib/load/finalize-collections.ts b/src/lib/load/finalize-collections.ts new file mode 100644 index 00000000..59c1bc10 --- /dev/null +++ b/src/lib/load/finalize-collections.ts @@ -0,0 +1,34 @@ +import {updateCollection} from '@directus/sdk' +import {ux} from '@oclif/core' + +import {DIRECTUS_PINK} from '../constants.js' +import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' +import catchError from '../utils/catch-error.js' +import readFile from '../utils/read-file.js' + +export default async function finalizeCollections(dir: string, plan?: TemplatePlan) { + const collections = readFile('collections', dir) + .filter((collection: any) => includesSchemaCollection(collection.collection, plan)) + .filter((collection: any) => !collection.collection.startsWith('directus_')) + + ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${collections.length} collections`)) + + const collectionNames = new Set(collections.map((collection: any) => collection.collection)) + + for await (const collection of collections) { + const meta = {...collection.meta} + if (meta.group && !collectionNames.has(meta.group)) { + ux.warn(`Skipping missing group "${meta.group}" for collection "${collection.collection}"`) + delete meta.group + } + + try { + await api.client.request(updateCollection(collection.collection, {meta})) + } catch (error) { + catchError(error, {context: {collection: collection.collection, group: meta.group}}) + } + } + + ux.action.stop() +} diff --git a/src/lib/load/finalize-fields.ts b/src/lib/load/finalize-fields.ts new file mode 100644 index 00000000..ae7b37f3 --- /dev/null +++ b/src/lib/load/finalize-fields.ts @@ -0,0 +1,29 @@ +import {updateField} from '@directus/sdk' +import {ux} from '@oclif/core' + +import {DIRECTUS_PINK} from '../constants.js' +import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' +import catchError from '../utils/catch-error.js' +import readFile from '../utils/read-file.js' + +export default async function finalizeFields(dir: string, plan?: TemplatePlan) { + const fields = readFile('fields', dir).filter((field: any) => includesSchemaCollection(field.collection, plan)) + + ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${fields.length} fields`)) + + for await (const field of fields) { + try { + await api.client.request( + updateField(field.collection, field.field, { + meta: field.meta ? {...field.meta} : undefined, + schema: field.schema ? {...field.schema} : undefined, + }), + ) + } catch (error) { + catchError(error) + } + } + + ux.action.stop() +} diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index 4061c9c1..b525bd99 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -5,6 +5,8 @@ import type {ApplyFlags} from './apply-flags.js' import {applyMetadataToPlan, buildTemplatePlan, readTemplateMetadata} from '../template-plan/index.js' import checkTemplate from '../utils/check-template.js' import loadAccess from './load-access.js' +import finalizeCollections from './finalize-collections.js' +import finalizeFields from './finalize-fields.js' import loadCollections from './load-collections.js' import loadDashboards from './load-dashboards.js' import loadData from './load-data.js' @@ -20,7 +22,6 @@ import loadRoles from './load-roles.js' import loadSettings from './load-settings.js' import loadTranslations from './load-translations.js' import loadUsers from './load-users.js' -import updateRequiredFields from './update-required-fields.js' export default async function apply(dir: string, flags: ApplyFlags) { const source = `${dir}/src` @@ -35,9 +36,11 @@ export default async function apply(dir: string, flags: ApplyFlags) { ux.warn('Template metadata indicates this is a partial template.') } - const brokenRelationWarnings = metadata?.warnings?.filter(warning => warning.type === 'excluded_relation') || [] + const brokenRelationWarnings = metadata?.warnings?.filter((warning) => warning.type === 'excluded_relation') || [] if (components.content && brokenRelationWarnings.length > 0 && !effectivePlan.allowBrokenRelations) { - ux.error('This partial template contains excluded relation references. Re-run with --allow-broken-relations to apply anyway.') + ux.error( + 'This partial template contains excluded relation references. Re-run with --allow-broken-relations to apply anyway.', + ) } if (!metadata || components.schema) { @@ -52,6 +55,8 @@ export default async function apply(dir: string, flags: ApplyFlags) { if (components.schema) { await loadCollections(source, effectivePlan) await loadRelations(source, effectivePlan) + await finalizeCollections(source, effectivePlan) + await finalizeFields(source, effectivePlan) } if (components.permissions || components.users) { @@ -75,10 +80,6 @@ export default async function apply(dir: string, flags: ApplyFlags) { await loadData(source, effectivePlan) } - if (components.schema) { - await updateRequiredFields(source, effectivePlan) - } - if (components.dashboards) { await loadDashboards(source) } diff --git a/src/lib/load/load-collections.ts b/src/lib/load/load-collections.ts index 852581c9..91f259a5 100644 --- a/src/lib/load/load-collections.ts +++ b/src/lib/load/load-collections.ts @@ -1,13 +1,11 @@ -import type {Collection, CollectionMeta, Field} from '@directus/types' +import type {Collection, Field} from '@directus/types' -import { - createCollection, createField, readCollections, readFields, updateCollection, -} from '@directus/sdk' +import {createCollection, createField, readCollections, readFields} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' @@ -17,15 +15,16 @@ import readFile from '../utils/read-file.js' * @returns {Promise} - Returns nothing */ export default async function loadCollections(dir: string, plan?: TemplatePlan) { - const collectionsToAdd = readFile('collections', dir) - .filter(collection => includesCollection(collection.collection, plan)) - const fieldsToAdd = readFile('fields', dir) - .filter(field => includesCollection(field.collection, plan)) + const collectionsToAdd = readFile('collections', dir).filter((collection) => + includesSchemaCollection(collection.collection, plan), + ) + const fieldsToAdd = readFile('fields', dir).filter((field) => includesSchemaCollection(field.collection, plan)) - ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading ${collectionsToAdd.length} collections and ${fieldsToAdd.length} fields`)) + ux.action.start( + ux.colorize(DIRECTUS_PINK, `Loading ${collectionsToAdd.length} collections and ${fieldsToAdd.length} fields`), + ) await processCollections(collectionsToAdd, fieldsToAdd) - await updateCollections(collectionsToAdd) await addCustomFieldsOnSystemCollections(fieldsToAdd) ux.action.stop() @@ -39,25 +38,25 @@ async function processCollections(collectionsToAdd: any[], fieldsToAdd: any[]) { try { const existingCollection = existingCollections.find((c: any) => c.collection === collection.collection) - await (existingCollection ? addNewFieldsToExistingCollection(collection.collection, fieldsToAdd, existingFields) : addNewCollectionWithFields(collection, fieldsToAdd)) + await (existingCollection + ? addNewFieldsToExistingCollection(collection.collection, fieldsToAdd, existingFields) + : addNewCollectionWithFields(collection, fieldsToAdd)) } catch (error) { catchError(error) } } } -const removeRequiredorIsNullable = (field:Field) => { +const removeRequiredorIsNullable = (field: Field) => { if (field.meta?.required === true) { field.meta.required = false } if (field.schema?.is_nullable === false) { - field.schema.is_nullable = true } if (field.schema?.is_unique === true) { - field.schema.is_unique = false } @@ -65,8 +64,9 @@ const removeRequiredorIsNullable = (field:Field) => { } async function addNewCollectionWithFields(collection: any, allFields: Field[]) { - const collectionFields = allFields.filter(field => field.collection === collection.collection) - .map(field => removeRequiredorIsNullable(field)) + const collectionFields = allFields + .filter((field) => field.collection === collection.collection) + .map((field) => removeRequiredorIsNullable(field)) const collectionWithoutGroup = { ...collection, fields: collectionFields, @@ -77,8 +77,9 @@ async function addNewCollectionWithFields(collection: any, allFields: Field[]) { } async function addNewFieldsToExistingCollection(collectionName: string, fieldsToAdd: Field[], existingFields: any[]) { - const collectionFieldsToAdd = fieldsToAdd.filter(field => field.collection === collectionName) - .map(field => removeRequiredorIsNullable(field)) + const collectionFieldsToAdd = fieldsToAdd + .filter((field) => field.collection === collectionName) + .map((field) => removeRequiredorIsNullable(field)) const existingCollectionFields = existingFields.filter((field: any) => field.collection === collectionName) @@ -94,34 +95,15 @@ async function addNewFieldsToExistingCollection(collectionName: string, fieldsTo } } -async function updateCollections(collections: any[]) { - for await (const collection of collections) { - try { - if (collection.meta.group) { - const pl = { - meta: { - group: collection.meta.group, - }, - } - await api.client.request(updateCollection(collection.collection, pl)) - } - } catch (error) { - catchError(error) - } - } -} - async function addCustomFieldsOnSystemCollections(fields: any[]) { - const customFields = fields.filter( - (field: any) => field.collection.startsWith('directus_'), - ) + const customFields = fields.filter((field: any) => field.collection.startsWith('directus_')) const existingFields = await api.client.request(readFields()) for await (const field of customFields) { try { - const fieldExists = existingFields.some((existingField: any) => - existingField.collection === field.collection && existingField.field === field.field, + const fieldExists = existingFields.some( + (existingField: any) => existingField.collection === field.collection && existingField.field === field.field, ) if (!fieldExists) { diff --git a/src/lib/load/load-data.ts b/src/lib/load/load-data.ts index 7b794f88..3a1870ac 100644 --- a/src/lib/load/load-data.ts +++ b/src/lib/load/load-data.ts @@ -1,17 +1,18 @@ import {createItems, readItems, updateItemsBatch, updateSingleton} from '@directus/sdk' import {ux} from '@oclif/core' +import fs from 'node:fs' import path from 'pathe' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import {chunkArray} from '../utils/chunk-array.js' import readFile from '../utils/read-file.js' const BATCH_SIZE = 50 -export default async function loadData(dir:string, plan?: TemplatePlan) { +export default async function loadData(dir: string, plan?: TemplatePlan) { const collections = getUserCollections(dir, plan) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading data for ${collections.length} collections`)) @@ -22,44 +23,91 @@ export default async function loadData(dir:string, plan?: TemplatePlan) { ux.action.stop() } +function getContentCollections(dir: string): Set { + const contentDir = path.resolve(dir, 'content') + if (!fs.existsSync(contentDir)) return new Set() + + return new Set( + fs + .readdirSync(contentDir) + .filter((file) => file.endsWith('.json')) + .map((file) => path.basename(file, '.json')), + ) +} + +function getBrokenJunctionCollections(dir: string, plan?: TemplatePlan): Set { + if (!plan?.partial) return new Set() + + const relations: Array<{ + collection: string + meta?: {junction_field?: null | string} + related_collection?: null | string + }> = readFile('relations', dir) + + const junctionCollections = new Set(relations.filter((r) => r.meta?.junction_field).map((r) => r.collection)) + + const broken = new Set() + for (const junction of junctionCollections) { + const targets = relations + .filter((r) => r.collection === junction && r.related_collection) + .map((r) => r.related_collection as string) + + if (targets.some((target) => !target.startsWith('directus_') && !includesCollection(target, plan))) { + broken.add(junction) + } + } + + return broken +} + function getUserCollections(dir: string, plan?: TemplatePlan) { + const contentCollections = getContentCollections(dir) const collections = readFile('collections', dir) + const brokenJunctions = getBrokenJunctionCollections(dir, plan) + + if (brokenJunctions.size > 0) { + ux.warn(`Skipping junction collections with excluded FK targets: ${[...brokenJunctions].join(', ')}`) + } + return collections - .filter(item => !item.collection.startsWith('directus_', 0)) - .filter(item => item.schema !== null) - .filter(item => includesCollection(item.collection, plan)) + .filter((item) => contentCollections.has(item.collection)) + .filter((item) => !item.collection.startsWith('directus_', 0)) + .filter((item) => item.schema !== null) + .filter((item) => includesCollection(item.collection, plan)) + .filter((item) => !brokenJunctions.has(item.collection)) } async function loadSkeletonRecords(dir: string, plan?: TemplatePlan) { ux.action.status = 'Loading skeleton records' const primaryKeyMap = await getCollectionPrimaryKeys(dir) - const userCollections = getUserCollections(dir, plan) - .filter(item => !item.meta.singleton) + const userCollections = getUserCollections(dir, plan).filter((item) => !item.meta.singleton) - await Promise.all(userCollections.map(async collection => { - const name = collection.collection - const primaryKeyField = getPrimaryKey(primaryKeyMap, name) - const sourceDir = path.resolve(dir, 'content') - const data = readFile(name, sourceDir) + await Promise.all( + userCollections.map(async (collection) => { + const name = collection.collection + const primaryKeyField = getPrimaryKey(primaryKeyMap, name) + const sourceDir = path.resolve(dir, 'content') + const data = readFile(name, sourceDir) - // Fetch existing primary keys - const existingPrimaryKeys = await getExistingPrimaryKeys(name, primaryKeyField) + // Fetch existing primary keys + const existingPrimaryKeys = await getExistingPrimaryKeys(name, primaryKeyField) - // Filter out existing records - const newData = data.filter(entry => !existingPrimaryKeys.has(entry[primaryKeyField])) + // Filter out existing records + const newData = data.filter((entry) => !existingPrimaryKeys.has(entry[primaryKeyField])) - if (newData.length === 0) { - // ux.stdout(`${ux.colorize('dim', '--')} Skipping ${name}: No new records to add`) - return - } + if (newData.length === 0) { + // ux.stdout(`${ux.colorize('dim', '--')} Skipping ${name}: No new records to add`) + return + } - const batches = chunkArray(newData, BATCH_SIZE).map(batch => - batch.map(entry => ({[primaryKeyField]: entry[primaryKeyField]})), - ) + const batches = chunkArray(newData, BATCH_SIZE).map((batch) => + batch.map((entry) => ({[primaryKeyField]: entry[primaryKeyField]})), + ) - await Promise.all(batches.map(batch => uploadBatch(name, batch, createItems))) - // ux.stdout(`${ux.colorize('dim', '--')} Added ${newData.length} new skeleton records to ${name}`) - })) + await Promise.all(batches.map((batch) => uploadBatch(name, batch, createItems))) + // ux.stdout(`${ux.colorize('dim', '--')} Added ${newData.length} new skeleton records to ${name}`) + }), + ) ux.action.status = 'Loaded skeleton records' } @@ -72,11 +120,13 @@ async function getExistingPrimaryKeys(collection: string, primaryKeyField: strin while (true) { try { // @ts-ignore - const response = await api.client.request(readItems(collection, { - fields: [primaryKeyField], - limit, - page, - })) + const response = await api.client.request( + readItems(collection, { + fields: [primaryKeyField], + limit, + page, + }), + ) if (response.length === 0) break @@ -101,43 +151,45 @@ async function uploadBatch(collection: string, batch: any[], method: Function) { } } -async function loadFullData(dir:string, plan?: TemplatePlan) { +async function loadFullData(dir: string, plan?: TemplatePlan) { ux.action.status = 'Updating records with full data' - const userCollections = getUserCollections(dir, plan) - .filter(item => !item.meta.singleton) + const userCollections = getUserCollections(dir, plan).filter((item) => !item.meta.singleton) - await Promise.all(userCollections.map(async collection => { - const name = collection.collection - const sourceDir = path.resolve(dir, 'content') - const data = readFile(name, sourceDir) + await Promise.all( + userCollections.map(async (collection) => { + const name = collection.collection + const sourceDir = path.resolve(dir, 'content') + const data = readFile(name, sourceDir) - const batches = chunkArray(data, BATCH_SIZE).map(batch => - batch.map(({user_created, user_updated, ...cleanedRow}) => cleanedRow), - ) + const batches = chunkArray(data, BATCH_SIZE).map((batch) => + batch.map(({user_created, user_updated, ...cleanedRow}) => cleanedRow), + ) - await Promise.all(batches.map(batch => uploadBatch(name, batch, updateItemsBatch))) - })) + await Promise.all(batches.map((batch) => uploadBatch(name, batch, updateItemsBatch))) + }), + ) ux.action.status = 'Updated records with full data' } -async function loadSingletons(dir:string, plan?: TemplatePlan) { +async function loadSingletons(dir: string, plan?: TemplatePlan) { ux.action.status = 'Loading data for singleton collections' - const singletonCollections = getUserCollections(dir, plan) - .filter(item => item.meta.singleton) - - await Promise.all(singletonCollections.map(async collection => { - const name = collection.collection - const sourceDir = path.resolve(dir, 'content') - const data = readFile(name, sourceDir) - try { - const {user_created, user_updated, ...cleanedData} = data as any - - await api.client.request(updateSingleton(name, cleanedData)) - } catch (error) { - catchError(error) - } - })) + const singletonCollections = getUserCollections(dir, plan).filter((item) => item.meta.singleton) + + await Promise.all( + singletonCollections.map(async (collection) => { + const name = collection.collection + const sourceDir = path.resolve(dir, 'content') + const data = readFile(name, sourceDir) + try { + const {user_created, user_updated, ...cleanedData} = data as any + + await api.client.request(updateSingleton(name, cleanedData)) + } catch (error) { + catchError(error) + } + }), + ) ux.action.status = 'Loaded data for singleton collections' } diff --git a/src/lib/load/load-relations.ts b/src/lib/load/load-relations.ts index 79a6da3d..8b95ad08 100644 --- a/src/lib/load/load-relations.ts +++ b/src/lib/load/load-relations.ts @@ -13,29 +13,32 @@ import readFile from '../utils/read-file.js' * @returns {Promise} - Returns nothing */ export default async function loadRelations(dir: string, plan?: TemplatePlan) { - const relations = readFile('relations', dir) - .filter(relation => includesRelation(relation.collection, relation.related_collection, plan)) + const relations = readFile('relations', dir).filter((relation) => + includesRelation(relation.collection, relation.related_collection, plan), + ) ux.action.start(ux.colorize(DIRECTUS_PINK, `Loading ${relations.length} relations`)) if (relations && relations.length > 0) { // Fetch existing relations const existingRelations = await api.client.request(readRelations()) - const existingRelationKeys = new Set(existingRelations.map(relation => - `${relation.collection}:${relation.field}:${relation.related_collection}`, - )) - - const relationsToAdd = relations.filter(relation => { - const key = `${relation.collection}:${relation.field}:${relation.related_collection}` - if (existingRelationKeys.has(key)) { - return false - } - - return true - }).map(relation => { - const cleanRelation = {...relation} - cleanRelation.meta.id = undefined - return cleanRelation - }) + const existingRelationKeys = new Set( + existingRelations.map((relation) => `${relation.collection}:${relation.field}:${relation.related_collection}`), + ) + + const relationsToAdd = relations + .filter((relation) => { + const key = `${relation.collection}:${relation.field}:${relation.related_collection}` + if (existingRelationKeys.has(key)) { + return false + } + + return true + }) + .map((relation) => { + const cleanRelation = {...relation} + cleanRelation.meta.id = undefined + return cleanRelation + }) await addRelations(relationsToAdd) } diff --git a/src/lib/load/update-required-fields.ts b/src/lib/load/update-required-fields.ts index 4e46ab9d..59ceda20 100644 --- a/src/lib/load/update-required-fields.ts +++ b/src/lib/load/update-required-fields.ts @@ -1,23 +1,27 @@ - import {updateField} from '@directus/sdk' import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' import {api} from '../sdk.js' +import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' export default async function updateRequiredFields(dir: string, plan?: TemplatePlan) { const fieldsToUpdate = readFile('fields', dir) - .filter(field => includesCollection(field.collection, plan)) - .filter(field => field.meta.required === true || field.schema?.is_nullable === false || field.schema?.is_unique === true) + .filter((field) => includesSchemaCollection(field.collection, plan)) + .filter( + (field) => + field.meta.required === true || field.schema?.is_nullable === false || field.schema?.is_unique === true, + ) ux.action.start(ux.colorize(DIRECTUS_PINK, `Updating ${fieldsToUpdate.length} fields to required`)) for await (const field of fieldsToUpdate) { try { - await api.client.request(updateField(field.collection, field.field, {meta: {...field.meta}, schema: {...field.schema}})) + await api.client.request( + updateField(field.collection, field.field, {meta: {...field.meta}, schema: {...field.schema}}), + ) } catch (error) { catchError(error) } diff --git a/src/lib/template-plan/collections.ts b/src/lib/template-plan/collections.ts index c9d63ad8..a4c6ebe2 100644 --- a/src/lib/template-plan/collections.ts +++ b/src/lib/template-plan/collections.ts @@ -6,9 +6,17 @@ export function includesCollection(collection: string, plan?: TemplatePlan): boo return true } +export function includesSchemaCollection(collection: string, plan?: TemplatePlan): boolean { + const schemaCollections = plan?.schemaCollections || plan?.collections + if (schemaCollections && !schemaCollections.includes(collection)) return false + if (plan?.excludeCollections?.includes(collection)) return false + return true +} + export function includesRelation(collection: string, relatedCollection?: null | string, plan?: TemplatePlan): boolean { - if (!includesCollection(collection, plan)) return false + if (!includesSchemaCollection(collection, plan)) return false if (!relatedCollection) return true - if (plan?.relationStrategy === 'ids') return true - return includesCollection(relatedCollection, plan) + if (plan?.excludeCollections?.includes(relatedCollection)) return plan.relationStrategy === 'preserve' + if (relatedCollection.startsWith('directus_')) return true + return includesSchemaCollection(relatedCollection, plan) } diff --git a/src/lib/template-plan/flags.ts b/src/lib/template-plan/flags.ts index 6c61c96d..46bed725 100644 --- a/src/lib/template-plan/flags.ts +++ b/src/lib/template-plan/flags.ts @@ -46,20 +46,24 @@ export const collections = Flags.string({ }) export const excludeCollections = Flags.string({ + aliases: ['exclude-collections'], description: 'Exclude these comma-separated collections', }) export const relationStrategy = Flags.string({ + aliases: ['relation-strategy'], description: 'How to handle relations to omitted data', - options: ['empty', 'ids', 'deep'], + options: ['empty', 'preserve', 'deep'], }) export const allowBrokenRelations = Flags.boolean({ + aliases: ['allow-broken-relations'], default: false, description: 'Allow intentionally incomplete relation references', }) export const noAssets = Flags.boolean({ + aliases: ['no-assets'], default: undefined, description: 'Shorthand for --no-files and --exclude-collections directus_files', }) diff --git a/src/lib/template-plan/index.ts b/src/lib/template-plan/index.ts index 136a5e2f..b5912951 100644 --- a/src/lib/template-plan/index.ts +++ b/src/lib/template-plan/index.ts @@ -1,8 +1,17 @@ -import type {RelationStrategy, TemplateComponents, TemplatePlan} from './types.js' +import type {RelationStrategy, TemplateComponent, TemplateComponents, TemplatePlan} from './types.js' import catchError from '../utils/catch-error.js' import {componentNames} from './flags.js' +type BuildFlags = Partial> & { + allowBrokenRelations?: boolean + collections?: string | string[] + excludeCollections?: string | string[] + noAssets?: boolean + partial?: boolean + relationStrategy?: string +} + function parseList(value?: string | string[]): string[] | undefined { if (!value) return undefined const raw = Array.isArray(value) ? value.join(',') : value @@ -13,8 +22,7 @@ function parseList(value?: string | string[]): string[] | undefined { return values.length > 0 ? values : undefined } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function hasPartialOnlyFlags(flags: any): boolean { +function hasPartialOnlyFlags(flags: BuildFlags): boolean { return Boolean( flags.collections || flags.excludeCollections || @@ -24,13 +32,15 @@ function hasPartialOnlyFlags(flags: any): boolean { ) } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function hasComponentFlags(flags: any): boolean { +function hasScopingFlags(flags: BuildFlags): boolean { + return Boolean(flags.collections || flags.excludeCollections || flags.noAssets === true || hasComponentFlags(flags)) +} + +function hasComponentFlags(flags: BuildFlags): boolean { return componentNames.some((component) => flags[component] !== undefined) } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function buildComponents(flags: any, partial: boolean): TemplateComponents { +function buildComponents(flags: BuildFlags, partial: boolean): TemplateComponents { const components = {} as TemplateComponents const enabled = componentNames.filter((component) => flags[component] === true) const disabled = componentNames.filter((component) => flags[component] === false) @@ -53,8 +63,7 @@ function buildComponents(flags: any, partial: boolean): TemplateComponents { return components } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function buildTemplatePlan(flags: any = {}): TemplatePlan { +export function buildTemplatePlan(flags: BuildFlags = {}): TemplatePlan { const collections = parseList(flags.collections) const excludeCollections = parseList(flags.excludeCollections) || (flags.noAssets ? [] : undefined) if (flags.noAssets && !excludeCollections?.includes('directus_files')) { @@ -74,7 +83,8 @@ export function buildTemplatePlan(flags: any = {}): TemplatePlan { components, excludeCollections, partial, - relationStrategy: (flags.relationStrategy || (partial ? 'ids' : 'deep')) as RelationStrategy, + relationStrategy: (flags.relationStrategy || + (partial ? (hasScopingFlags(flags) ? 'empty' : 'preserve') : 'deep')) as RelationStrategy, } } diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts index 8a245086..df6de550 100644 --- a/src/lib/template-plan/metadata-plan.ts +++ b/src/lib/template-plan/metadata-plan.ts @@ -4,7 +4,7 @@ import {componentNames} from './flags.js' function intersectCollections(requested?: string[], available?: string[]): string[] | undefined { if (requested && available) { - const collections = requested.filter(collection => available.includes(collection)) + const collections = requested.filter((collection) => available.includes(collection)) return collections.length > 0 ? collections : undefined } @@ -24,11 +24,13 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad components[component] = components[component] && metadata.components[component] } - const partial = metadata.partial || componentNames.some(component => components[component] !== plan.components[component]) + const partial = + metadata.partial || componentNames.some((component) => components[component] !== plan.components[component]) return { ...plan, collections: intersectCollections(plan.collections, metadata.collections), + schemaCollections: intersectCollections(plan.schemaCollections, metadata.schemaCollections), components, excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), partial, diff --git a/src/lib/template-plan/metadata.ts b/src/lib/template-plan/metadata.ts index c227de9f..e9e97af8 100644 --- a/src/lib/template-plan/metadata.ts +++ b/src/lib/template-plan/metadata.ts @@ -11,6 +11,7 @@ export function createTemplateMetadata(plan: TemplatePlan, warnings: TemplateWar return { allowBrokenRelations: plan.allowBrokenRelations, collections: plan.collections, + schemaCollections: plan.schemaCollections, components: plan.components, excludedCollections: plan.excludeCollections, partial: plan.partial, diff --git a/src/lib/template-plan/types.ts b/src/lib/template-plan/types.ts index b2a1d866..67c6e018 100644 --- a/src/lib/template-plan/types.ts +++ b/src/lib/template-plan/types.ts @@ -1,4 +1,4 @@ -export type RelationStrategy = 'deep' | 'empty' | 'ids' +export type RelationStrategy = 'deep' | 'empty' | 'preserve' export interface TemplateComponents { content: boolean @@ -19,6 +19,7 @@ export interface TemplatePlan { excludeCollections?: string[] partial: boolean relationStrategy: RelationStrategy + schemaCollections?: string[] } export type TemplateWarning = { diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index fdc8c92d..2128ed49 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -1,6 +1,11 @@ import {expect} from 'chai' -import {applyMetadataToPlan, buildTemplatePlan, createTemplateMetadata, includesRelation} from '../src/lib/template-plan/index.js' +import { + applyMetadataToPlan, + buildTemplatePlan, + createTemplateMetadata, + includesRelation, +} from '../src/lib/template-plan/index.js' describe('template plan', () => { it('defaults to full template', () => { @@ -15,7 +20,7 @@ describe('template plan', () => { const plan = buildTemplatePlan({partial: true}) expect(plan.partial).to.equal(true) - expect(plan.relationStrategy).to.equal('ids') + expect(plan.relationStrategy).to.equal('preserve') expect(Object.values(plan.components).every(Boolean)).to.equal(true) }) @@ -23,7 +28,7 @@ describe('template plan', () => { const plan = buildTemplatePlan({content: true, schema: true}) expect(plan.partial).to.equal(true) - expect(plan.relationStrategy).to.equal('ids') + expect(plan.relationStrategy).to.equal('empty') expect(plan.components.content).to.equal(true) expect(plan.components.schema).to.equal(true) expect(plan.components.files).to.equal(false) @@ -95,8 +100,8 @@ describe('template plan', () => { expect(plan.excludeCollections).to.deep.equal(['analytics_events', 'assets', 'directus_files']) }) - it('keeps relations to excluded collections for ids strategy', () => { - const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'ids'}) + it('keeps schema relations to excluded collections for preserve strategy', () => { + const plan = buildTemplatePlan({excludeCollections: 'assets', relationStrategy: 'preserve'}) expect(includesRelation('posts', 'assets', plan)).to.equal(true) }) @@ -119,14 +124,34 @@ describe('template plan', () => { expect(includesRelation('posts', 'assets', plan)).to.equal(false) }) + it('keeps relations to system collections not explicitly excluded', () => { + const plan = buildTemplatePlan({collections: 'posts', relationStrategy: 'empty'}) + + expect(includesRelation('posts', 'directus_users', plan)).to.equal(true) + }) + + it('drops relations to system collections when explicitly excluded', () => { + const plan = buildTemplatePlan({excludeCollections: 'directus_files', relationStrategy: 'empty'}) + + expect(includesRelation('posts', 'directus_files', plan)).to.equal(false) + }) + + it('keeps relations to explicitly excluded system collections for preserve strategy', () => { + const plan = buildTemplatePlan({excludeCollections: 'directus_files', relationStrategy: 'preserve'}) + + expect(includesRelation('posts', 'directus_files', plan)).to.equal(true) + }) + it('preserves metadata warnings', () => { - const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'}), [{ - collection: 'posts', - count: 1, - field: 'image', - relatedCollection: 'directus_files', - type: 'excluded_relation', - }]) + const metadata = createTemplateMetadata(buildTemplatePlan({excludeCollections: 'directus_files'}), [ + { + collection: 'posts', + count: 1, + field: 'image', + relatedCollection: 'directus_files', + type: 'excluded_relation', + }, + ]) expect(metadata.warnings).to.have.length(1) }) From c34793648e2eabf3e46ede6e56a79a27c75a3694 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Wed, 13 May 2026 11:07:42 -0400 Subject: [PATCH 09/12] lint and format --- src/lib/extract/expand-schema-plan.ts | 2 +- src/lib/extract/extract-assets.ts | 5 +++++ src/lib/load/finalize-collections.ts | 16 ++++++++++++---- src/lib/load/finalize-fields.ts | 11 ++++++++++- src/lib/load/index.ts | 2 +- src/lib/template-plan/metadata-plan.ts | 2 +- src/lib/template-plan/metadata.ts | 2 +- 7 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/lib/extract/expand-schema-plan.ts b/src/lib/extract/expand-schema-plan.ts index 6b4dbf39..15777c7c 100644 --- a/src/lib/extract/expand-schema-plan.ts +++ b/src/lib/extract/expand-schema-plan.ts @@ -10,7 +10,7 @@ interface CollectionInfo { meta?: { group?: null | string } - schema?: Record | null + schema?: null | Record } interface RelationInfo { diff --git a/src/lib/extract/extract-assets.ts b/src/lib/extract/extract-assets.ts index 2d37d834..208da9ed 100644 --- a/src/lib/extract/extract-assets.ts +++ b/src/lib/extract/extract-assets.ts @@ -20,6 +20,7 @@ async function getAssetPage(page: number): Promise { } async function downloadFile(file: DirectusFile, dir: string) { + // eslint-disable-next-line n/no-unsupported-features/node-builtins const response: Response | string = await api.client.request(() => ({ method: 'GET', path: `/assets/${file.id}`, @@ -45,8 +46,12 @@ export async function downloadAllFiles(dir: string) { let page = 1 while (true) { ux.action.status = `Downloading assets page ${page}` + // Intentional: page asset metadata sequentially to avoid queuing all downloads at once. + // eslint-disable-next-line no-await-in-loop const fileList = await getAssetPage(page) + // Intentional: finish one asset page before fetching the next page. + // eslint-disable-next-line no-await-in-loop await Promise.all( fileList.map((file) => downloadFile(file, dir).catch((error) => { diff --git a/src/lib/load/finalize-collections.ts b/src/lib/load/finalize-collections.ts index 59c1bc10..b4812073 100644 --- a/src/lib/load/finalize-collections.ts +++ b/src/lib/load/finalize-collections.ts @@ -7,14 +7,22 @@ import {includesSchemaCollection, type TemplatePlan} from '../template-plan/inde import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' +interface TemplateCollection { + collection: string + meta?: { + [key: string]: unknown + group?: null | string + } +} + export default async function finalizeCollections(dir: string, plan?: TemplatePlan) { - const collections = readFile('collections', dir) - .filter((collection: any) => includesSchemaCollection(collection.collection, plan)) - .filter((collection: any) => !collection.collection.startsWith('directus_')) + const collections = (readFile('collections', dir) as TemplateCollection[]) + .filter((collection) => includesSchemaCollection(collection.collection, plan)) + .filter((collection) => !collection.collection.startsWith('directus_')) ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${collections.length} collections`)) - const collectionNames = new Set(collections.map((collection: any) => collection.collection)) + const collectionNames = new Set(collections.map((collection) => collection.collection)) for await (const collection of collections) { const meta = {...collection.meta} diff --git a/src/lib/load/finalize-fields.ts b/src/lib/load/finalize-fields.ts index ae7b37f3..16da69c0 100644 --- a/src/lib/load/finalize-fields.ts +++ b/src/lib/load/finalize-fields.ts @@ -7,8 +7,17 @@ import {includesSchemaCollection, type TemplatePlan} from '../template-plan/inde import catchError from '../utils/catch-error.js' import readFile from '../utils/read-file.js' +interface TemplateField { + collection: string + field: string + meta?: Record + schema?: Record +} + export default async function finalizeFields(dir: string, plan?: TemplatePlan) { - const fields = readFile('fields', dir).filter((field: any) => includesSchemaCollection(field.collection, plan)) + const fields = (readFile('fields', dir) as TemplateField[]).filter((field) => + includesSchemaCollection(field.collection, plan), + ) ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${fields.length} fields`)) diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index b525bd99..d2f34338 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -4,9 +4,9 @@ import type {ApplyFlags} from './apply-flags.js' import {applyMetadataToPlan, buildTemplatePlan, readTemplateMetadata} from '../template-plan/index.js' import checkTemplate from '../utils/check-template.js' -import loadAccess from './load-access.js' import finalizeCollections from './finalize-collections.js' import finalizeFields from './finalize-fields.js' +import loadAccess from './load-access.js' import loadCollections from './load-collections.js' import loadDashboards from './load-dashboards.js' import loadData from './load-data.js' diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts index df6de550..9d448b02 100644 --- a/src/lib/template-plan/metadata-plan.ts +++ b/src/lib/template-plan/metadata-plan.ts @@ -30,9 +30,9 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad return { ...plan, collections: intersectCollections(plan.collections, metadata.collections), - schemaCollections: intersectCollections(plan.schemaCollections, metadata.schemaCollections), components, excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), partial, + schemaCollections: intersectCollections(plan.schemaCollections, metadata.schemaCollections), } } diff --git a/src/lib/template-plan/metadata.ts b/src/lib/template-plan/metadata.ts index e9e97af8..ab84c10f 100644 --- a/src/lib/template-plan/metadata.ts +++ b/src/lib/template-plan/metadata.ts @@ -11,11 +11,11 @@ export function createTemplateMetadata(plan: TemplatePlan, warnings: TemplateWar return { allowBrokenRelations: plan.allowBrokenRelations, collections: plan.collections, - schemaCollections: plan.schemaCollections, components: plan.components, excludedCollections: plan.excludeCollections, partial: plan.partial, relationStrategy: plan.relationStrategy, + schemaCollections: plan.schemaCollections, version: 2, warnings, } From 56b83b58f9afc66eaaf61fb3c674b672e5d8abff Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Wed, 13 May 2026 14:11:40 -0400 Subject: [PATCH 10/12] refactors --- src/commands/apply.ts | 4 +- src/lib/extract/expand-deep-plan.ts | 85 +++++++++++++------------- src/lib/extract/expand-schema-plan.ts | 85 +++++++++++++------------- src/lib/extract/extract-assets.ts | 3 +- src/lib/extract/extract-content.ts | 33 +++------- src/lib/extract/index.ts | 13 ++-- src/lib/load/apply-flags.ts | 16 ----- src/lib/load/finalize-fields.ts | 8 +-- src/lib/load/load-collections.ts | 6 +- src/lib/load/load-data.ts | 42 +++---------- src/lib/load/load-relations.ts | 8 ++- src/lib/load/update-required-fields.ts | 31 ---------- src/lib/template-plan/index.ts | 1 + src/lib/template-plan/junctions.ts | 33 ++++++++++ src/lib/template-plan/metadata-plan.ts | 17 ++++-- test/template-plan.test.ts | 82 ++++++++++++++++++++++++- 16 files changed, 245 insertions(+), 222 deletions(-) delete mode 100644 src/lib/load/update-required-fields.ts create mode 100644 src/lib/template-plan/junctions.ts diff --git a/src/commands/apply.ts b/src/commands/apply.ts index e4487f3a..85461577 100644 --- a/src/commands/apply.ts +++ b/src/commands/apply.ts @@ -12,7 +12,7 @@ import { DIRECTUS_PURPLE, SEPARATOR, } from '../lib/constants.js' -import {type ApplyFlags, validateInteractiveFlags, validateProgrammaticFlags} from '../lib/load/apply-flags.js' +import {type ApplyFlags, validateProgrammaticFlags} from '../lib/load/apply-flags.js' import apply from '../lib/load/index.js' import * as templatePlanFlags from '../lib/template-plan/flags.js' import {animatedBunny} from '../lib/utils/animated-bunny.js' @@ -102,7 +102,7 @@ export default class ApplyCommand extends BaseCommand { * @returns {Promise} - Returns nothing */ private async runInteractive(flags: ApplyFlags): Promise { - const validatedFlags = validateInteractiveFlags(flags) + const validatedFlags = flags // Show animated intro await animatedBunny("Let's apply a template!") diff --git a/src/lib/extract/expand-deep-plan.ts b/src/lib/extract/expand-deep-plan.ts index 08e5f827..d5142469 100644 --- a/src/lib/extract/expand-deep-plan.ts +++ b/src/lib/extract/expand-deep-plan.ts @@ -3,7 +3,6 @@ import {ux} from '@oclif/core' import {api} from '../sdk.js' import {includesCollection, type TemplatePlan} from '../template-plan/index.js' -import catchError from '../utils/catch-error.js' interface RelationInfo { collection: string @@ -17,61 +16,59 @@ interface RelationInfo { export async function expandDeepPlan(plan: TemplatePlan): Promise { if (!plan.partial || plan.relationStrategy !== 'deep' || !plan.collections) return plan - try { - const collections = await api.client.request(readCollections()) - const availableCollections = collections - .filter((collection) => !collection.collection.startsWith('directus_', 0)) - .filter((collection) => collection.schema !== null) - .map((collection) => collection.collection) - .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) + const collections = await api.client.request(readCollections()) + const availableCollections = collections + .filter((collection) => !collection.collection.startsWith('directus_', 0)) + .filter((collection) => collection.schema !== null) + .map((collection) => collection.collection) + .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) - const available = new Set(availableCollections) - const missingCollections = plan.collections.filter((collection) => !available.has(collection)) - if (missingCollections.length > 0) { - ux.warn(`Requested collections not found or excluded: ${missingCollections.join(', ')}`) - } + const available = new Set(availableCollections) + const missingCollections = plan.collections.filter((collection) => !available.has(collection)) + if (missingCollections.length > 0) { + ux.warn(`Requested collections not found or excluded: ${missingCollections.join(', ')}`) + } - const selected = new Set(plan.collections.filter((collection) => available.has(collection))) - const relations = (await api.client.request(readRelations())) as RelationInfo[] + const selected = new Set(plan.collections.filter((collection) => available.has(collection))) + const relations = (await api.client.request(readRelations())) as RelationInfo[] - let changed = true - while (changed) { - changed = false - const candidates: string[] = [] + let changed = true + while (changed) { + changed = false + const candidates: string[] = [] - for (const relation of relations) { - candidates.push( - ...([ - selected.has(relation.collection) && relation.related_collection, - relation.related_collection && selected.has(relation.related_collection) && relation.collection, - selected.has(relation.collection) && relation.meta?.one_allowed_collections, - ] - .flat() - .filter(Boolean) as string[]), - ) + for (const relation of relations) { + if (selected.has(relation.collection) && relation.related_collection) { + candidates.push(relation.related_collection) } - for (const collection of candidates) { - if (!available.has(collection)) continue - if (selected.has(collection)) continue + if (relation.related_collection && selected.has(relation.related_collection)) { + candidates.push(relation.collection) + } - selected.add(collection) - changed = true + if (selected.has(relation.collection) && relation.meta?.one_allowed_collections) { + candidates.push(...relation.meta.one_allowed_collections) } } - const expandedCollections = [...selected] - const addedCollections = expandedCollections.filter((collection) => !plan.collections?.includes(collection)) + for (const collection of candidates) { + if (!available.has(collection)) continue + if (selected.has(collection)) continue - if (addedCollections.length > 0) { - ux.warn(`Deep relation strategy expanded collections: ${addedCollections.join(', ')}`) + selected.add(collection) + changed = true } + } - return { - ...plan, - collections: expandedCollections, - } - } catch (error) { - catchError(error, {fatal: true}) + const expandedCollections = [...selected] + const addedCollections = expandedCollections.filter((collection) => !plan.collections?.includes(collection)) + + if (addedCollections.length > 0) { + ux.warn(`Deep relation strategy expanded collections: ${addedCollections.join(', ')}`) + } + + return { + ...plan, + collections: expandedCollections, } } diff --git a/src/lib/extract/expand-schema-plan.ts b/src/lib/extract/expand-schema-plan.ts index 15777c7c..3952c205 100644 --- a/src/lib/extract/expand-schema-plan.ts +++ b/src/lib/extract/expand-schema-plan.ts @@ -3,7 +3,6 @@ import {ux} from '@oclif/core' import {api} from '../sdk.js' import {includesCollection, type TemplatePlan} from '../template-plan/index.js' -import catchError from '../utils/catch-error.js' interface CollectionInfo { collection: string @@ -25,62 +24,60 @@ interface RelationInfo { export async function expandSchemaPlan(plan: TemplatePlan): Promise { if (!plan.partial || !plan.collections) return plan - try { - const collections = (await api.client.request(readCollections())) as CollectionInfo[] - const availableCollections = collections - .filter((collection) => !collection.collection.startsWith('directus_', 0)) - .map((collection) => collection.collection) - .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) - const collectionMap = new Map(collections.map((collection) => [collection.collection, collection])) + const collections = (await api.client.request(readCollections())) as CollectionInfo[] + const availableCollections = collections + .filter((collection) => !collection.collection.startsWith('directus_', 0)) + .map((collection) => collection.collection) + .filter((collection) => includesCollection(collection, {...plan, collections: undefined})) + const collectionMap = new Map(collections.map((collection) => [collection.collection, collection])) - const available = new Set(availableCollections) - const selected = new Set(plan.collections.filter((collection) => available.has(collection))) - const relations = (await api.client.request(readRelations())) as RelationInfo[] + const available = new Set(availableCollections) + const selected = new Set(plan.collections.filter((collection) => available.has(collection))) + const relations = (await api.client.request(readRelations())) as RelationInfo[] - let changed = true - while (changed) { - changed = false + let changed = true + while (changed) { + changed = false - const candidates: string[] = [] + const candidates: string[] = [] - for (const collection of selected) { - const group = collectionMap.get(collection)?.meta?.group - if (group) candidates.push(group) - } + for (const collection of selected) { + const group = collectionMap.get(collection)?.meta?.group + if (group) candidates.push(group) + } - for (const relation of relations) { - candidates.push( - ...([ - selected.has(relation.collection) && relation.related_collection, - relation.related_collection && selected.has(relation.related_collection) && relation.collection, - selected.has(relation.collection) && relation.meta?.one_allowed_collections, - ] - .flat() - .filter(Boolean) as string[]), - ) + for (const relation of relations) { + if (selected.has(relation.collection) && relation.related_collection) { + candidates.push(relation.related_collection) } - for (const collection of candidates) { - if (!available.has(collection)) continue - if (selected.has(collection)) continue + if (relation.related_collection && selected.has(relation.related_collection)) { + candidates.push(relation.collection) + } - selected.add(collection) - changed = true + if (selected.has(relation.collection) && relation.meta?.one_allowed_collections) { + candidates.push(...relation.meta.one_allowed_collections) } } - const schemaCollections = [...selected] - const addedCollections = schemaCollections.filter((collection) => !plan.collections?.includes(collection)) + for (const collection of candidates) { + if (!available.has(collection)) continue + if (selected.has(collection)) continue - if (addedCollections.length > 0) { - ux.warn(`Schema scope expanded collections: ${addedCollections.join(', ')}`) + selected.add(collection) + changed = true } + } - return { - ...plan, - schemaCollections, - } - } catch (error) { - catchError(error, {fatal: true}) + const schemaCollections = [...selected] + const addedCollections = schemaCollections.filter((collection) => !plan.collections?.includes(collection)) + + if (addedCollections.length > 0) { + ux.warn(`Schema scope expanded collections: ${addedCollections.join(', ')}`) + } + + return { + ...plan, + schemaCollections, } } diff --git a/src/lib/extract/extract-assets.ts b/src/lib/extract/extract-assets.ts index 208da9ed..958d4d77 100644 --- a/src/lib/extract/extract-assets.ts +++ b/src/lib/extract/extract-assets.ts @@ -46,11 +46,10 @@ export async function downloadAllFiles(dir: string) { let page = 1 while (true) { ux.action.status = `Downloading assets page ${page}` - // Intentional: page asset metadata sequentially to avoid queuing all downloads at once. + // Page asset metadata sequentially and finish each page before fetching the next, to avoid queuing all downloads at once. // eslint-disable-next-line no-await-in-loop const fileList = await getAssetPage(page) - // Intentional: finish one asset page before fetching the next page. // eslint-disable-next-line no-await-in-loop await Promise.all( fileList.map((file) => diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index 6d387616..5c2bda3d 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -3,7 +3,12 @@ import {ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' import {api} from '../sdk.js' -import {includesCollection, type TemplatePlan, type TemplateWarning} from '../template-plan/index.js' +import { + getBrokenJunctionCollections, + includesCollection, + type TemplatePlan, + type TemplateWarning, +} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import writeToFile from '../utils/write-to-file.js' @@ -26,31 +31,9 @@ interface ExcludedRelationField { type: 'alias' | 'm2o' } -function getJunctionCollectionsWithBrokenFKs(relations: RelationInfo[], plan?: TemplatePlan): Set { - if (!plan?.partial) return new Set() - - // Directus sets meta.junction_field on both FK legs of an M2M to point to the other FK. - // Any collection appearing as `collection` on such a relation is a junction table. - const junctionCollections = new Set(relations.filter((r) => r.meta?.junction_field).map((r) => r.collection)) - - const broken = new Set() - for (const junction of junctionCollections) { - const targets = relations - .filter((r) => r.collection === junction && r.related_collection) - .map((r) => r.related_collection as string) - - // System collections always exist on every instance — never treat them as broken FK targets. - if (targets.some((target) => !target.startsWith('directus_') && !includesCollection(target, plan))) { - broken.add(junction) - } - } - - return broken -} - async function getCollections(relations: RelationInfo[], plan?: TemplatePlan) { const response = await api.client.request(readCollections()) - const brokenJunctions = getJunctionCollectionsWithBrokenFKs(relations, plan) + const brokenJunctions = getBrokenJunctionCollections(relations, plan) return response .filter((item) => !item.collection.startsWith('directus_', 0)) .filter((item) => item.schema !== null) @@ -170,7 +153,7 @@ async function getDataFromCollection( context: {collection, function: 'getDataFromCollection'}, fatal: true, }) - return [] + throw error } } diff --git a/src/lib/extract/index.ts b/src/lib/extract/index.ts index cf569476..b6286864 100644 --- a/src/lib/extract/index.ts +++ b/src/lib/extract/index.ts @@ -1,5 +1,5 @@ import {ux} from '@oclif/core' -import fs from 'node:fs' +import fs from 'node:fs/promises' import { buildTemplatePlan, @@ -35,13 +35,10 @@ export default async function extract(dir: string, plan: TemplatePlan = buildTem const schemaPlan = await expandSchemaPlan(plan) const effectivePlan = await expandDeepPlan(schemaPlan) - if (!fs.existsSync(destination)) { - ux.stdout(`Attempting to create directory at: ${destination}`) - try { - fs.mkdirSync(destination, {recursive: true}) - } catch (error) { - catchError(error, {context: {destination}, fatal: true}) - } + try { + await fs.mkdir(destination, {recursive: true}) + } catch (error) { + catchError(error, {context: {destination}, fatal: true}) } if (effectivePlan.components.schema) { diff --git a/src/lib/load/apply-flags.ts b/src/lib/load/apply-flags.ts index 9e1b5e75..c6568bd0 100644 --- a/src/lib/load/apply-flags.ts +++ b/src/lib/load/apply-flags.ts @@ -27,18 +27,6 @@ export interface ApplyFlags { users?: boolean } -export const loadFlags = [ - 'content', - 'dashboards', - 'extensions', - 'files', - 'flows', - 'permissions', - 'schema', - 'settings', - 'users', -] as const - export function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags { const {directusToken, directusUrl, templateLocation, userEmail, userPassword} = flags @@ -49,7 +37,3 @@ export function validateProgrammaticFlags(flags: ApplyFlags): ApplyFlags { return flags } - -export function validateInteractiveFlags(flags: ApplyFlags): ApplyFlags { - return flags -} diff --git a/src/lib/load/finalize-fields.ts b/src/lib/load/finalize-fields.ts index 16da69c0..2a9d7ecb 100644 --- a/src/lib/load/finalize-fields.ts +++ b/src/lib/load/finalize-fields.ts @@ -15,9 +15,9 @@ interface TemplateField { } export default async function finalizeFields(dir: string, plan?: TemplatePlan) { - const fields = (readFile('fields', dir) as TemplateField[]).filter((field) => - includesSchemaCollection(field.collection, plan), - ) + const fields = (readFile('fields', dir) as TemplateField[]) + .filter((field) => includesSchemaCollection(field.collection, plan)) + .filter((field) => field.schema) ux.action.start(ux.colorize(DIRECTUS_PINK, `Finalizing metadata for ${fields.length} fields`)) @@ -30,7 +30,7 @@ export default async function finalizeFields(dir: string, plan?: TemplatePlan) { }), ) } catch (error) { - catchError(error) + catchError(error, {context: {collection: field.collection, field: field.field}}) } } diff --git a/src/lib/load/load-collections.ts b/src/lib/load/load-collections.ts index 91f259a5..2637faed 100644 --- a/src/lib/load/load-collections.ts +++ b/src/lib/load/load-collections.ts @@ -42,7 +42,7 @@ async function processCollections(collectionsToAdd: any[], fieldsToAdd: any[]) { ? addNewFieldsToExistingCollection(collection.collection, fieldsToAdd, existingFields) : addNewCollectionWithFields(collection, fieldsToAdd)) } catch (error) { - catchError(error) + catchError(error, {context: {collection: collection.collection}}) } } } @@ -89,7 +89,7 @@ async function addNewFieldsToExistingCollection(collectionName: string, fieldsTo // @ts-ignore - ignore await api.client.request(createField(collectionName, field)) } catch (error) { - catchError(error) + catchError(error, {context: {collection: collectionName, field: field.field}}) } } } @@ -111,7 +111,7 @@ async function addCustomFieldsOnSystemCollections(fields: any[]) { await api.client.request(createField(field.collection, field)) } } catch (error) { - catchError(error) + catchError(error, {context: {collection: field.collection, field: field.field}}) } } } diff --git a/src/lib/load/load-data.ts b/src/lib/load/load-data.ts index 3a1870ac..902ae7a3 100644 --- a/src/lib/load/load-data.ts +++ b/src/lib/load/load-data.ts @@ -5,7 +5,7 @@ import path from 'pathe' import {DIRECTUS_PINK} from '../constants.js' import {api} from '../sdk.js' -import {includesCollection, type TemplatePlan} from '../template-plan/index.js' +import {getBrokenJunctionCollections, includesCollection, type TemplatePlan} from '../template-plan/index.js' import catchError from '../utils/catch-error.js' import {chunkArray} from '../utils/chunk-array.js' import readFile from '../utils/read-file.js' @@ -35,35 +35,11 @@ function getContentCollections(dir: string): Set { ) } -function getBrokenJunctionCollections(dir: string, plan?: TemplatePlan): Set { - if (!plan?.partial) return new Set() - - const relations: Array<{ - collection: string - meta?: {junction_field?: null | string} - related_collection?: null | string - }> = readFile('relations', dir) - - const junctionCollections = new Set(relations.filter((r) => r.meta?.junction_field).map((r) => r.collection)) - - const broken = new Set() - for (const junction of junctionCollections) { - const targets = relations - .filter((r) => r.collection === junction && r.related_collection) - .map((r) => r.related_collection as string) - - if (targets.some((target) => !target.startsWith('directus_') && !includesCollection(target, plan))) { - broken.add(junction) - } - } - - return broken -} - function getUserCollections(dir: string, plan?: TemplatePlan) { const contentCollections = getContentCollections(dir) const collections = readFile('collections', dir) - const brokenJunctions = getBrokenJunctionCollections(dir, plan) + const relations = plan?.partial ? readFile('relations', dir) : [] + const brokenJunctions = getBrokenJunctionCollections(relations, plan) if (brokenJunctions.size > 0) { ux.warn(`Skipping junction collections with excluded FK targets: ${[...brokenJunctions].join(', ')}`) @@ -95,17 +71,13 @@ async function loadSkeletonRecords(dir: string, plan?: TemplatePlan) { // Filter out existing records const newData = data.filter((entry) => !existingPrimaryKeys.has(entry[primaryKeyField])) - if (newData.length === 0) { - // ux.stdout(`${ux.colorize('dim', '--')} Skipping ${name}: No new records to add`) - return - } + if (newData.length === 0) return const batches = chunkArray(newData, BATCH_SIZE).map((batch) => batch.map((entry) => ({[primaryKeyField]: entry[primaryKeyField]})), ) await Promise.all(batches.map((batch) => uploadBatch(name, batch, createItems))) - // ux.stdout(`${ux.colorize('dim', '--')} Added ${newData.length} new skeleton records to ${name}`) }), ) @@ -135,7 +107,7 @@ async function getExistingPrimaryKeys(collection: string, primaryKeyField: strin if (response.length < limit) break page++ } catch (error) { - catchError(error) + catchError(error, {context: {collection, page}}) break } } @@ -147,7 +119,7 @@ async function uploadBatch(collection: string, batch: any[], method: Function) { try { await api.client.request(method(collection, batch)) } catch (error) { - catchError(error) + catchError(error, {context: {batchSize: batch.length, collection}}) } } @@ -186,7 +158,7 @@ async function loadSingletons(dir: string, plan?: TemplatePlan) { await api.client.request(updateSingleton(name, cleanedData)) } catch (error) { - catchError(error) + catchError(error, {context: {collection: name}}) } }), ) diff --git a/src/lib/load/load-relations.ts b/src/lib/load/load-relations.ts index 8b95ad08..94ae42c3 100644 --- a/src/lib/load/load-relations.ts +++ b/src/lib/load/load-relations.ts @@ -54,7 +54,13 @@ async function addRelations(relations: TemplateRelation[]) { // eslint-disable-next-line no-await-in-loop await api.client.request(createRelation(relation)) } catch (error) { - catchError(error) + catchError(error, { + context: { + collection: relation.collection, + field: relation.field, + related: relation.related_collection, + }, + }) } } } diff --git a/src/lib/load/update-required-fields.ts b/src/lib/load/update-required-fields.ts deleted file mode 100644 index 59ceda20..00000000 --- a/src/lib/load/update-required-fields.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {updateField} from '@directus/sdk' -import {ux} from '@oclif/core' - -import {DIRECTUS_PINK} from '../constants.js' -import {api} from '../sdk.js' -import {includesSchemaCollection, type TemplatePlan} from '../template-plan/index.js' -import catchError from '../utils/catch-error.js' -import readFile from '../utils/read-file.js' - -export default async function updateRequiredFields(dir: string, plan?: TemplatePlan) { - const fieldsToUpdate = readFile('fields', dir) - .filter((field) => includesSchemaCollection(field.collection, plan)) - .filter( - (field) => - field.meta.required === true || field.schema?.is_nullable === false || field.schema?.is_unique === true, - ) - - ux.action.start(ux.colorize(DIRECTUS_PINK, `Updating ${fieldsToUpdate.length} fields to required`)) - - for await (const field of fieldsToUpdate) { - try { - await api.client.request( - updateField(field.collection, field.field, {meta: {...field.meta}, schema: {...field.schema}}), - ) - } catch (error) { - catchError(error) - } - } - - ux.action.stop() -} diff --git a/src/lib/template-plan/index.ts b/src/lib/template-plan/index.ts index b5912951..6bbc74db 100644 --- a/src/lib/template-plan/index.ts +++ b/src/lib/template-plan/index.ts @@ -90,6 +90,7 @@ export function buildTemplatePlan(flags: BuildFlags = {}): TemplatePlan { export * from './collections.js' export * from './flags.js' +export * from './junctions.js' export * from './metadata-plan.js' export * from './metadata.js' export * from './types.js' diff --git a/src/lib/template-plan/junctions.ts b/src/lib/template-plan/junctions.ts new file mode 100644 index 00000000..95fcee17 --- /dev/null +++ b/src/lib/template-plan/junctions.ts @@ -0,0 +1,33 @@ +import type {TemplatePlan} from './types.js' + +import {includesCollection} from './collections.js' + +interface JunctionRelation { + collection: string + meta?: { + junction_field?: null | string + } + related_collection?: null | string +} + +// Directus sets meta.junction_field on both FK legs of an M2M to point to the other FK. +// Any collection appearing as `collection` on such a relation is a junction table. +// System collections (directus_*) always exist on every instance — never treat them as broken FK targets. +export function getBrokenJunctionCollections(relations: JunctionRelation[], plan?: TemplatePlan): Set { + if (!plan?.partial) return new Set() + + const junctionCollections = new Set(relations.filter((r) => r.meta?.junction_field).map((r) => r.collection)) + + const broken = new Set() + for (const junction of junctionCollections) { + const targets = relations + .filter((r) => r.collection === junction && r.related_collection) + .map((r) => r.related_collection as string) + + if (targets.some((target) => !target.startsWith('directus_') && !includesCollection(target, plan))) { + broken.add(junction) + } + } + + return broken +} diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts index 9d448b02..7583a724 100644 --- a/src/lib/template-plan/metadata-plan.ts +++ b/src/lib/template-plan/metadata-plan.ts @@ -1,11 +1,20 @@ import type {TemplateMetadata, TemplatePlan} from './types.js' +import catchError from '../utils/catch-error.js' import {componentNames} from './flags.js' -function intersectCollections(requested?: string[], available?: string[]): string[] | undefined { +function intersectCollections( + scope: string, + requested?: string[], + available?: string[], +): string[] | undefined { if (requested && available) { const collections = requested.filter((collection) => available.includes(collection)) - return collections.length > 0 ? collections : undefined + if (collections.length === 0) { + catchError(new Error(`No requested ${scope} match this template`), {fatal: true}) + } + + return collections } return requested || available @@ -29,10 +38,10 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad return { ...plan, - collections: intersectCollections(plan.collections, metadata.collections), + collections: intersectCollections('collections', plan.collections, metadata.collections), components, excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), partial, - schemaCollections: intersectCollections(plan.schemaCollections, metadata.schemaCollections), + schemaCollections: intersectCollections('schema collections', plan.schemaCollections, metadata.schemaCollections), } } diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 2128ed49..701480ee 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -4,6 +4,7 @@ import { applyMetadataToPlan, buildTemplatePlan, createTemplateMetadata, + getBrokenJunctionCollections, includesRelation, } from '../src/lib/template-plan/index.js' @@ -178,10 +179,85 @@ describe('template plan', () => { expect(plan.collections).to.deep.equal(['posts']) }) - it('returns undefined collections when requested has no overlap with metadata', () => { + it('errors when requested collections have no overlap with metadata', () => { const metadata = createTemplateMetadata(buildTemplatePlan({collections: 'posts'})) - const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'authors'}), metadata) - expect(plan.collections).to.equal(undefined) + expect(() => applyMetadataToPlan(buildTemplatePlan({collections: 'authors'}), metadata)).to.throw( + /No requested collections match this template/, + ) + }) + + it('errors when at least one component must be enabled', () => { + expect(() => buildTemplatePlan({content: false, dashboards: false, extensions: false, files: false, flows: false, permissions: false, schema: false, settings: false, users: false})).to.throw( + /At least one template component must be enabled/, + ) + }) + + it('parses array collection input', () => { + const plan = buildTemplatePlan({collections: ['posts', 'pages']}) + expect(plan.collections).to.deep.equal(['posts', 'pages']) + }) + + it('round-trips metadata with mixed flags', () => { + const original = buildTemplatePlan({ + collections: 'posts,pages', + excludeCollections: 'directus_files', + noAssets: true, + }) + const metadata = createTemplateMetadata(original) + const restored = applyMetadataToPlan(buildTemplatePlan({collections: 'posts,pages'}), metadata) + + expect(restored.components.files).to.equal(false) + expect(restored.excludeCollections).to.include('directus_files') + expect(restored.collections).to.deep.equal(['posts', 'pages']) + expect(restored.partial).to.equal(true) + }) + + describe('getBrokenJunctionCollections', () => { + it('returns empty set for non-partial plans', () => { + const plan = buildTemplatePlan({}) + const relations = [ + {collection: 'posts_tags', meta: {junction_field: 'tag_id'}, related_collection: 'posts'}, + {collection: 'posts_tags', meta: {junction_field: 'post_id'}, related_collection: 'tags'}, + ] + expect(getBrokenJunctionCollections(relations, plan).size).to.equal(0) + }) + + it('marks junction broken when one FK target is excluded', () => { + const plan = buildTemplatePlan({collections: 'posts,posts_tags'}) + const relations = [ + {collection: 'posts_tags', meta: {junction_field: 'tag_id'}, related_collection: 'posts'}, + {collection: 'posts_tags', meta: {junction_field: 'post_id'}, related_collection: 'tags'}, + ] + const broken = getBrokenJunctionCollections(relations, plan) + expect(broken.has('posts_tags')).to.equal(true) + }) + + it('does not mark junction broken when target is a system collection', () => { + const plan = buildTemplatePlan({collections: 'posts,posts_files'}) + const relations = [ + {collection: 'posts_files', meta: {junction_field: 'directus_files_id'}, related_collection: 'posts'}, + {collection: 'posts_files', meta: {junction_field: 'posts_id'}, related_collection: 'directus_files'}, + ] + const broken = getBrokenJunctionCollections(relations, plan) + expect(broken.has('posts_files')).to.equal(false) + }) + + it('does not mark junction broken when both targets included', () => { + const plan = buildTemplatePlan({collections: 'posts,tags,posts_tags'}) + const relations = [ + {collection: 'posts_tags', meta: {junction_field: 'tag_id'}, related_collection: 'posts'}, + {collection: 'posts_tags', meta: {junction_field: 'post_id'}, related_collection: 'tags'}, + ] + expect(getBrokenJunctionCollections(relations, plan).size).to.equal(0) + }) + + it('ignores non-junction relations', () => { + const plan = buildTemplatePlan({collections: 'posts'}) + const relations = [ + {collection: 'posts', meta: {}, related_collection: 'missing'}, + ] + expect(getBrokenJunctionCollections(relations, plan).size).to.equal(0) + }) }) }) From ac9a85f6f2b9ff93bb040b288dad34dd56c2c401 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Wed, 13 May 2026 15:24:39 -0400 Subject: [PATCH 11/12] 0.8.0-partials --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e65d9c53..5ea3c21e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "directus-template-cli", - "version": "0.7.7", + "version": "0.8.0-partials.0", "description": "CLI Utility for applying templates to a Directus instance.", "author": "bryantgillespie @bryantgillespie", "type": "module", From e8bfd4b2a341b7965b05e8ed38a4a32f0b7b56e4 Mon Sep 17 00:00:00 2001 From: bryantgillespie Date: Wed, 3 Jun 2026 16:16:06 -0400 Subject: [PATCH 12/12] fix: address partial template review feedback --- src/lib/extract/extract-content.ts | 6 ++++-- src/lib/load/index.ts | 6 +++++- src/lib/load/load-data.ts | 5 +++-- src/lib/template-plan/metadata-plan.ts | 5 ++++- src/lib/template-plan/metadata.ts | 17 ++++++++++------ test/lib/load/load-data.test.ts | 28 ++++++++++++++++++++++++++ test/template-plan.test.ts | 18 +++++++++++++++++ 7 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 test/lib/load/load-data.test.ts diff --git a/src/lib/extract/extract-content.ts b/src/lib/extract/extract-content.ts index 5c2bda3d..3765996f 100644 --- a/src/lib/extract/extract-content.ts +++ b/src/lib/extract/extract-content.ts @@ -1,5 +1,5 @@ import {readCollections, readItems, readRelations} from '@directus/sdk' -import {ux} from '@oclif/core' +import {Errors, ux} from '@oclif/core' import {DIRECTUS_PINK} from '../constants.js' import {api} from '../sdk.js' @@ -153,7 +153,6 @@ async function getDataFromCollection( context: {collection, function: 'getDataFromCollection'}, fatal: true, }) - throw error } } @@ -166,11 +165,14 @@ export async function extractContent(dir: string, plan?: TemplatePlan): Promise< const collections = await getCollections(relations, plan) for (const collection of collections) { + // Keep extraction sequential so the shared ux.action.status reflects the active collection. // eslint-disable-next-line no-await-in-loop const collectionWarnings = await getDataFromCollection(collection, dir, relations, plan) warnings.push(...collectionWarnings) } } catch (error) { + if (error instanceof Errors.CLIError) throw error + catchError(error, { context: {function: 'extractContent'}, fatal: true, diff --git a/src/lib/load/index.ts b/src/lib/load/index.ts index d2f34338..ac20db56 100644 --- a/src/lib/load/index.ts +++ b/src/lib/load/index.ts @@ -56,7 +56,6 @@ export default async function apply(dir: string, flags: ApplyFlags) { await loadCollections(source, effectivePlan) await loadRelations(source, effectivePlan) await finalizeCollections(source, effectivePlan) - await finalizeFields(source, effectivePlan) } if (components.permissions || components.users) { @@ -80,6 +79,11 @@ export default async function apply(dir: string, flags: ApplyFlags) { await loadData(source, effectivePlan) } + if (components.schema) { + // Finalize fields after data loading because skeleton records rely on relaxed constraints. + await finalizeFields(source, effectivePlan) + } + if (components.dashboards) { await loadDashboards(source) } diff --git a/src/lib/load/load-data.ts b/src/lib/load/load-data.ts index 902ae7a3..d48f9e22 100644 --- a/src/lib/load/load-data.ts +++ b/src/lib/load/load-data.ts @@ -35,10 +35,11 @@ function getContentCollections(dir: string): Set { ) } -function getUserCollections(dir: string, plan?: TemplatePlan) { +export function getUserCollections(dir: string, plan?: TemplatePlan) { const contentCollections = getContentCollections(dir) const collections = readFile('collections', dir) - const relations = plan?.partial ? readFile('relations', dir) : [] + const relationsPath = path.join(dir, 'relations.json') + const relations = plan?.partial && fs.existsSync(relationsPath) ? readFile('relations', dir) : [] const brokenJunctions = getBrokenJunctionCollections(relations, plan) if (brokenJunctions.size > 0) { diff --git a/src/lib/template-plan/metadata-plan.ts b/src/lib/template-plan/metadata-plan.ts index 7583a724..6f3694d6 100644 --- a/src/lib/template-plan/metadata-plan.ts +++ b/src/lib/template-plan/metadata-plan.ts @@ -34,7 +34,9 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad } const partial = - metadata.partial || componentNames.some((component) => components[component] !== plan.components[component]) + plan.partial || + metadata.partial || + componentNames.some((component) => components[component] !== plan.components[component]) return { ...plan, @@ -42,6 +44,7 @@ export function applyMetadataToPlan(plan: TemplatePlan, metadata?: TemplateMetad components, excludeCollections: mergeExcludedCollections(plan.excludeCollections, metadata.excludedCollections), partial, + relationStrategy: metadata.relationStrategy ?? plan.relationStrategy, schemaCollections: intersectCollections('schema collections', plan.schemaCollections, metadata.schemaCollections), } } diff --git a/src/lib/template-plan/metadata.ts b/src/lib/template-plan/metadata.ts index ab84c10f..3e5331bb 100644 --- a/src/lib/template-plan/metadata.ts +++ b/src/lib/template-plan/metadata.ts @@ -29,16 +29,21 @@ export function readTemplateMetadata(dir: string): TemplateMetadata | undefined const filePath = getTemplateMetadataPath(dir) if (!fs.existsSync(filePath)) return undefined - try { - const metadata = JSON.parse(fs.readFileSync(filePath, 'utf8')) as TemplateMetadata - if (metadata.version !== 2) { - catchError(new Error(`Unsupported template metadata version: ${metadata.version}`), {fatal: true}) - } + let metadata: TemplateMetadata - return metadata + try { + metadata = JSON.parse(fs.readFileSync(filePath, 'utf8')) as TemplateMetadata } catch (error) { catchError(error, {fatal: true}) + return undefined } + + if (metadata.version !== 2) { + catchError(new Error(`Unsupported template metadata version: ${metadata.version}`), {fatal: true}) + return undefined + } + + return metadata } export async function writeTemplateMetadata( diff --git a/test/lib/load/load-data.test.ts b/test/lib/load/load-data.test.ts new file mode 100644 index 00000000..fd6012a2 --- /dev/null +++ b/test/lib/load/load-data.test.ts @@ -0,0 +1,28 @@ +import {expect} from 'chai' +import fs from 'node:fs' +import os from 'node:os' +import path from 'pathe' + +import {getUserCollections} from '../../../src/lib/load/load-data.js' +import {buildTemplatePlan} from '../../../src/lib/template-plan/index.js' + +describe('load data', () => { + it('does not require relations.json for content-only partial templates', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'directus-template-cli-')) + + try { + fs.mkdirSync(path.join(dir, 'content')) + fs.writeFileSync( + path.join(dir, 'collections.json'), + JSON.stringify([{collection: 'posts', meta: {singleton: false}, schema: {}}]), + ) + fs.writeFileSync(path.join(dir, 'content', 'posts.json'), JSON.stringify([])) + + const collections = getUserCollections(dir, buildTemplatePlan({content: true})) + + expect(collections.map((collection) => collection.collection)).to.deep.equal(['posts']) + } finally { + fs.rmSync(dir, {force: true, recursive: true}) + } + }) +}) diff --git a/test/template-plan.test.ts b/test/template-plan.test.ts index 701480ee..f61cc8d7 100644 --- a/test/template-plan.test.ts +++ b/test/template-plan.test.ts @@ -172,6 +172,24 @@ describe('template plan', () => { expect(plan.components.files).to.equal(false) }) + it('preserves requested partial mode when applying full template metadata', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({})) + const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'posts'}), metadata) + + expect(plan.partial).to.equal(true) + }) + + it('uses metadata relation strategy when applying partial template metadata', () => { + const metadata = createTemplateMetadata(buildTemplatePlan({ + excludeCollections: 'assets', + relationStrategy: 'preserve', + })) + const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'posts'}), metadata) + + expect(plan.relationStrategy).to.equal('preserve') + expect(includesRelation('posts', 'assets', plan)).to.equal(true) + }) + it('does not apply requested collections outside metadata bounds', () => { const metadata = createTemplateMetadata(buildTemplatePlan({collections: 'posts,pages'})) const plan = applyMetadataToPlan(buildTemplatePlan({collections: 'posts,authors'}), metadata)