diff --git a/__tests__/__fakes__/fakeDestination.repository.ts b/__tests__/__fakes__/fakeDestination.repository.ts index 239f4e6..6aedf40 100644 --- a/__tests__/__fakes__/fakeDestination.repository.ts +++ b/__tests__/__fakes__/fakeDestination.repository.ts @@ -152,4 +152,15 @@ export class FakeDestinationRepository // no-op in fake repository for testing return Promise.resolve(); } + + async deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }): Promise { + // no-op in fake repository for testing + return Promise.resolve(); + } } diff --git a/docs/content/docs/cli/guides/cli-commands.mdx b/docs/content/docs/cli/guides/cli-commands.mdx index 99389f9..0346db6 100644 --- a/docs/content/docs/cli/guides/cli-commands.mdx +++ b/docs/content/docs/cli/guides/cli-commands.mdx @@ -48,6 +48,8 @@ mk-notes sync -i -d -k - `-l, --lock`: Lock the Notion page after syncing to prevent further editing. This is useful when you want to preserve the synchronized content and prevent accidental modifications. +- `-f, --flat`: **Database destination only** - Flattens the directory structure so all markdown files are created as direct children of the database, not nested as child pages. This option is only available when the destination is a Notion database. If used with a page destination, an error will be thrown. + ### Destination Types Mk Notes supports two types of destinations: @@ -192,6 +194,31 @@ This command will: 2. Delete those existing pages 3. Create new pages as database items with the content from your markdown files +#### Flat Sync to a Database + +```bash +mk-notes sync \ + --input ./my-docs \ + --destination https://notion.so/myworkspace/database-123456 \ + --notion-api-key secret_abc123... \ + --flat +``` + +The `--flat` option creates all markdown files as direct children of the database, ignoring the directory structure. This is useful when you want a flat list of all documents in your database without nested page hierarchies. + +This command will: + +1. Read all markdown files in the directory +2. Flatten the directory structure +3. Create all files as direct database items (no nested pages) +4. Each file becomes a separate database entry + + + +The `--flat` option only works with database destinations. If you try to use it with a page destination, MK Notes will throw an error. + + + ## `preview-sync` The `preview-sync` command lets you preview how your markdown files will be organized in Notion before actually performing the synchronization. This is useful for verifying the structure before making any changes. diff --git a/docs/content/docs/cli/guides/database-sync.mdx b/docs/content/docs/cli/guides/database-sync.mdx index 0651637..62324aa 100644 --- a/docs/content/docs/cli/guides/database-sync.mdx +++ b/docs/content/docs/cli/guides/database-sync.mdx @@ -134,6 +134,45 @@ Clean sync only works for pages that have an `id` defined in their frontmatter. +### Flat Sync + +By default, MK Notes preserves your directory structure when syncing to a database, creating nested pages for subdirectories. The `--flat` option flattens this structure, creating all markdown files as direct children of the database: + +```bash +mk-notes sync \ + --input ./docs \ + --destination https://notion.so/myworkspace/database-123456 \ + --notion-api-key secret_abc123... \ + --flat +``` + +**When to use flat sync:** + +- You want all documents at the same level in your database +- You prefer to organize using database views and filters rather than page hierarchies +- Your directory structure is just for file organization, not content hierarchy + +**Example:** + +With a directory structure like: +``` +docs/ + guides/ + getting-started.md + advanced.md + api/ + authentication.md +``` + +- **Without `--flat`**: Creates nested pages (guides → getting-started, advanced) +- **With `--flat`**: Creates all four files as direct database entries + + + +The `--flat` option only works with database destinations. Using it with a page destination will result in an error. + + + --- ## Adding Custom Database Properties diff --git a/preview/index.js b/preview/index.js index fa906bb..a9e326c 100644 --- a/preview/index.js +++ b/preview/index.js @@ -75002,7 +75002,7 @@ class MkNotes { /** * Preview the synchronization of a markdown file to Notion */ - async previewSynchronization({ inputPath, format, output, }) { + async previewSynchronization({ inputPath, format, output, flat = false, }) { const previewSynchronizationFeature = new domains_1.PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, }); @@ -75010,6 +75010,7 @@ class MkNotes { path: inputPath, }, { format, + flat, }); if (!output) { return result; @@ -75020,7 +75021,7 @@ class MkNotes { /** * Synchronize a markdown file to Notion */ - async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, }) { + async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, flat = false, }) { const synchronizeMarkdownToNotion = new domains_1.SynchronizeMarkdownToNotion({ logger: this.logger, destinationRepository: this.infrastructureInstances.notionDestination, @@ -75032,6 +75033,7 @@ class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } @@ -75812,7 +75814,7 @@ class PreviewSynchronization { constructor(params) { this.sourceRepository = params.sourceRepository; } - async execute(args, { format } = {}) { + async execute(args, { format, flat = false, } = {}) { // Check if the GitHub repository is accessible try { await this.sourceRepository.sourceIsAccessible(args); @@ -75840,6 +75842,9 @@ class PreviewSynchronization { } const filePaths = await this.sourceRepository.getFilePathList(args); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } return sitemapSerializer(siteMap); } } @@ -75869,7 +75874,7 @@ class SynchronizeMarkdownToNotion { this.logger = params.logger; } async execute(args) { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { notionParentPageUrl, cleanSync, lockPage, flat = false, ...others } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, }); @@ -75895,10 +75900,16 @@ class SynchronizeMarkdownToNotion { if (parentObjectType === 'unknown') { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error('Flat sync is only supported for database destinations. Pages do not support flat sync.'); + } try { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -75906,6 +75917,7 @@ class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); } @@ -75995,32 +76007,19 @@ class SynchronizeMarkdownToNotion { return newPage.pageId; } async cleanSyncDatabase({ databaseId, pageElement, }) { - const dataSourceId = await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); if (pageElement.mkNotesInternalId === undefined) { this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync'); return; } - const objectIds = await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId({ - dataSourceId, + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, mkNotesInternalId: pageElement.mkNotesInternalId, }); - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - if (objectIds.length > 1) { - this.logger.info(`Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects`); - } - await Promise.all(objectIds.map(async (objectId) => this.destinationRepository.deleteObjectById({ - objectId, - }))); } /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, }) { + async synchronizeChildNode({ childNode, parentObjectId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76030,10 +76029,22 @@ class SynchronizeMarkdownToNotion { if (childNode.children.length > 0) { pageElement.addElementToEnd(new elements_1.DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync for child node'); + } + else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentObjectId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectId, + parentObjectType, filePath, }); this.logger.info(`Created Notion page for file: ${filePath}`); @@ -76044,8 +76055,9 @@ class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, + cleanSync, }); } await this.lockPageIfNeeded(newPage.pageId, lockPage); @@ -76053,30 +76065,21 @@ class SynchronizeMarkdownToNotion { /** * Main orchestrator for synchronizing a tree node and its children */ - async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, }) { + async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; + const isFlatSync = flat && parentObjectType === 'database'; switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -76096,8 +76099,10 @@ class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { @@ -76108,6 +76113,36 @@ class SynchronizeMarkdownToNotion { } } } + async handleDatabaseSynchronization({ node, parentObjectId, parentObjectType, lockPage, cleanSync, isFlatSync, }) { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } getIsRootNode(node) { return node.parent === null && !['', undefined].includes(node.filepath); } @@ -76482,7 +76517,12 @@ class SiteMap { removeUselessNodesTree(node) { while (node.children.length === 1 && path.extname(node.children[0].filepath) === '') { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; } @@ -76493,6 +76533,32 @@ class SiteMap { this.removeUselessNodesTree(this._root); this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten() { + const allNodes = []; + // Collect all nodes except root + const collectNodes = (node) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + // Start collection from root's children + this._root.children.forEach(collectNodes); + // Clear existing children of root + this._root.children = []; + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } /** * TODO: Implement mkdocs.yaml sitemap parsing * @@ -79173,6 +79239,25 @@ class NotionDestinationRepository { }); return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ databaseId, mkNotesInternalId, }) { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + if (objectIds.length > 1) { + this.logger.info(`Multiple object IDs found with ${mkNotesInternalId}, deleting all objects`); + } + await Promise.all(objectIds.map(async (objectId) => this.deleteObjectById({ + objectId, + }))); + } async deleteObjectById({ objectId }) { await this.client.blocks.delete({ block_id: objectId }); } diff --git a/src/MkNotes.ts b/src/MkNotes.ts index b1d014c..7e5dee1 100644 --- a/src/MkNotes.ts +++ b/src/MkNotes.ts @@ -48,10 +48,12 @@ export class MkNotes { inputPath, format, output, + flat = false, }: { inputPath: string; format: PreviewFormat; output?: string; + flat?: boolean; }): Promise { const previewSynchronizationFeature = new PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, @@ -63,6 +65,7 @@ export class MkNotes { }, { format, + flat, } ); @@ -83,11 +86,13 @@ export class MkNotes { parentNotionPageId, cleanSync = false, lockPage = false, + flat = false, }: { inputPath: string; parentNotionPageId: string; cleanSync?: boolean; lockPage?: boolean; + flat?: boolean; }): Promise { const synchronizeMarkdownToNotion = new SynchronizeMarkdownToNotion({ logger: this.logger, @@ -101,6 +106,7 @@ export class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } diff --git a/src/bin/cli/commands/preview.ts b/src/bin/cli/commands/preview.ts index 9c0de72..9852ec7 100644 --- a/src/bin/cli/commands/preview.ts +++ b/src/bin/cli/commands/preview.ts @@ -53,6 +53,11 @@ command.addOption( command.option('-o, --output ', 'Output file path'); +command.option( + '--flat', + 'Flat sync - If destination is a database, all files will be created as direct children of the database, not nested' +); + command.option('-v, --verbosity ', 'Verbosity level', 'error'); interface PreviewOptions { @@ -60,9 +65,16 @@ interface PreviewOptions { format: PreviewFormat; output?: string; verbosity?: string; + flat?: boolean; } command.action(async (opts: PreviewOptions) => { - const { input: directoryPath, format, output, verbosity = 'error' } = opts; + const { + input: directoryPath, + format, + output, + verbosity = 'error', + flat = false, + } = opts; if (!isValidVerbosity(verbosity)) { throw new Error(`Invalid verbosity: ${verbosity}`); @@ -77,6 +89,7 @@ command.action(async (opts: PreviewOptions) => { inputPath: directoryPath, format, output, + flat, }); // eslint-disable-next-line no-console diff --git a/src/bin/cli/commands/sync.ts b/src/bin/cli/commands/sync.ts index 00bde5f..d071fe9 100644 --- a/src/bin/cli/commands/sync.ts +++ b/src/bin/cli/commands/sync.ts @@ -34,6 +34,11 @@ command.option( command.option('-l, --lock', 'Lock the Notion page after syncing'); +command.option( + '-f, --flat', + 'Flat sync - If destination is a database, all files will be created as direct children of the database, not nested' +); + command.option('-v, --verbosity ', 'Verbosity level', 'error'); interface SyncOptions { input: string; @@ -42,6 +47,7 @@ interface SyncOptions { clean?: boolean; lock?: boolean; verbosity?: string; + flat?: boolean; } command.action(async (opts: SyncOptions) => { @@ -52,6 +58,7 @@ command.action(async (opts: SyncOptions) => { clean = false, lock = false, verbosity = 'error', + flat = false, } = opts; if (!isValidVerbosity(verbosity)) { @@ -68,6 +75,7 @@ command.action(async (opts: SyncOptions) => { parentNotionPageId: notionParentPageUrl, cleanSync: clean, lockPage: lock, + flat, }); // eslint-disable-next-line no-console diff --git a/src/domains/features/previewSynchronization.test.ts b/src/domains/features/previewSynchronization.test.ts index 5835616..7fa0078 100644 --- a/src/domains/features/previewSynchronization.test.ts +++ b/src/domains/features/previewSynchronization.test.ts @@ -91,5 +91,34 @@ describe('PreviewSynchronization', () => { expect(buildFromFilePathsSpy).toHaveBeenCalledWith(filePaths); }); + + it('should flatten the sitemap when flat option is true', async () => { + // Use a structure that prevents root collapsing of the directory + // root -> section, other.md + // section -> child1.md, child2.md + // section will consume child1.md + // child2.md will remain as child of section + const filePaths = ['section/child1.md', 'section/child2.md', 'other.md']; + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(filePaths); + + // When flattened, children should be at root level + const result = await previewSync.execute( + { path: 'test/path' }, + { format: 'json', flat: true } + ); + + const parsedResult = JSON.parse(result); + + const childrenNames = parsedResult.children.map((c: any) => c.name); + // 'section' node exists (having consumed child1.md) + expect(childrenNames).toContain('section'); + // 'child2.md' node exists (was nested, now flat) + expect(childrenNames).toContain('child2.md'); + // 'other.md' node exists + expect(childrenNames).toContain('other.md'); + expect(parsedResult.children.length).toBe(3); + }); }); }); diff --git a/src/domains/features/previewSynchronization.ts b/src/domains/features/previewSynchronization.ts index 7ee7aa4..304d96a 100644 --- a/src/domains/features/previewSynchronization.ts +++ b/src/domains/features/previewSynchronization.ts @@ -27,7 +27,10 @@ export class PreviewSynchronization { async execute( args: T, - { format }: { format?: PreviewFormat; output?: string } = {} + { + format, + flat = false, + }: { format?: PreviewFormat; output?: string; flat?: boolean } = {} ): Promise { // Check if the GitHub repository is accessible try { @@ -58,6 +61,10 @@ export class PreviewSynchronization { const siteMap = SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } + return sitemapSerializer(siteMap); } } diff --git a/src/domains/features/synchronizeMarkdownToNotion.ts b/src/domains/features/synchronizeMarkdownToNotion.ts index 2df2df7..acf5043 100644 --- a/src/domains/features/synchronizeMarkdownToNotion.ts +++ b/src/domains/features/synchronizeMarkdownToNotion.ts @@ -29,6 +29,9 @@ export interface SynchronizeOptions { /** When true, lock the Notion page after syncing */ lockPage: boolean; + + /** When true, and destination is a database, create all files as direct children */ + flat?: boolean; } export class SynchronizeMarkdownToNotion { @@ -49,7 +52,13 @@ export class SynchronizeMarkdownToNotion { notionParentPageUrl: string; } & SynchronizeOptions ): Promise { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { + notionParentPageUrl, + cleanSync, + lockPage, + flat = false, + ...others + } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, @@ -82,6 +91,12 @@ export class SynchronizeMarkdownToNotion { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error( + 'Flat sync is only supported for database destinations. Pages do not support flat sync.' + ); + } + try { this.logger.info('Starting synchronization process'); @@ -91,6 +106,10 @@ export class SynchronizeMarkdownToNotion { const siteMap = SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } + // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -98,6 +117,7 @@ export class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); @@ -233,11 +253,6 @@ export class SynchronizeMarkdownToNotion { databaseId: string; pageElement: PageElement; }): Promise { - const dataSourceId = - await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); - if (pageElement.mkNotesInternalId === undefined) { this.logger.warn( 'mk-notes-internal-id is undefined, skipping clean sync' @@ -245,44 +260,26 @@ export class SynchronizeMarkdownToNotion { return; } - const objectIds = - await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId( - { - dataSourceId, - mkNotesInternalId: pageElement.mkNotesInternalId, - } - ); - - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - - if (objectIds.length > 1) { - this.logger.info( - `Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects` - ); - } - - await Promise.all( - objectIds.map(async (objectId) => - this.destinationRepository.deleteObjectById({ - objectId, - }) - ) - ); + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); } /** * Synchronizes a child node and its descendants recursively */ private async synchronizeChildNode({ childNode, - parentPageId, + parentObjectId, lockPage, + cleanSync, + parentObjectType = 'page', }: { childNode: TreeNode; - parentPageId: string; + parentObjectId: string; lockPage: boolean; + cleanSync: boolean; + parentObjectType?: ObjectType; }): Promise { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); @@ -297,10 +294,24 @@ export class SynchronizeMarkdownToNotion { pageElement.addElementToEnd(new DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn( + 'mk-notes-internal-id is undefined, skipping clean sync for child node' + ); + } else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentObjectId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } + const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectId, + parentObjectType, filePath, }); @@ -314,8 +325,9 @@ export class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, + cleanSync, }); } @@ -331,36 +343,31 @@ export class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat = false, }: { node: TreeNode; parentObjectId: string; parentObjectType: ObjectType; lockPage: boolean; cleanSync: boolean; + flat?: boolean; }): Promise { let parentPageId: string = parentObjectId; + const isFlatSync = flat && parentObjectType === 'database'; + switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -381,8 +388,10 @@ export class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { this.logger.error(`Failed to synchronize file: ${childNode.filepath}`, { @@ -393,6 +402,53 @@ export class SynchronizeMarkdownToNotion { } } + private async handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }: { + node: TreeNode; + parentObjectId: string; + parentObjectType: ObjectType; + lockPage: boolean; + cleanSync: boolean; + isFlatSync: boolean; + }): Promise { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + private getIsRootNode(node: TreeNode): boolean { return node.parent === null && !['', undefined].includes(node.filepath); } diff --git a/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts new file mode 100644 index 0000000..4b6199d --- /dev/null +++ b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts @@ -0,0 +1,161 @@ +import { FakeFileConverter } from '../../../__tests__/__fakes__/fakeConverter.repository'; +import { FakeDestinationRepository } from '../../../__tests__/__fakes__/fakeDestination.repository'; +import { FakeFile } from '../../../__tests__/__fakes__/fakeFile'; +import { fakeLogger } from '../../../__tests__/__fakes__/fakeLogger'; +import { FakeNotionPage } from '../../../__tests__/__fakes__/fakePage'; +import { FakeSourceRepository } from '../../../__tests__/__fakes__/fakeSource.repository'; +import { PageElement, TextElement } from '../elements'; +import { SynchronizeMarkdownToNotion } from './synchronizeMarkdownToNotion'; + +describe('SynchronizeMarkdownToNotion - Flat Sync', () => { + let synchronizer: SynchronizeMarkdownToNotion; + let sourceRepository: FakeSourceRepository; + let destinationRepository: FakeDestinationRepository; + let elementConverter: FakeFileConverter; + + const databaseId = 'database-id-12345678901234567890123456789012'; + const databaseUrl = `https://www.notion.so/workspace/${databaseId}`; + + beforeEach(() => { + sourceRepository = new FakeSourceRepository(); + destinationRepository = new FakeDestinationRepository(); + elementConverter = new FakeFileConverter({ + logger: fakeLogger, + htmlParser: {} as any, + markdownParser: {} as any, + }); + + synchronizer = new SynchronizeMarkdownToNotion({ + sourceRepository, + destinationRepository, + elementConverter, + logger: fakeLogger, + }); + + jest + .spyOn(destinationRepository, 'destinationIsAccessible') + .mockResolvedValue(true); + jest.spyOn(sourceRepository, 'sourceIsAccessible').mockResolvedValue(true); + jest + .spyOn(destinationRepository, 'getObjectType') + .mockResolvedValue('database'); + jest + .spyOn(destinationRepository, 'createPage') + .mockResolvedValue(new FakeNotionPage({ pageId: 'new-page-id' })); + jest + .spyOn(destinationRepository, 'getDataSourceIdFromDatabaseId') + .mockResolvedValue('datasource-id'); + jest + .spyOn(destinationRepository, 'deletePagesInDatabaseByInternalId') + .mockResolvedValue(undefined); + }); + + it('should create all files as direct children of the database when flat is true', async () => { + // Setup hierarchical file structure + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue([ + 'parent.md', + 'child/child.md', + 'child/grandchild/grandchild.md', + ]); + + const pageElement = new PageElement({ + title: 'Test', + content: [new TextElement({ text: '# Test' })], + mkNotesInternalId: 'test-id', + }); + + jest + .spyOn(sourceRepository, 'getFile') + .mockResolvedValue(new FakeFile({ content: '# Test' })); + jest + .spyOn(elementConverter, 'convertToElement') + .mockReturnValue(pageElement); + + const createPageSpy = jest.spyOn(destinationRepository, 'createPage'); + + await synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: false, + lockPage: false, + flat: true, + path: 'test', + }); + + // Should create 3 pages + expect(createPageSpy).toHaveBeenCalledTimes(3); + + // All pages should be created with parentObjectId = databaseId and parentObjectType = 'database' + const calls = createPageSpy.mock.calls; + + for (const call of calls) { + expect(call[0]).toMatchObject({ + parentObjectId: '12345678901234567890123456789012', + parentObjectType: 'database', + }); + } + }); + + it('should respect cleanSync by calling deletePagesInDatabaseByInternalId in flat mode', async () => { + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(['child.md']); + + const pageElement = new PageElement({ + title: 'Child', + content: [], + mkNotesInternalId: 'child-internal-id', + }); + + jest + .spyOn(sourceRepository, 'getFile') + .mockResolvedValue(new FakeFile({ content: '' })); + jest + .spyOn(elementConverter, 'convertToElement') + .mockReturnValue(pageElement); + + const deleteSpy = jest.spyOn( + destinationRepository, + 'deletePagesInDatabaseByInternalId' + ); + + await synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: true, + lockPage: false, + flat: true, + path: 'test', + }); + + expect(deleteSpy).toHaveBeenCalledWith({ + databaseId: '12345678901234567890123456789012', + mkNotesInternalId: 'child-internal-id', + }); + }); + + it('should throw error if flat option is used with page destination', async () => { + // Mock destination as a page instead of a database + jest.spyOn(destinationRepository, 'getObjectType').mockResolvedValue('page'); + jest + .spyOn(destinationRepository, 'appendToPage') + .mockResolvedValue(undefined); + + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(['parent.md', 'child/child.md']); + + await expect( + synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: false, + lockPage: false, + flat: true, + path: 'test', + }) + ).rejects.toThrow( + 'Flat sync is only supported for database destinations. Pages do not support flat sync.' + ); + }); +}); + diff --git a/src/domains/sitemap/SiteMap.test.ts b/src/domains/sitemap/SiteMap.test.ts index ce401b6..e9181c8 100644 --- a/src/domains/sitemap/SiteMap.test.ts +++ b/src/domains/sitemap/SiteMap.test.ts @@ -134,5 +134,43 @@ describe('SiteMap', () => { it('should throw error for invalid JSON data', () => { expect(() => SiteMap.fromJSON({})).toThrow('Invalid data'); }); + it('should collapse directory structure if it has only one child (default behavior)', () => { + const filePaths = ['root/dir1/dir2/file.md']; + const sitemap = SiteMap.buildFromFilePaths(filePaths); + + // The tool collapses intermediate directories with single children + // root -> file.md + + expect(sitemap.root.children).toHaveLength(1); + const child = sitemap.root.children[0]; + expect(child.name).toBe('file.md'); + expect(child.children).toHaveLength(0); + }); + + it('should preserve directory structure if it has multiple children (though first file consumes parent)', () => { + const filePaths = ['dir1/file1.md', 'dir1/file2.md', 'other.md']; + const sitemap = SiteMap.buildFromFilePaths(filePaths); + + // root -> dir1, other.md + // dir1 has file1.md, file2.md + // BUT traverseAndUpdate applies "First Child Rule" to dir1 (since no index.md) + // dir1 consumes file1.md. dir1.filepath = file1.md path. + // dir1 children = [file2.md]. + + expect(sitemap.root.children).toHaveLength(2); + + const dir1 = sitemap.root.children.find(c => c.name === 'dir1'); + const other = sitemap.root.children.find(c => c.name === 'other.md'); + + expect(dir1).toBeDefined(); + expect(other).toBeDefined(); + + // Expect dir1 to have consumed file1.md and kept file2.md as child + expect(dir1!.children).toHaveLength(1); + expect(dir1!.children[0].name).toBe('file2.md'); + // dir1 filepath should match file1.md + expect(dir1!.filepath).toContain('file1.md'); + }); + }); }); diff --git a/src/domains/sitemap/SiteMap.ts b/src/domains/sitemap/SiteMap.ts index dda04e1..9728bd4 100644 --- a/src/domains/sitemap/SiteMap.ts +++ b/src/domains/sitemap/SiteMap.ts @@ -133,7 +133,12 @@ export class SiteMap { node.children.length === 1 && path.extname(node.children[0].filepath) === '' ) { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; @@ -146,6 +151,37 @@ export class SiteMap { this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten(): void { + const allNodes: TreeNode[] = []; + + // Collect all nodes except root + const collectNodes = (node: TreeNode) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + + // Start collection from root's children + this._root.children.forEach(collectNodes); + + // Clear existing children of root + this._root.children = []; + + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } + /** * TODO: Implement mkdocs.yaml sitemap parsing * diff --git a/src/domains/synchronization/destination.repository.ts b/src/domains/synchronization/destination.repository.ts index e7a34a0..781f955 100644 --- a/src/domains/synchronization/destination.repository.ts +++ b/src/domains/synchronization/destination.repository.ts @@ -82,4 +82,11 @@ export interface DestinationRepository { }: { databaseId: string; }) => Promise; + deletePagesInDatabaseByInternalId: ({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }) => Promise; } diff --git a/src/infrastructure/notion/notion.destination.ts b/src/infrastructure/notion/notion.destination.ts index 46d86a8..c60bb47 100644 --- a/src/infrastructure/notion/notion.destination.ts +++ b/src/infrastructure/notion/notion.destination.ts @@ -527,6 +527,42 @@ export class NotionDestinationRepository return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }): Promise { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + + if (objectIds.length > 1) { + this.logger.info( + `Multiple object IDs found with ${mkNotesInternalId}, deleting all objects` + ); + } + + await Promise.all( + objectIds.map(async (objectId) => + this.deleteObjectById({ + objectId, + }) + ) + ); + } + async deleteObjectById({ objectId }: { objectId: string }): Promise { await this.client.blocks.delete({ block_id: objectId }); } diff --git a/sync/index.js b/sync/index.js index b01126d..b4d0060 100644 --- a/sync/index.js +++ b/sync/index.js @@ -75002,7 +75002,7 @@ class MkNotes { /** * Preview the synchronization of a markdown file to Notion */ - async previewSynchronization({ inputPath, format, output, }) { + async previewSynchronization({ inputPath, format, output, flat = false, }) { const previewSynchronizationFeature = new domains_1.PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, }); @@ -75010,6 +75010,7 @@ class MkNotes { path: inputPath, }, { format, + flat, }); if (!output) { return result; @@ -75020,7 +75021,7 @@ class MkNotes { /** * Synchronize a markdown file to Notion */ - async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, }) { + async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, flat = false, }) { const synchronizeMarkdownToNotion = new domains_1.SynchronizeMarkdownToNotion({ logger: this.logger, destinationRepository: this.infrastructureInstances.notionDestination, @@ -75032,6 +75033,7 @@ class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } @@ -75852,7 +75854,7 @@ class PreviewSynchronization { constructor(params) { this.sourceRepository = params.sourceRepository; } - async execute(args, { format } = {}) { + async execute(args, { format, flat = false, } = {}) { // Check if the GitHub repository is accessible try { await this.sourceRepository.sourceIsAccessible(args); @@ -75880,6 +75882,9 @@ class PreviewSynchronization { } const filePaths = await this.sourceRepository.getFilePathList(args); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } return sitemapSerializer(siteMap); } } @@ -75909,7 +75914,7 @@ class SynchronizeMarkdownToNotion { this.logger = params.logger; } async execute(args) { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { notionParentPageUrl, cleanSync, lockPage, flat = false, ...others } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, }); @@ -75935,10 +75940,16 @@ class SynchronizeMarkdownToNotion { if (parentObjectType === 'unknown') { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error('Flat sync is only supported for database destinations. Pages do not support flat sync.'); + } try { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -75946,6 +75957,7 @@ class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); } @@ -76035,32 +76047,19 @@ class SynchronizeMarkdownToNotion { return newPage.pageId; } async cleanSyncDatabase({ databaseId, pageElement, }) { - const dataSourceId = await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); if (pageElement.mkNotesInternalId === undefined) { this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync'); return; } - const objectIds = await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId({ - dataSourceId, + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, mkNotesInternalId: pageElement.mkNotesInternalId, }); - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - if (objectIds.length > 1) { - this.logger.info(`Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects`); - } - await Promise.all(objectIds.map(async (objectId) => this.destinationRepository.deleteObjectById({ - objectId, - }))); } /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, }) { + async synchronizeChildNode({ childNode, parentObjectId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76070,10 +76069,22 @@ class SynchronizeMarkdownToNotion { if (childNode.children.length > 0) { pageElement.addElementToEnd(new elements_1.DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync for child node'); + } + else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentObjectId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectId, + parentObjectType, filePath, }); this.logger.info(`Created Notion page for file: ${filePath}`); @@ -76084,8 +76095,9 @@ class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, + cleanSync, }); } await this.lockPageIfNeeded(newPage.pageId, lockPage); @@ -76093,30 +76105,21 @@ class SynchronizeMarkdownToNotion { /** * Main orchestrator for synchronizing a tree node and its children */ - async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, }) { + async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; + const isFlatSync = flat && parentObjectType === 'database'; switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -76136,8 +76139,10 @@ class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { @@ -76148,6 +76153,36 @@ class SynchronizeMarkdownToNotion { } } } + async handleDatabaseSynchronization({ node, parentObjectId, parentObjectType, lockPage, cleanSync, isFlatSync, }) { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } getIsRootNode(node) { return node.parent === null && !['', undefined].includes(node.filepath); } @@ -76522,7 +76557,12 @@ class SiteMap { removeUselessNodesTree(node) { while (node.children.length === 1 && path.extname(node.children[0].filepath) === '') { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; } @@ -76533,6 +76573,32 @@ class SiteMap { this.removeUselessNodesTree(this._root); this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten() { + const allNodes = []; + // Collect all nodes except root + const collectNodes = (node) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + // Start collection from root's children + this._root.children.forEach(collectNodes); + // Clear existing children of root + this._root.children = []; + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } /** * TODO: Implement mkdocs.yaml sitemap parsing * @@ -79213,6 +79279,25 @@ class NotionDestinationRepository { }); return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ databaseId, mkNotesInternalId, }) { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + if (objectIds.length > 1) { + this.logger.info(`Multiple object IDs found with ${mkNotesInternalId}, deleting all objects`); + } + await Promise.all(objectIds.map(async (objectId) => this.deleteObjectById({ + objectId, + }))); + } async deleteObjectById({ objectId }) { await this.client.blocks.delete({ block_id: objectId }); }