From fec2955b5ee1ca886abff0d95808bf119dd3aaca Mon Sep 17 00:00:00 2001 From: Oliver Kaiser <2995870+oliverkaiser@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:46:09 +0100 Subject: [PATCH 1/4] Add --flat option for database syncs to sync larger folder structures into a DB and ensure each md file becomes a DB notion page. --- .../__fakes__/fakeDestination.repository.ts | 11 ++ src/MkNotes.ts | 6 + src/bin/cli/commands/preview.ts | 15 +- src/bin/cli/commands/sync.ts | 8 + .../features/previewSynchronization.test.ts | 29 +++ .../features/previewSynchronization.ts | 9 +- .../features/synchronizeMarkdownToNotion.ts | 147 +++++++++----- .../synchronizeMarkdownToNotion_flat.test.ts | 185 ++++++++++++++++++ src/domains/sitemap/SiteMap.test.ts | 38 ++++ src/domains/sitemap/SiteMap.ts | 38 +++- .../synchronization/destination.repository.ts | 7 + .../notion/notion.destination.ts | 36 ++++ 12 files changed, 478 insertions(+), 51 deletions(-) create mode 100644 src/domains/features/synchronizeMarkdownToNotion_flat.test.ts diff --git a/__tests__/__fakes__/fakeDestination.repository.ts b/__tests__/__fakes__/fakeDestination.repository.ts index 239f4e6..6aedf40 100644 --- a/__tests__/__fakes__/fakeDestination.repository.ts +++ b/__tests__/__fakes__/fakeDestination.repository.ts @@ -152,4 +152,15 @@ export class FakeDestinationRepository // no-op in fake repository for testing return Promise.resolve(); } + + async deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }): Promise { + // no-op in fake repository for testing + return Promise.resolve(); + } } diff --git a/src/MkNotes.ts b/src/MkNotes.ts index b1d014c..7e5dee1 100644 --- a/src/MkNotes.ts +++ b/src/MkNotes.ts @@ -48,10 +48,12 @@ export class MkNotes { inputPath, format, output, + flat = false, }: { inputPath: string; format: PreviewFormat; output?: string; + flat?: boolean; }): Promise { const previewSynchronizationFeature = new PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, @@ -63,6 +65,7 @@ export class MkNotes { }, { format, + flat, } ); @@ -83,11 +86,13 @@ export class MkNotes { parentNotionPageId, cleanSync = false, lockPage = false, + flat = false, }: { inputPath: string; parentNotionPageId: string; cleanSync?: boolean; lockPage?: boolean; + flat?: boolean; }): Promise { const synchronizeMarkdownToNotion = new SynchronizeMarkdownToNotion({ logger: this.logger, @@ -101,6 +106,7 @@ export class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } diff --git a/src/bin/cli/commands/preview.ts b/src/bin/cli/commands/preview.ts index 9c0de72..9852ec7 100644 --- a/src/bin/cli/commands/preview.ts +++ b/src/bin/cli/commands/preview.ts @@ -53,6 +53,11 @@ command.addOption( command.option('-o, --output ', 'Output file path'); +command.option( + '--flat', + 'Flat sync - If destination is a database, all files will be created as direct children of the database, not nested' +); + command.option('-v, --verbosity ', 'Verbosity level', 'error'); interface PreviewOptions { @@ -60,9 +65,16 @@ interface PreviewOptions { format: PreviewFormat; output?: string; verbosity?: string; + flat?: boolean; } command.action(async (opts: PreviewOptions) => { - const { input: directoryPath, format, output, verbosity = 'error' } = opts; + const { + input: directoryPath, + format, + output, + verbosity = 'error', + flat = false, + } = opts; if (!isValidVerbosity(verbosity)) { throw new Error(`Invalid verbosity: ${verbosity}`); @@ -77,6 +89,7 @@ command.action(async (opts: PreviewOptions) => { inputPath: directoryPath, format, output, + flat, }); // eslint-disable-next-line no-console diff --git a/src/bin/cli/commands/sync.ts b/src/bin/cli/commands/sync.ts index 00bde5f..d071fe9 100644 --- a/src/bin/cli/commands/sync.ts +++ b/src/bin/cli/commands/sync.ts @@ -34,6 +34,11 @@ command.option( command.option('-l, --lock', 'Lock the Notion page after syncing'); +command.option( + '-f, --flat', + 'Flat sync - If destination is a database, all files will be created as direct children of the database, not nested' +); + command.option('-v, --verbosity ', 'Verbosity level', 'error'); interface SyncOptions { input: string; @@ -42,6 +47,7 @@ interface SyncOptions { clean?: boolean; lock?: boolean; verbosity?: string; + flat?: boolean; } command.action(async (opts: SyncOptions) => { @@ -52,6 +58,7 @@ command.action(async (opts: SyncOptions) => { clean = false, lock = false, verbosity = 'error', + flat = false, } = opts; if (!isValidVerbosity(verbosity)) { @@ -68,6 +75,7 @@ command.action(async (opts: SyncOptions) => { parentNotionPageId: notionParentPageUrl, cleanSync: clean, lockPage: lock, + flat, }); // eslint-disable-next-line no-console diff --git a/src/domains/features/previewSynchronization.test.ts b/src/domains/features/previewSynchronization.test.ts index 5835616..7fa0078 100644 --- a/src/domains/features/previewSynchronization.test.ts +++ b/src/domains/features/previewSynchronization.test.ts @@ -91,5 +91,34 @@ describe('PreviewSynchronization', () => { expect(buildFromFilePathsSpy).toHaveBeenCalledWith(filePaths); }); + + it('should flatten the sitemap when flat option is true', async () => { + // Use a structure that prevents root collapsing of the directory + // root -> section, other.md + // section -> child1.md, child2.md + // section will consume child1.md + // child2.md will remain as child of section + const filePaths = ['section/child1.md', 'section/child2.md', 'other.md']; + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(filePaths); + + // When flattened, children should be at root level + const result = await previewSync.execute( + { path: 'test/path' }, + { format: 'json', flat: true } + ); + + const parsedResult = JSON.parse(result); + + const childrenNames = parsedResult.children.map((c: any) => c.name); + // 'section' node exists (having consumed child1.md) + expect(childrenNames).toContain('section'); + // 'child2.md' node exists (was nested, now flat) + expect(childrenNames).toContain('child2.md'); + // 'other.md' node exists + expect(childrenNames).toContain('other.md'); + expect(parsedResult.children.length).toBe(3); + }); }); }); diff --git a/src/domains/features/previewSynchronization.ts b/src/domains/features/previewSynchronization.ts index 7ee7aa4..304d96a 100644 --- a/src/domains/features/previewSynchronization.ts +++ b/src/domains/features/previewSynchronization.ts @@ -27,7 +27,10 @@ export class PreviewSynchronization { async execute( args: T, - { format }: { format?: PreviewFormat; output?: string } = {} + { + format, + flat = false, + }: { format?: PreviewFormat; output?: string; flat?: boolean } = {} ): Promise { // Check if the GitHub repository is accessible try { @@ -58,6 +61,10 @@ export class PreviewSynchronization { const siteMap = SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } + return sitemapSerializer(siteMap); } } diff --git a/src/domains/features/synchronizeMarkdownToNotion.ts b/src/domains/features/synchronizeMarkdownToNotion.ts index 2df2df7..994639e 100644 --- a/src/domains/features/synchronizeMarkdownToNotion.ts +++ b/src/domains/features/synchronizeMarkdownToNotion.ts @@ -29,6 +29,9 @@ export interface SynchronizeOptions { /** When true, lock the Notion page after syncing */ lockPage: boolean; + + /** When true, and destination is a database, create all files as direct children */ + flat?: boolean; } export class SynchronizeMarkdownToNotion { @@ -49,7 +52,13 @@ export class SynchronizeMarkdownToNotion { notionParentPageUrl: string; } & SynchronizeOptions ): Promise { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { + notionParentPageUrl, + cleanSync, + lockPage, + flat = false, + ...others + } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, @@ -91,6 +100,10 @@ export class SynchronizeMarkdownToNotion { const siteMap = SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } + // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -98,6 +111,7 @@ export class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); @@ -233,11 +247,6 @@ export class SynchronizeMarkdownToNotion { databaseId: string; pageElement: PageElement; }): Promise { - const dataSourceId = - await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); - if (pageElement.mkNotesInternalId === undefined) { this.logger.warn( 'mk-notes-internal-id is undefined, skipping clean sync' @@ -245,32 +254,10 @@ export class SynchronizeMarkdownToNotion { return; } - const objectIds = - await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId( - { - dataSourceId, - mkNotesInternalId: pageElement.mkNotesInternalId, - } - ); - - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - - if (objectIds.length > 1) { - this.logger.info( - `Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects` - ); - } - - await Promise.all( - objectIds.map(async (objectId) => - this.destinationRepository.deleteObjectById({ - objectId, - }) - ) - ); + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); } /** * Synchronizes a child node and its descendants recursively @@ -279,10 +266,14 @@ export class SynchronizeMarkdownToNotion { childNode, parentPageId, lockPage, + cleanSync, + parentObjectType = 'page', }: { childNode: TreeNode; parentPageId: string; lockPage: boolean; + cleanSync: boolean; + parentObjectType?: ObjectType; }): Promise { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); @@ -297,10 +288,24 @@ export class SynchronizeMarkdownToNotion { pageElement.addElementToEnd(new DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn( + 'mk-notes-internal-id is undefined, skipping clean sync for child node' + ); + } else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentPageId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } + const newPage = await this.destinationRepository.createPage({ pageElement, parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectType, filePath, }); @@ -316,6 +321,7 @@ export class SynchronizeMarkdownToNotion { childNode: grandChild, parentPageId: newPage.pageId, lockPage, + cleanSync, }); } @@ -331,35 +337,78 @@ export class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat = false, }: { node: TreeNode; parentObjectId: string; parentObjectType: ObjectType; lockPage: boolean; cleanSync: boolean; + flat?: boolean; }): Promise { let parentPageId: string = parentObjectId; + if (flat && parentObjectType !== 'database') { + this.logger.warn( + 'Flat option ignored because destination is not a database' + ); + } + + const isFlatSync = flat && parentObjectType === 'database'; + + // If flat sync is enabled, flatten the sitemap starting from this node + // Since we are passing the root node here usually, this affects the whole tree + // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap + // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. + // Since we don't have the SiteMap instance here, we can implement a helper or just assume + // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." + // But we don't have the siteMap instance here. + // Correction: We call synchronizeTreeNode with siteMap.root. + // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. + // BUT `execute` has the SiteMap instance! + // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." + // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. + // The `execute` method has the `SiteMap` instance. + // So I will modify `execute` instead to flatten the map. + + // Wait, I already modified `execute` but didn't add the flatten call there. + // I will add the flatten logic in `execute` in a separate tool call. + switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (isFlatSync) { + // In flat sync, if the root has content, we sync it to the DB. + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + // Children will also be synced to the DB (parentObjectId) + parentPageId = parentObjectId; } else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (this.getIsRootNode(node)) { + parentPageId = await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } else { + parentPageId = await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } } break; case 'page': @@ -383,6 +432,8 @@ export class SynchronizeMarkdownToNotion { childNode, parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { this.logger.error(`Failed to synchronize file: ${childNode.filepath}`, { diff --git a/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts new file mode 100644 index 0000000..261c123 --- /dev/null +++ b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts @@ -0,0 +1,185 @@ +import { FakeFileConverter } from '../../../__tests__/__fakes__/fakeConverter.repository'; +import { FakeDestinationRepository } from '../../../__tests__/__fakes__/fakeDestination.repository'; +import { FakeFile } from '../../../__tests__/__fakes__/fakeFile'; +import { fakeLogger } from '../../../__tests__/__fakes__/fakeLogger'; +import { FakeNotionPage } from '../../../__tests__/__fakes__/fakePage'; +import { FakeSourceRepository } from '../../../__tests__/__fakes__/fakeSource.repository'; +import { PageElement, TextElement } from '../elements'; +import { SynchronizeMarkdownToNotion } from './synchronizeMarkdownToNotion'; + +describe('SynchronizeMarkdownToNotion - Flat Sync', () => { + let synchronizer: SynchronizeMarkdownToNotion; + let sourceRepository: FakeSourceRepository; + let destinationRepository: FakeDestinationRepository; + let elementConverter: FakeFileConverter; + + const databaseId = 'database-id-12345678901234567890123456789012'; + const databaseUrl = `https://www.notion.so/workspace/${databaseId}`; + + beforeEach(() => { + sourceRepository = new FakeSourceRepository(); + destinationRepository = new FakeDestinationRepository(); + elementConverter = new FakeFileConverter({ + logger: fakeLogger, + htmlParser: {} as any, + markdownParser: {} as any, + }); + + synchronizer = new SynchronizeMarkdownToNotion({ + sourceRepository, + destinationRepository, + elementConverter, + logger: fakeLogger, + }); + + jest + .spyOn(destinationRepository, 'destinationIsAccessible') + .mockResolvedValue(true); + jest.spyOn(sourceRepository, 'sourceIsAccessible').mockResolvedValue(true); + jest + .spyOn(destinationRepository, 'getObjectType') + .mockResolvedValue('database'); + jest + .spyOn(destinationRepository, 'createPage') + .mockResolvedValue(new FakeNotionPage({ pageId: 'new-page-id' })); + jest + .spyOn(destinationRepository, 'getDataSourceIdFromDatabaseId') + .mockResolvedValue('datasource-id'); + jest + .spyOn(destinationRepository, 'deletePagesInDatabaseByInternalId') + .mockResolvedValue(undefined); + }); + + it('should create all files as direct children of the database when flat is true', async () => { + // Setup hierarchical file structure + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue([ + 'parent.md', + 'child/child.md', + 'child/grandchild/grandchild.md', + ]); + + const pageElement = new PageElement({ + title: 'Test', + content: [new TextElement({ text: '# Test' })], + mkNotesInternalId: 'test-id', + }); + + jest + .spyOn(sourceRepository, 'getFile') + .mockResolvedValue(new FakeFile({ content: '# Test' })); + jest + .spyOn(elementConverter, 'convertToElement') + .mockReturnValue(pageElement); + + const createPageSpy = jest.spyOn(destinationRepository, 'createPage'); + + await synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: false, + lockPage: false, + flat: true, + path: 'test', + }); + + // Should create 3 pages + expect(createPageSpy).toHaveBeenCalledTimes(3); + + // All pages should be created with parentObjectId = databaseId and parentObjectType = 'database' + const calls = createPageSpy.mock.calls; + + for (const call of calls) { + expect(call[0]).toMatchObject({ + parentObjectId: '12345678901234567890123456789012', + parentObjectType: 'database', + }); + } + }); + + it('should respect cleanSync by calling deletePagesInDatabaseByInternalId in flat mode', async () => { + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(['child.md']); + + const pageElement = new PageElement({ + title: 'Child', + content: [], + mkNotesInternalId: 'child-internal-id', + }); + + jest + .spyOn(sourceRepository, 'getFile') + .mockResolvedValue(new FakeFile({ content: '' })); + jest + .spyOn(elementConverter, 'convertToElement') + .mockReturnValue(pageElement); + + const deleteSpy = jest.spyOn( + destinationRepository, + 'deletePagesInDatabaseByInternalId' + ); + + await synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: true, + lockPage: false, + flat: true, + path: 'test', + }); + + expect(deleteSpy).toHaveBeenCalledWith({ + databaseId: '12345678901234567890123456789012', + mkNotesInternalId: 'child-internal-id', + }); + }); + + it('should ignore flat option if destination is not a database', async () => { + // Mock destination as a page instead of a database + jest.spyOn(destinationRepository, 'getObjectType').mockResolvedValue('page'); + jest + .spyOn(destinationRepository, 'appendToPage') + .mockResolvedValue(undefined); + + jest + .spyOn(sourceRepository, 'getFilePathList') + .mockResolvedValue(['parent.md', 'child/child.md']); + + const pageElement = new PageElement({ + title: 'Test', + content: [], + mkNotesInternalId: 'id', + }); + jest + .spyOn(elementConverter, 'convertToElement') + .mockReturnValue(pageElement); + jest + .spyOn(sourceRepository, 'getFile') + .mockResolvedValue(new FakeFile({ content: '' })); + + const createPageSpy = jest.spyOn(destinationRepository, 'createPage'); + + await synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: false, + lockPage: false, + flat: true, // Should be ignored + path: 'test', + }); + + // Should behave like hierarchical sync + // parent.md -> created as child of root page + // child/child.md -> created as child of root page + // It will create 2 pages because there is no index.md handling in this fake setup that merges them + + expect(createPageSpy).toHaveBeenCalledTimes(2); + expect(createPageSpy).toHaveBeenCalledWith( + expect.objectContaining({ + parentObjectType: 'page', + // parentObjectId should be the root page ID (databaseId in this mock setup acting as pageId) + parentObjectId: '12345678901234567890123456789012', + }) + ); + }); +}); + diff --git a/src/domains/sitemap/SiteMap.test.ts b/src/domains/sitemap/SiteMap.test.ts index ce401b6..e9181c8 100644 --- a/src/domains/sitemap/SiteMap.test.ts +++ b/src/domains/sitemap/SiteMap.test.ts @@ -134,5 +134,43 @@ describe('SiteMap', () => { it('should throw error for invalid JSON data', () => { expect(() => SiteMap.fromJSON({})).toThrow('Invalid data'); }); + it('should collapse directory structure if it has only one child (default behavior)', () => { + const filePaths = ['root/dir1/dir2/file.md']; + const sitemap = SiteMap.buildFromFilePaths(filePaths); + + // The tool collapses intermediate directories with single children + // root -> file.md + + expect(sitemap.root.children).toHaveLength(1); + const child = sitemap.root.children[0]; + expect(child.name).toBe('file.md'); + expect(child.children).toHaveLength(0); + }); + + it('should preserve directory structure if it has multiple children (though first file consumes parent)', () => { + const filePaths = ['dir1/file1.md', 'dir1/file2.md', 'other.md']; + const sitemap = SiteMap.buildFromFilePaths(filePaths); + + // root -> dir1, other.md + // dir1 has file1.md, file2.md + // BUT traverseAndUpdate applies "First Child Rule" to dir1 (since no index.md) + // dir1 consumes file1.md. dir1.filepath = file1.md path. + // dir1 children = [file2.md]. + + expect(sitemap.root.children).toHaveLength(2); + + const dir1 = sitemap.root.children.find(c => c.name === 'dir1'); + const other = sitemap.root.children.find(c => c.name === 'other.md'); + + expect(dir1).toBeDefined(); + expect(other).toBeDefined(); + + // Expect dir1 to have consumed file1.md and kept file2.md as child + expect(dir1!.children).toHaveLength(1); + expect(dir1!.children[0].name).toBe('file2.md'); + // dir1 filepath should match file1.md + expect(dir1!.filepath).toContain('file1.md'); + }); + }); }); diff --git a/src/domains/sitemap/SiteMap.ts b/src/domains/sitemap/SiteMap.ts index dda04e1..9728bd4 100644 --- a/src/domains/sitemap/SiteMap.ts +++ b/src/domains/sitemap/SiteMap.ts @@ -133,7 +133,12 @@ export class SiteMap { node.children.length === 1 && path.extname(node.children[0].filepath) === '' ) { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; @@ -146,6 +151,37 @@ export class SiteMap { this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten(): void { + const allNodes: TreeNode[] = []; + + // Collect all nodes except root + const collectNodes = (node: TreeNode) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + + // Start collection from root's children + this._root.children.forEach(collectNodes); + + // Clear existing children of root + this._root.children = []; + + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } + /** * TODO: Implement mkdocs.yaml sitemap parsing * diff --git a/src/domains/synchronization/destination.repository.ts b/src/domains/synchronization/destination.repository.ts index e7a34a0..781f955 100644 --- a/src/domains/synchronization/destination.repository.ts +++ b/src/domains/synchronization/destination.repository.ts @@ -82,4 +82,11 @@ export interface DestinationRepository { }: { databaseId: string; }) => Promise; + deletePagesInDatabaseByInternalId: ({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }) => Promise; } diff --git a/src/infrastructure/notion/notion.destination.ts b/src/infrastructure/notion/notion.destination.ts index 46d86a8..c60bb47 100644 --- a/src/infrastructure/notion/notion.destination.ts +++ b/src/infrastructure/notion/notion.destination.ts @@ -527,6 +527,42 @@ export class NotionDestinationRepository return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ + databaseId, + mkNotesInternalId, + }: { + databaseId: string; + mkNotesInternalId: string; + }): Promise { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + + if (objectIds.length > 1) { + this.logger.info( + `Multiple object IDs found with ${mkNotesInternalId}, deleting all objects` + ); + } + + await Promise.all( + objectIds.map(async (objectId) => + this.deleteObjectById({ + objectId, + }) + ) + ); + } + async deleteObjectById({ objectId }: { objectId: string }): Promise { await this.client.blocks.delete({ block_id: objectId }); } From 69851a9538492072fb584779b9d65a05c08926d3 Mon Sep 17 00:00:00 2001 From: Oliver Kaiser <2995870+oliverkaiser@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:46:56 +0100 Subject: [PATCH 2/4] add missed files --- preview/index.js | 173 ++++++++++++++++++++++++++++++++++++----------- sync/index.js | 173 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 270 insertions(+), 76 deletions(-) diff --git a/preview/index.js b/preview/index.js index fa906bb..d952019 100644 --- a/preview/index.js +++ b/preview/index.js @@ -75002,7 +75002,7 @@ class MkNotes { /** * Preview the synchronization of a markdown file to Notion */ - async previewSynchronization({ inputPath, format, output, }) { + async previewSynchronization({ inputPath, format, output, flat = false, }) { const previewSynchronizationFeature = new domains_1.PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, }); @@ -75010,6 +75010,7 @@ class MkNotes { path: inputPath, }, { format, + flat, }); if (!output) { return result; @@ -75020,7 +75021,7 @@ class MkNotes { /** * Synchronize a markdown file to Notion */ - async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, }) { + async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, flat = false, }) { const synchronizeMarkdownToNotion = new domains_1.SynchronizeMarkdownToNotion({ logger: this.logger, destinationRepository: this.infrastructureInstances.notionDestination, @@ -75032,6 +75033,7 @@ class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } @@ -75812,7 +75814,7 @@ class PreviewSynchronization { constructor(params) { this.sourceRepository = params.sourceRepository; } - async execute(args, { format } = {}) { + async execute(args, { format, flat = false, } = {}) { // Check if the GitHub repository is accessible try { await this.sourceRepository.sourceIsAccessible(args); @@ -75840,6 +75842,9 @@ class PreviewSynchronization { } const filePaths = await this.sourceRepository.getFilePathList(args); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } return sitemapSerializer(siteMap); } } @@ -75869,7 +75874,7 @@ class SynchronizeMarkdownToNotion { this.logger = params.logger; } async execute(args) { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { notionParentPageUrl, cleanSync, lockPage, flat = false, ...others } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, }); @@ -75899,6 +75904,9 @@ class SynchronizeMarkdownToNotion { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -75906,6 +75914,7 @@ class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); } @@ -75995,32 +76004,19 @@ class SynchronizeMarkdownToNotion { return newPage.pageId; } async cleanSyncDatabase({ databaseId, pageElement, }) { - const dataSourceId = await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); if (pageElement.mkNotesInternalId === undefined) { this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync'); return; } - const objectIds = await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId({ - dataSourceId, + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, mkNotesInternalId: pageElement.mkNotesInternalId, }); - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - if (objectIds.length > 1) { - this.logger.info(`Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects`); - } - await Promise.all(objectIds.map(async (objectId) => this.destinationRepository.deleteObjectById({ - objectId, - }))); } /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, }) { + async synchronizeChildNode({ childNode, parentPageId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76030,10 +76026,22 @@ class SynchronizeMarkdownToNotion { if (childNode.children.length > 0) { pageElement.addElementToEnd(new elements_1.DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync for child node'); + } + else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentPageId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } const newPage = await this.destinationRepository.createPage({ pageElement, parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectType, filePath, }); this.logger.info(`Created Notion page for file: ${filePath}`); @@ -76046,6 +76054,7 @@ class SynchronizeMarkdownToNotion { childNode: grandChild, parentPageId: newPage.pageId, lockPage, + cleanSync, }); } await this.lockPageIfNeeded(newPage.pageId, lockPage); @@ -76053,29 +76062,65 @@ class SynchronizeMarkdownToNotion { /** * Main orchestrator for synchronizing a tree node and its children */ - async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, }) { + async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; + if (flat && parentObjectType !== 'database') { + this.logger.warn('Flat option ignored because destination is not a database'); + } + const isFlatSync = flat && parentObjectType === 'database'; + // If flat sync is enabled, flatten the sitemap starting from this node + // Since we are passing the root node here usually, this affects the whole tree + // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap + // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. + // Since we don't have the SiteMap instance here, we can implement a helper or just assume + // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." + // But we don't have the siteMap instance here. + // Correction: We call synchronizeTreeNode with siteMap.root. + // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. + // BUT `execute` has the SiteMap instance! + // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." + // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. + // The `execute` method has the `SiteMap` instance. + // So I will modify `execute` instead to flatten the map. + // Wait, I already modified `execute` but didn't add the flatten call there. + // I will add the flatten logic in `execute` in a separate tool call. switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (isFlatSync) { + // In flat sync, if the root has content, we sync it to the DB. + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + // Children will also be synced to the DB (parentObjectId) + parentPageId = parentObjectId; } else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (this.getIsRootNode(node)) { + parentPageId = await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + else { + parentPageId = await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } } break; case 'page': @@ -76098,6 +76143,8 @@ class SynchronizeMarkdownToNotion { childNode, parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { @@ -76482,7 +76529,12 @@ class SiteMap { removeUselessNodesTree(node) { while (node.children.length === 1 && path.extname(node.children[0].filepath) === '') { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; } @@ -76493,6 +76545,32 @@ class SiteMap { this.removeUselessNodesTree(this._root); this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten() { + const allNodes = []; + // Collect all nodes except root + const collectNodes = (node) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + // Start collection from root's children + this._root.children.forEach(collectNodes); + // Clear existing children of root + this._root.children = []; + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } /** * TODO: Implement mkdocs.yaml sitemap parsing * @@ -79173,6 +79251,25 @@ class NotionDestinationRepository { }); return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ databaseId, mkNotesInternalId, }) { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + if (objectIds.length > 1) { + this.logger.info(`Multiple object IDs found with ${mkNotesInternalId}, deleting all objects`); + } + await Promise.all(objectIds.map(async (objectId) => this.deleteObjectById({ + objectId, + }))); + } async deleteObjectById({ objectId }) { await this.client.blocks.delete({ block_id: objectId }); } diff --git a/sync/index.js b/sync/index.js index b01126d..83a8b71 100644 --- a/sync/index.js +++ b/sync/index.js @@ -75002,7 +75002,7 @@ class MkNotes { /** * Preview the synchronization of a markdown file to Notion */ - async previewSynchronization({ inputPath, format, output, }) { + async previewSynchronization({ inputPath, format, output, flat = false, }) { const previewSynchronizationFeature = new domains_1.PreviewSynchronization({ sourceRepository: this.infrastructureInstances.fileSystemSource, }); @@ -75010,6 +75010,7 @@ class MkNotes { path: inputPath, }, { format, + flat, }); if (!output) { return result; @@ -75020,7 +75021,7 @@ class MkNotes { /** * Synchronize a markdown file to Notion */ - async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, }) { + async synchronizeMarkdownToNotionFromFileSystem({ inputPath, parentNotionPageId, cleanSync = false, lockPage = false, flat = false, }) { const synchronizeMarkdownToNotion = new domains_1.SynchronizeMarkdownToNotion({ logger: this.logger, destinationRepository: this.infrastructureInstances.notionDestination, @@ -75032,6 +75033,7 @@ class MkNotes { notionParentPageUrl: parentNotionPageId, cleanSync, lockPage, + flat, }); } } @@ -75852,7 +75854,7 @@ class PreviewSynchronization { constructor(params) { this.sourceRepository = params.sourceRepository; } - async execute(args, { format } = {}) { + async execute(args, { format, flat = false, } = {}) { // Check if the GitHub repository is accessible try { await this.sourceRepository.sourceIsAccessible(args); @@ -75880,6 +75882,9 @@ class PreviewSynchronization { } const filePaths = await this.sourceRepository.getFilePathList(args); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat) { + siteMap.flatten(); + } return sitemapSerializer(siteMap); } } @@ -75909,7 +75914,7 @@ class SynchronizeMarkdownToNotion { this.logger = params.logger; } async execute(args) { - const { notionParentPageUrl, cleanSync, lockPage, ...others } = args; + const { notionParentPageUrl, cleanSync, lockPage, flat = false, ...others } = args; const notionObjectId = this.destinationRepository.getObjectIdFromObjectUrl({ objectUrl: notionParentPageUrl, }); @@ -75939,6 +75944,9 @@ class SynchronizeMarkdownToNotion { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); const siteMap = sitemap_1.SiteMap.buildFromFilePaths(filePaths); + if (flat && parentObjectType === 'database') { + siteMap.flatten(); + } // Traverse the SiteMap and synchronize files await this.synchronizeTreeNode({ node: siteMap.root, @@ -75946,6 +75954,7 @@ class SynchronizeMarkdownToNotion { parentObjectType, lockPage, cleanSync, + flat, }); this.logger.info('Synchronization process completed successfully'); } @@ -76035,32 +76044,19 @@ class SynchronizeMarkdownToNotion { return newPage.pageId; } async cleanSyncDatabase({ databaseId, pageElement, }) { - const dataSourceId = await this.destinationRepository.getDataSourceIdFromDatabaseId({ - databaseId, - }); if (pageElement.mkNotesInternalId === undefined) { this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync'); return; } - const objectIds = await this.destinationRepository.getObjectIdInDatabaseByMkNotesInternalId({ - dataSourceId, + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId, mkNotesInternalId: pageElement.mkNotesInternalId, }); - if (objectIds.length === 0) { - this.logger.warn('No object IDs found, skipping clean sync'); - return; - } - if (objectIds.length > 1) { - this.logger.info(`Multiple object IDs found with ${pageElement.mkNotesInternalId}, deleting all objects`); - } - await Promise.all(objectIds.map(async (objectId) => this.destinationRepository.deleteObjectById({ - objectId, - }))); } /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, }) { + async synchronizeChildNode({ childNode, parentPageId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76070,10 +76066,22 @@ class SynchronizeMarkdownToNotion { if (childNode.children.length > 0) { pageElement.addElementToEnd(new elements_1.DividerElement()); } + // If parent is a database (e.g. in flat sync), we need to clean up previous version of this specific page + if (parentObjectType === 'database' && cleanSync) { + if (!pageElement.mkNotesInternalId) { + this.logger.warn('mk-notes-internal-id is undefined, skipping clean sync for child node'); + } + else { + await this.destinationRepository.deletePagesInDatabaseByInternalId({ + databaseId: parentPageId, + mkNotesInternalId: pageElement.mkNotesInternalId, + }); + } + } const newPage = await this.destinationRepository.createPage({ pageElement, parentObjectId: parentPageId, - parentObjectType: 'page', + parentObjectType, filePath, }); this.logger.info(`Created Notion page for file: ${filePath}`); @@ -76086,6 +76094,7 @@ class SynchronizeMarkdownToNotion { childNode: grandChild, parentPageId: newPage.pageId, lockPage, + cleanSync, }); } await this.lockPageIfNeeded(newPage.pageId, lockPage); @@ -76093,29 +76102,65 @@ class SynchronizeMarkdownToNotion { /** * Main orchestrator for synchronizing a tree node and its children */ - async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, }) { + async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; + if (flat && parentObjectType !== 'database') { + this.logger.warn('Flat option ignored because destination is not a database'); + } + const isFlatSync = flat && parentObjectType === 'database'; + // If flat sync is enabled, flatten the sitemap starting from this node + // Since we are passing the root node here usually, this affects the whole tree + // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap + // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. + // Since we don't have the SiteMap instance here, we can implement a helper or just assume + // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." + // But we don't have the siteMap instance here. + // Correction: We call synchronizeTreeNode with siteMap.root. + // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. + // BUT `execute` has the SiteMap instance! + // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." + // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. + // The `execute` method has the `SiteMap` instance. + // So I will modify `execute` instead to flatten the map. + // Wait, I already modified `execute` but didn't add the flatten call there. + // I will add the flatten logic in `execute` in a separate tool call. switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (isFlatSync) { + // In flat sync, if the root has content, we sync it to the DB. + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + // Children will also be synced to the DB (parentObjectId) + parentPageId = parentObjectId; } else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); + if (this.getIsRootNode(node)) { + parentPageId = await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + else { + parentPageId = await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } } break; case 'page': @@ -76138,6 +76183,8 @@ class SynchronizeMarkdownToNotion { childNode, parentPageId, lockPage, + cleanSync, + parentObjectType: isFlatSync ? 'database' : 'page', }); } catch (error) { @@ -76522,7 +76569,12 @@ class SiteMap { removeUselessNodesTree(node) { while (node.children.length === 1 && path.extname(node.children[0].filepath) === '') { - node.children = node.children[0].children; + const [child] = node.children; + node.children = child.children; + // Fix parent pointers for the adopted children + node.children.forEach((grandChild) => { + grandChild.parent = node; + }); } return node; } @@ -76533,6 +76585,32 @@ class SiteMap { this.removeUselessNodesTree(this._root); this.traverseAndUpdate(this._root); } + /** + * Flattens the sitemap structure so all files are direct children of the root. + * This is useful for flat synchronization mode. + */ + flatten() { + const allNodes = []; + // Collect all nodes except root + const collectNodes = (node) => { + // We don't include the current node if it's the root + if (node !== this._root) { + allNodes.push(node); + } + node.children.forEach(collectNodes); + }; + // Start collection from root's children + this._root.children.forEach(collectNodes); + // Clear existing children of root + this._root.children = []; + // Reassign all collected nodes as direct children of root + // And clear their children since they are now flat + allNodes.forEach((node) => { + node.parent = this._root; + node.children = []; + this._root.children.push(node); + }); + } /** * TODO: Implement mkdocs.yaml sitemap parsing * @@ -79213,6 +79291,25 @@ class NotionDestinationRepository { }); return items.results.map((item) => item.id); } + async deletePagesInDatabaseByInternalId({ databaseId, mkNotesInternalId, }) { + const dataSourceId = await this.getDataSourceIdFromDatabaseId({ + databaseId, + }); + const objectIds = await this.getObjectIdInDatabaseByMkNotesInternalId({ + dataSourceId, + mkNotesInternalId, + }); + if (objectIds.length === 0) { + this.logger.warn('No object IDs found, skipping clean sync'); + return; + } + if (objectIds.length > 1) { + this.logger.info(`Multiple object IDs found with ${mkNotesInternalId}, deleting all objects`); + } + await Promise.all(objectIds.map(async (objectId) => this.deleteObjectById({ + objectId, + }))); + } async deleteObjectById({ objectId }) { await this.client.blocks.delete({ block_id: objectId }); } From 0c99828b5d835bc8dbf21259b2611c223d58aa59 Mon Sep 17 00:00:00 2001 From: Oliver Kaiser <2995870+oliverkaiser@users.noreply.github.com> Date: Wed, 3 Dec 2025 07:12:39 +0000 Subject: [PATCH 3/4] apply PR feedback --- docs/content/docs/cli/guides/cli-commands.mdx | 27 ++++ .../content/docs/cli/guides/database-sync.mdx | 39 ++++++ .../features/synchronizeMarkdownToNotion.ts | 129 +++++++++--------- .../synchronizeMarkdownToNotion_flat.test.ts | 44 ++---- 4 files changed, 143 insertions(+), 96 deletions(-) diff --git a/docs/content/docs/cli/guides/cli-commands.mdx b/docs/content/docs/cli/guides/cli-commands.mdx index 99389f9..0346db6 100644 --- a/docs/content/docs/cli/guides/cli-commands.mdx +++ b/docs/content/docs/cli/guides/cli-commands.mdx @@ -48,6 +48,8 @@ mk-notes sync -i -d -k - `-l, --lock`: Lock the Notion page after syncing to prevent further editing. This is useful when you want to preserve the synchronized content and prevent accidental modifications. +- `-f, --flat`: **Database destination only** - Flattens the directory structure so all markdown files are created as direct children of the database, not nested as child pages. This option is only available when the destination is a Notion database. If used with a page destination, an error will be thrown. + ### Destination Types Mk Notes supports two types of destinations: @@ -192,6 +194,31 @@ This command will: 2. Delete those existing pages 3. Create new pages as database items with the content from your markdown files +#### Flat Sync to a Database + +```bash +mk-notes sync \ + --input ./my-docs \ + --destination https://notion.so/myworkspace/database-123456 \ + --notion-api-key secret_abc123... \ + --flat +``` + +The `--flat` option creates all markdown files as direct children of the database, ignoring the directory structure. This is useful when you want a flat list of all documents in your database without nested page hierarchies. + +This command will: + +1. Read all markdown files in the directory +2. Flatten the directory structure +3. Create all files as direct database items (no nested pages) +4. Each file becomes a separate database entry + + + +The `--flat` option only works with database destinations. If you try to use it with a page destination, MK Notes will throw an error. + + + ## `preview-sync` The `preview-sync` command lets you preview how your markdown files will be organized in Notion before actually performing the synchronization. This is useful for verifying the structure before making any changes. diff --git a/docs/content/docs/cli/guides/database-sync.mdx b/docs/content/docs/cli/guides/database-sync.mdx index 0651637..62324aa 100644 --- a/docs/content/docs/cli/guides/database-sync.mdx +++ b/docs/content/docs/cli/guides/database-sync.mdx @@ -134,6 +134,45 @@ Clean sync only works for pages that have an `id` defined in their frontmatter. +### Flat Sync + +By default, MK Notes preserves your directory structure when syncing to a database, creating nested pages for subdirectories. The `--flat` option flattens this structure, creating all markdown files as direct children of the database: + +```bash +mk-notes sync \ + --input ./docs \ + --destination https://notion.so/myworkspace/database-123456 \ + --notion-api-key secret_abc123... \ + --flat +``` + +**When to use flat sync:** + +- You want all documents at the same level in your database +- You prefer to organize using database views and filters rather than page hierarchies +- Your directory structure is just for file organization, not content hierarchy + +**Example:** + +With a directory structure like: +``` +docs/ + guides/ + getting-started.md + advanced.md + api/ + authentication.md +``` + +- **Without `--flat`**: Creates nested pages (guides → getting-started, advanced) +- **With `--flat`**: Creates all four files as direct database entries + + + +The `--flat` option only works with database destinations. Using it with a page destination will result in an error. + + + --- ## Adding Custom Database Properties diff --git a/src/domains/features/synchronizeMarkdownToNotion.ts b/src/domains/features/synchronizeMarkdownToNotion.ts index 994639e..acf5043 100644 --- a/src/domains/features/synchronizeMarkdownToNotion.ts +++ b/src/domains/features/synchronizeMarkdownToNotion.ts @@ -91,6 +91,12 @@ export class SynchronizeMarkdownToNotion { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error( + 'Flat sync is only supported for database destinations. Pages do not support flat sync.' + ); + } + try { this.logger.info('Starting synchronization process'); @@ -264,13 +270,13 @@ export class SynchronizeMarkdownToNotion { */ private async synchronizeChildNode({ childNode, - parentPageId, + parentObjectId, lockPage, cleanSync, parentObjectType = 'page', }: { childNode: TreeNode; - parentPageId: string; + parentObjectId: string; lockPage: boolean; cleanSync: boolean; parentObjectType?: ObjectType; @@ -296,7 +302,7 @@ export class SynchronizeMarkdownToNotion { ); } else { await this.destinationRepository.deletePagesInDatabaseByInternalId({ - databaseId: parentPageId, + databaseId: parentObjectId, mkNotesInternalId: pageElement.mkNotesInternalId, }); } @@ -304,7 +310,7 @@ export class SynchronizeMarkdownToNotion { const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, + parentObjectId, parentObjectType, filePath, }); @@ -319,7 +325,7 @@ export class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, cleanSync, }); @@ -348,68 +354,20 @@ export class SynchronizeMarkdownToNotion { }): Promise { let parentPageId: string = parentObjectId; - if (flat && parentObjectType !== 'database') { - this.logger.warn( - 'Flat option ignored because destination is not a database' - ); - } - const isFlatSync = flat && parentObjectType === 'database'; - // If flat sync is enabled, flatten the sitemap starting from this node - // Since we are passing the root node here usually, this affects the whole tree - // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap - // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. - // Since we don't have the SiteMap instance here, we can implement a helper or just assume - // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." - // But we don't have the siteMap instance here. - // Correction: We call synchronizeTreeNode with siteMap.root. - // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. - // BUT `execute` has the SiteMap instance! - // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." - // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. - // The `execute` method has the `SiteMap` instance. - // So I will modify `execute` instead to flatten the map. - - // Wait, I already modified `execute` but didn't add the flatten call there. - // I will add the flatten logic in `execute` in a separate tool call. - switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (isFlatSync) { - // In flat sync, if the root has content, we sync it to the DB. - if (this.getIsRootNode(node)) { - await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - // Children will also be synced to the DB (parentObjectId) - parentPageId = parentObjectId; - } else { - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -430,7 +388,7 @@ export class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, cleanSync, parentObjectType: isFlatSync ? 'database' : 'page', @@ -444,6 +402,53 @@ export class SynchronizeMarkdownToNotion { } } + private async handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }: { + node: TreeNode; + parentObjectId: string; + parentObjectType: ObjectType; + lockPage: boolean; + cleanSync: boolean; + isFlatSync: boolean; + }): Promise { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + private getIsRootNode(node: TreeNode): boolean { return node.parent === null && !['', undefined].includes(node.filepath); } diff --git a/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts index 261c123..4b6199d 100644 --- a/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts +++ b/src/domains/features/synchronizeMarkdownToNotion_flat.test.ts @@ -134,7 +134,7 @@ describe('SynchronizeMarkdownToNotion - Flat Sync', () => { }); }); - it('should ignore flat option if destination is not a database', async () => { + it('should throw error if flat option is used with page destination', async () => { // Mock destination as a page instead of a database jest.spyOn(destinationRepository, 'getObjectType').mockResolvedValue('page'); jest @@ -145,40 +145,16 @@ describe('SynchronizeMarkdownToNotion - Flat Sync', () => { .spyOn(sourceRepository, 'getFilePathList') .mockResolvedValue(['parent.md', 'child/child.md']); - const pageElement = new PageElement({ - title: 'Test', - content: [], - mkNotesInternalId: 'id', - }); - jest - .spyOn(elementConverter, 'convertToElement') - .mockReturnValue(pageElement); - jest - .spyOn(sourceRepository, 'getFile') - .mockResolvedValue(new FakeFile({ content: '' })); - - const createPageSpy = jest.spyOn(destinationRepository, 'createPage'); - - await synchronizer.execute({ - notionParentPageUrl: databaseUrl, - cleanSync: false, - lockPage: false, - flat: true, // Should be ignored - path: 'test', - }); - - // Should behave like hierarchical sync - // parent.md -> created as child of root page - // child/child.md -> created as child of root page - // It will create 2 pages because there is no index.md handling in this fake setup that merges them - - expect(createPageSpy).toHaveBeenCalledTimes(2); - expect(createPageSpy).toHaveBeenCalledWith( - expect.objectContaining({ - parentObjectType: 'page', - // parentObjectId should be the root page ID (databaseId in this mock setup acting as pageId) - parentObjectId: '12345678901234567890123456789012', + await expect( + synchronizer.execute({ + notionParentPageUrl: databaseUrl, + cleanSync: false, + lockPage: false, + flat: true, + path: 'test', }) + ).rejects.toThrow( + 'Flat sync is only supported for database destinations. Pages do not support flat sync.' ); }); }); From b1d9d5c281f2115428a24360a89d916bcf1a6505 Mon Sep 17 00:00:00 2001 From: Oliver Kaiser <2995870+oliverkaiser@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:48:33 +0000 Subject: [PATCH 4/4] add missed files --- preview/index.js | 104 +++++++++++++++++++++-------------------------- sync/index.js | 104 +++++++++++++++++++++-------------------------- 2 files changed, 92 insertions(+), 116 deletions(-) diff --git a/preview/index.js b/preview/index.js index d952019..a9e326c 100644 --- a/preview/index.js +++ b/preview/index.js @@ -75900,6 +75900,9 @@ class SynchronizeMarkdownToNotion { if (parentObjectType === 'unknown') { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error('Flat sync is only supported for database destinations. Pages do not support flat sync.'); + } try { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); @@ -76016,7 +76019,7 @@ class SynchronizeMarkdownToNotion { /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, cleanSync, parentObjectType = 'page', }) { + async synchronizeChildNode({ childNode, parentObjectId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76033,14 +76036,14 @@ class SynchronizeMarkdownToNotion { } else { await this.destinationRepository.deletePagesInDatabaseByInternalId({ - databaseId: parentPageId, + databaseId: parentObjectId, mkNotesInternalId: pageElement.mkNotesInternalId, }); } } const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, + parentObjectId, parentObjectType, filePath, }); @@ -76052,7 +76055,7 @@ class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, cleanSync, }); @@ -76064,64 +76067,19 @@ class SynchronizeMarkdownToNotion { */ async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; - if (flat && parentObjectType !== 'database') { - this.logger.warn('Flat option ignored because destination is not a database'); - } const isFlatSync = flat && parentObjectType === 'database'; - // If flat sync is enabled, flatten the sitemap starting from this node - // Since we are passing the root node here usually, this affects the whole tree - // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap - // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. - // Since we don't have the SiteMap instance here, we can implement a helper or just assume - // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." - // But we don't have the siteMap instance here. - // Correction: We call synchronizeTreeNode with siteMap.root. - // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. - // BUT `execute` has the SiteMap instance! - // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." - // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. - // The `execute` method has the `SiteMap` instance. - // So I will modify `execute` instead to flatten the map. - // Wait, I already modified `execute` but didn't add the flatten call there. - // I will add the flatten logic in `execute` in a separate tool call. switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (isFlatSync) { - // In flat sync, if the root has content, we sync it to the DB. - if (this.getIsRootNode(node)) { - await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - // Children will also be synced to the DB (parentObjectId) - parentPageId = parentObjectId; - } - else { - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -76141,7 +76099,7 @@ class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, cleanSync, parentObjectType: isFlatSync ? 'database' : 'page', @@ -76155,6 +76113,36 @@ class SynchronizeMarkdownToNotion { } } } + async handleDatabaseSynchronization({ node, parentObjectId, parentObjectType, lockPage, cleanSync, isFlatSync, }) { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } getIsRootNode(node) { return node.parent === null && !['', undefined].includes(node.filepath); } diff --git a/sync/index.js b/sync/index.js index 83a8b71..b4d0060 100644 --- a/sync/index.js +++ b/sync/index.js @@ -75940,6 +75940,9 @@ class SynchronizeMarkdownToNotion { if (parentObjectType === 'unknown') { throw new Error('Parent object type is unknown'); } + if (flat && parentObjectType === 'page') { + throw new Error('Flat sync is only supported for database destinations. Pages do not support flat sync.'); + } try { this.logger.info('Starting synchronization process'); const filePaths = await this.sourceRepository.getFilePathList(others); @@ -76056,7 +76059,7 @@ class SynchronizeMarkdownToNotion { /** * Synchronizes a child node and its descendants recursively */ - async synchronizeChildNode({ childNode, parentPageId, lockPage, cleanSync, parentObjectType = 'page', }) { + async synchronizeChildNode({ childNode, parentObjectId, lockPage, cleanSync, parentObjectType = 'page', }) { const filePath = childNode.filepath; this.logger.info(`Processing file: ${filePath}`); const pageElement = await this.fetchAndConvertToPageElement(filePath); @@ -76073,14 +76076,14 @@ class SynchronizeMarkdownToNotion { } else { await this.destinationRepository.deletePagesInDatabaseByInternalId({ - databaseId: parentPageId, + databaseId: parentObjectId, mkNotesInternalId: pageElement.mkNotesInternalId, }); } } const newPage = await this.destinationRepository.createPage({ pageElement, - parentObjectId: parentPageId, + parentObjectId, parentObjectType, filePath, }); @@ -76092,7 +76095,7 @@ class SynchronizeMarkdownToNotion { for (const grandChild of childNode.children) { await this.synchronizeChildNode({ childNode: grandChild, - parentPageId: newPage.pageId, + parentObjectId: newPage.pageId, lockPage, cleanSync, }); @@ -76104,64 +76107,19 @@ class SynchronizeMarkdownToNotion { */ async synchronizeTreeNode({ node, parentObjectId, parentObjectType, lockPage, cleanSync, flat = false, }) { let parentPageId = parentObjectId; - if (flat && parentObjectType !== 'database') { - this.logger.warn('Flat option ignored because destination is not a database'); - } const isFlatSync = flat && parentObjectType === 'database'; - // If flat sync is enabled, flatten the sitemap starting from this node - // Since we are passing the root node here usually, this affects the whole tree - // But we rely on SiteMap.flatten() which works on the whole structure anyway if we had access to SiteMap - // Here we only have the root node. But wait, SiteMap.flatten() modifies the tree structure in place. - // Since we don't have the SiteMap instance here, we can implement a helper or just assume - // the caller has done it? No, the plan says "If flat is true, call siteMap.flatten() immediately." - // But we don't have the siteMap instance here. - // Correction: We call synchronizeTreeNode with siteMap.root. - // We should probably move the flatten call to `execute` BEFORE calling synchronizeTreeNode. - // BUT `execute` has the SiteMap instance! - // Let's revert to the plan: "In synchronizeTreeNode: If flat is true, call siteMap.flatten() immediately." - // Ah, `synchronizeTreeNode` receives a `node`. It doesn't have the `SiteMap` instance. - // The `execute` method has the `SiteMap` instance. - // So I will modify `execute` instead to flatten the map. - // Wait, I already modified `execute` but didn't add the flatten call there. - // I will add the flatten logic in `execute` in a separate tool call. switch (parentObjectType) { case 'unknown': throw new Error('Parent object type is unknown'); case 'database': - if (isFlatSync) { - // In flat sync, if the root has content, we sync it to the DB. - if (this.getIsRootNode(node)) { - await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - // Children will also be synced to the DB (parentObjectId) - parentPageId = parentObjectId; - } - else { - if (this.getIsRootNode(node)) { - parentPageId = await this.synchronizeRootNode({ - node, - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - else { - parentPageId = await this.synchronizeRootNode({ - node: node.children[0], - parentObjectId, - parentObjectType, - lockPage, - cleanSync, - }); - } - } + parentPageId = await this.handleDatabaseSynchronization({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + isFlatSync, + }); break; case 'page': if (this.getIsRootNode(node)) { @@ -76181,7 +76139,7 @@ class SynchronizeMarkdownToNotion { try { await this.synchronizeChildNode({ childNode, - parentPageId, + parentObjectId: parentPageId, lockPage, cleanSync, parentObjectType: isFlatSync ? 'database' : 'page', @@ -76195,6 +76153,36 @@ class SynchronizeMarkdownToNotion { } } } + async handleDatabaseSynchronization({ node, parentObjectId, parentObjectType, lockPage, cleanSync, isFlatSync, }) { + if (isFlatSync) { + if (this.getIsRootNode(node)) { + await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return parentObjectId; + } + if (this.getIsRootNode(node)) { + return await this.synchronizeRootNode({ + node, + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } + return await this.synchronizeRootNode({ + node: node.children[0], + parentObjectId, + parentObjectType, + lockPage, + cleanSync, + }); + } getIsRootNode(node) { return node.parent === null && !['', undefined].includes(node.filepath); }