Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions apps/api/src/lib/import/adapters/mock.adapter.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { Buffer } from 'node:buffer';
import type { IPageScraperAdapter, ScrapedPageData } from '@minimalblock/core';

function normalizeDomain(url: URL): string {
Expand All @@ -7,7 +6,7 @@ function normalizeDomain(url: URL): string {

function buildMockDataUrl(label: string, color: string): string {
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="1200"><rect width="100%" height="100%" fill="${color}"/><text x="50%" y="50%" font-family="Arial" font-size="84" text-anchor="middle" fill="#ffffff">${label}</text></svg>`;
return `data:image/svg+xml;base64,${Buffer.from(svg).toString('base64')}`;
return `data:image/svg+xml;base64,${btoa(svg)}`;
}

export class MockAdapter implements IPageScraperAdapter {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { Buffer } from 'node:buffer';
import type { ImportedImageCandidate, ProductImportData } from '@minimalblock/core';
import { GeminiImageClassifier, ImageDeduplicationService } from '@minimalblock/ai';

Expand Down Expand Up @@ -33,15 +32,15 @@ export class ImageIntelligencePipeline {
headers: { 'user-agent': 'MinimalBlockBot/1.0', accept: 'image/*' },
});
if (!res.ok) return null;
return Buffer.from(await res.arrayBuffer());
return new Uint8Array(await res.arrayBuffer());
} catch {
return null;
}
}),
);

// Perceptual deduplication
const hashes = buffers.map((buf) => buf ? this.deduplicator.computeHash(new Uint8Array(buf)) : '0000000000000000');
const hashes = buffers.map((buf) => buf ? this.deduplicator.computeHash(buf) : '0000000000000000');
const duplicateIndexes = new Set(this.deduplicator.findDuplicates(hashes));

// Build base64 images for Gemini classification (only non-failed uploads)
Expand All @@ -50,7 +49,7 @@ export class ImageIntelligencePipeline {
const buf = buffers[i];
if (buf && candidates[i].mimeType && candidates[i].mimeType !== 'image/svg+xml') {
geminiImages.push({
base64: buf.toString('base64'),
base64: btoa(String.fromCharCode(...buf)),
mimeType: candidates[i].mimeType!,
originalIndex: i,
});
Expand Down
10 changes: 6 additions & 4 deletions apps/api/src/lib/import/pipeline/image-upload.pipeline.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { Buffer } from 'node:buffer';
import { generateId, type ImportedImageCandidate, type MediaAssetType } from '@minimalblock/core';
import type { ScrapedImageCandidate } from '@minimalblock/core';
import type { SupabaseClient } from '@supabase/supabase-js';
Expand All @@ -17,11 +16,14 @@ function mimeExtension(mimeType: string): string {
}
}

function decodeDataUrl(raw: string): { mimeType: string; buffer: Buffer } | null {
function decodeDataUrl(raw: string): { mimeType: string; buffer: Uint8Array } | null {
if (!raw.startsWith('data:')) return null;
const match = raw.match(/^data:([^;]+);base64,(.+)$/);
if (!match) return null;
return { mimeType: match[1], buffer: Buffer.from(match[2], 'base64') };
const binaryStr = atob(match[2]);
const bytes = new Uint8Array(binaryStr.length);
for (let i = 0; i < binaryStr.length; i++) bytes[i] = binaryStr.charCodeAt(i);
return { mimeType: match[1], buffer: bytes };
}

export type UploadedImportImage = ImportedImageCandidate;
Expand Down Expand Up @@ -76,7 +78,7 @@ export class ImageUploadPipeline {
? mimeType
: 'image/jpeg';

const bytes = decoded?.buffer ?? Buffer.from(await response!.arrayBuffer());
const bytes = decoded?.buffer ?? new Uint8Array(await response!.arrayBuffer());
const fileName = `${Date.now()}-${slugify(image.title ?? image.alt ?? `import-${image.ordinal}`)}.${mimeExtension(normalizedMimeType)}`;
const storageKey = `${this.ownerId}/imports/${fileName}`;

Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ export default {
async fetch(request: Request, env: ApiEnv): Promise<Response> {
return handleRequest(request, env);
},
} satisfies ExportedHandler<ApiEnv>;
};
4 changes: 4 additions & 0 deletions apps/docs/src/.vitepress/config.mts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ import { defineConfig } from 'vitepress'

export default defineConfig({
srcDir: '../../../docs',
head: [
['meta', { name: 'robots', content: 'noindex, nofollow, noarchive, noimageindex' }],
['meta', { name: 'googlebot', content: 'noindex, nofollow' }],
],

locales: {
en: {
Expand Down
20 changes: 20 additions & 0 deletions apps/docs/src/.vitepress/public/robots.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
User-agent: *
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /
2 changes: 2 additions & 0 deletions apps/web/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
<meta name="msapplication-TileImage" content="/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">

<meta name="robots" content="noindex, nofollow, noarchive, noimageindex" />
<meta name="googlebot" content="noindex, nofollow" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="stylesheet" href="/src/styles.css" />
Expand Down
20 changes: 20 additions & 0 deletions apps/web/public/robots.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
User-agent: *
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /
Loading