diff --git a/client/src/components/models/ModelComparison.jsx b/client/src/components/models/ModelComparison.jsx index 99687bcb06..59bb82292b 100644 --- a/client/src/components/models/ModelComparison.jsx +++ b/client/src/components/models/ModelComparison.jsx @@ -279,15 +279,12 @@ export default function ModelComparison() { }; const handleSyncAA = () => { - if (!syncKey.trim()) { - setSyncError('Please enter an Artificial Analysis API key.'); - return; - } setSyncing(true); setSyncError(''); setSyncStatus('Connecting to Artificial Analysis and syncing models…'); - syncArtificialAnalysis({ apiKey: syncKey.trim() }, { silent: true }) + syncArtificialAnalysis({ ...(syncKey.trim() ? { apiKey: syncKey.trim() } : {}) }, { silent: true }) .then(res => { + setSyncKey(''); setSyncStatus(`Sync successful! Updated ${res.observations} models (${res.total} total).`); refreshView(); }) @@ -651,7 +648,7 @@ export default function ModelComparison() {

Fetch the latest benchmark evaluations, pricing, response times, and reasoning effort measurements from the - Artificial Analysis Free API. + Artificial Analysis Free API. A key entered here is saved privately after authentication succeeds. Leave blank to reuse a configured key, or manage it in Settings → Credentials.

@@ -113,6 +126,22 @@ export function CredentialsTab() {

)} + {credential.editable && ( +
+ + setDrafts(previous => ({ ...previous, [credential.id]: event.target.value }))} + className="w-full bg-port-bg border border-port-border rounded p-2" /> +
+ + +
+ {receipt?.id === credential.id &&

{receipt.message}

} +
+ )}
{credential.configurePath && ( ({ getCredentialInventory: vi.fn(), + saveCredential: vi.fn(), })); vi.mock('../../services/api', () => mock); @@ -88,3 +89,19 @@ describe('CredentialsTab', () => { expect(screen.queryByText('Not configured')).not.toBeInTheDocument(); }); }); + +it('saves and clears a write-only key with immediate feedback and no displayed stored value', async () => { + const row = { ...PAYLOAD.credentials[0], editable: true }; + mock.getCredentialInventory.mockResolvedValue({ ...PAYLOAD, credentials: [row] }); + mock.saveCredential.mockResolvedValue(row); + render(); + const input = await screen.findByLabelText('New Hugging Face key'); + expect(input.value).toBe(''); + fireEvent.change(input, { target: { value: 'hf_example' } }); + fireEvent.click(screen.getByRole('button', { name: 'Save key' })); + expect(screen.getByRole('status').textContent).toBe('Saving…'); + await waitFor(() => expect(input.value).toBe('')); + expect(mock.saveCredential).toHaveBeenCalledWith('huggingface', 'hf_example', { silent: true }); + fireEvent.click(screen.getByRole('button', { name: 'Clear saved key' })); + await waitFor(() => expect(mock.saveCredential).toHaveBeenCalledWith('huggingface', '', { silent: true })); +}); diff --git a/client/src/services/README.md b/client/src/services/README.md index 6f25fcc8ce..6c55d93151 100644 --- a/client/src/services/README.md +++ b/client/src/services/README.md @@ -65,7 +65,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire | `apiSchedules.js` | Automation schedules. | | `apiQuotaBurn.js` | Quota Burn plan + live status, the job-type catalog its config form renders, and manual runs (`getQuotaBurn`/`getQuotaBurnCatalog`/`saveQuotaBurn`/`runQuotaBurn`), plus `rearmQuotaBurn` to put spent `run once` steps back into the rotation. | | `apiRapidReader.js` | Rapid Reader's optional author-hosted Accelerando loader and machine-local shelf API. | -| `apiSystem.js` | System info (CPU/memory/ports/alerts/active processing and local hardware capabilities) + D&D-style character sheet getter, plus the usage cost report and explicit historical reconciliation (`getUsage`, `getProviderUsage`, `getUsageBackfillStatus`/`startUsageBackfill`, `updateSubscriptionCosts` for the subscription-vs-API savings comparison, `updateUsageFleetBilling` to exclude an API-billed federated instance from Across Instances totals). Also `getCredentialInventory` (`GET /settings/credentials`) — presence and source of each PortOS credential, never a value. | +| `apiSystem.js` | System info (CPU/memory/ports/alerts/active processing and local hardware capabilities) + D&D-style character sheet getter, plus the usage cost report and explicit historical reconciliation (`getUsage`, `getProviderUsage`, `getUsageBackfillStatus`/`startUsageBackfill`, `updateSubscriptionCosts` for the subscription-vs-API savings comparison, `updateUsageFleetBilling` to exclude an API-billed federated instance from Across Instances totals). Also `saveCredential` (write-only integration key save/clear) and `getCredentialInventory` (`GET /settings/credentials`) — presence and source of each PortOS credential, never a value. | | `apiAuth.js` | Optional login password — status, login, set/clear password. | | `apiLoops.js` | Scheduled loops. | diff --git a/client/src/services/apiSystem.js b/client/src/services/apiSystem.js index 78c5932b88..69d84a171e 100644 --- a/client/src/services/apiSystem.js +++ b/client/src/services/apiSystem.js @@ -476,3 +476,5 @@ export const updateGoalScorecardSettings = (partial) => request('/insights/goal- export const getEidoverseDestinations = (options) => request('/eidoverse/travel/destinations', options); export const departEidoverse = (peerId, options) => request('/eidoverse/travel/depart', { method: 'POST', body: JSON.stringify({ peerId }), ...options }); + +export const saveCredential = (id, value, options) => request(`/settings/credentials/${encodeURIComponent(id)}`, { ...options, method: 'PUT', body: JSON.stringify({ value }) }); diff --git a/docs/STORAGE.md b/docs/STORAGE.md index 53bb2a0151..6c199aa710 100644 --- a/docs/STORAGE.md +++ b/docs/STORAGE.md @@ -226,3 +226,29 @@ Apply this checklist to **every new feature that persists data**, and require it `data/model-comparison.json` is `file-primary`: a bounded, externally researched reference snapshot, directly inspectable/importable as a portable JSON document, with no app-record foreign keys, cross-record queries, search index or accumulated history. It follows the local-assessment reference pattern, rather than representing app-native relational records. It is intentionally machine-local and never federated because configuration and quota interpretation can be install-specific. Schema version 1 is seeded for new installs and migration 351 preserves existing catalogs. Rsync backups include it; no backup exclusion, sync cursor or tombstone is added. Source dates and exact benchmark/configuration identities remain attached to metrics. The server rejects future/malformed versions and merges imports through a serialized last-good-preserving write. See [Models Comparison](MODEL-COMPARISON.md). Optional SDK environments under `data/venvs/` are machine-local, regenerable runtime files, not application records. Reactor provisions its pinned SDK, private Python and checksum-verified uv manager on the first authorized render (or optionally through `npm run setup:reactor`); no seed, migration, database table, or peer synchronization is needed. Data Manager identifies these environments but does not purge them while render processes may use them. + +### Private integration API keys + +`data/private/api-keys.json` is machine-local `file-primary` configuration, +not a relational record store or a federation payload. Its directory is owner-only +(0700) and the file is owner-readable/writable (0600) on POSIX systems. It is not +mounted under the HTTP asset routes. Filesystem backups include it: protect backup +access as carefully as the install. This is permission-protected storage, not +application-level encryption. + +Settings > Credentials manages Artificial Analysis, Hugging Face, CivitAI, fal.ai, +and reactor.inc keys through a write-only endpoint. Existing integration settings +screens use the same store. Server-side settings readers retain their legacy shape; +public settings and inventory responses never contain these values. Stored keys +win over environment fallbacks; clearing a saved key allows an existing environment +credential (or Hugging Face CLI login) to apply again. + +Migration 357 copies existing settings keys before removing their old fields and +preserves keys already in the private store on retry. There is no seed file. The +runtime also reads legacy settings until their next save, so independently updated +installs do not need to re-enter keys. Environment credentials remain supported and +are not copied automatically. Artificial Analysis saves a supplied key after a +successful API fetch and subsequent syncs can omit it. Downgrades to versions before +this store require re-entering keys through the older settings UI or environment. +Other credentials (provider connections, account-specific logins, and auth) retain +their existing dedicated stores and management flows. diff --git a/scripts/lib/migrationOwnedPaths.js b/scripts/lib/migrationOwnedPaths.js index 979570047d..b511062ab8 100644 --- a/scripts/lib/migrationOwnedPaths.js +++ b/scripts/lib/migrationOwnedPaths.js @@ -11,6 +11,7 @@ /** Paths relative to `data/` (and to `data.reference/`), always posix-spelled. */ export const MIGRATION_OWNED_PATHS = new Set([ + 'private/api-keys.json', // Derived from legacy integration settings; never seeded. 'eidoverse/portos-world.json', // Per-install state and explicit aliases; never seed over it. // Migration 339 lifts the durable CoS config out of data/cos/state.json. // Absent, `loadConfig()` in server/services/cosState.js returns DEFAULT_CONFIG. diff --git a/scripts/migrations/357-private-api-keys.js b/scripts/migrations/357-private-api-keys.js new file mode 100644 index 0000000000..613d720c32 --- /dev/null +++ b/scripts/migrations/357-private-api-keys.js @@ -0,0 +1,16 @@ +import { join } from 'node:path'; +import { atomicWrite, readJSONFileStrict } from '../../server/lib/fileUtils.js'; +import { persistPrivateKeys } from '../../server/services/privateKeyStore.js'; + +export default { + async up({ rootDir }) { + const file = join(rootDir, 'data/settings.json'); + const { ok, value } = await readJSONFileStrict(file, null); + if (!ok) throw new Error('Cannot migrate unreadable settings'); + if (!value) return { success: true, skipped: 'no settings' }; + // Write keys before removing legacy copies. A retry preserves newer keys. + const settings = await persistPrivateKeys(value, join(rootDir, 'data'), { preserveExisting: true }); + await atomicWrite(file, settings); + return { success: true }; + }, +}; diff --git a/scripts/migrations/357-private-api-keys.test.js b/scripts/migrations/357-private-api-keys.test.js new file mode 100644 index 0000000000..dbc56f3ea9 --- /dev/null +++ b/scripts/migrations/357-private-api-keys.test.js @@ -0,0 +1,19 @@ +import { it, expect } from 'vitest'; +import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import migration from './357-private-api-keys.js'; + +it('migrates from input even with an existing destination and preserves newer stored keys on retry', async () => { + const rootDir = await mkdtemp(join(tmpdir(), 'keys-migration-')); + try { + await mkdir(join(rootDir, 'data/private'), { recursive: true }); + await writeFile(join(rootDir, 'data/private/api-keys.json'), JSON.stringify({ schemaVersion: 1, keys: { civitai: 'example-new' } })); + await writeFile(join(rootDir, 'data/settings.json'), JSON.stringify({ civitai: { apiKey: 'example-old' }, imageGen: { hfToken: 'hf_example', mode: 'local' } })); + await migration.up({ rootDir }); + await migration.up({ rootDir }); + const store = JSON.parse(await readFile(join(rootDir, 'data/private/api-keys.json'), 'utf8')); + expect(store.keys).toEqual({ civitai: 'example-new', huggingface: 'hf_example' }); + expect(JSON.parse(await readFile(join(rootDir, 'data/settings.json'), 'utf8'))).toEqual({ civitai: {}, imageGen: { mode: 'local' } }); + } finally { await rm(rootDir, { recursive: true, force: true }); } +}); diff --git a/server/lib/apiRouteCatalog.generated.json b/server/lib/apiRouteCatalog.generated.json index eb45634980..f4c67cc978 100644 --- a/server/lib/apiRouteCatalog.generated.json +++ b/server/lib/apiRouteCatalog.generated.json @@ -14585,6 +14585,14 @@ "server/routes/settings.js" ] }, + { + "method": "PUT", + "path": "/api/settings/credentials/:id", + "mountPath": "/api/settings", + "sources": [ + "server/routes/settings.js" + ] + }, { "method": "GET", "path": "/api/settings/features", @@ -17812,8 +17820,8 @@ ], "stats": { "mounts": 150, - "operations": 2206, - "declarations": 2214, + "operations": 2207, + "declarations": 2215, "sourceFiles": 233 } } diff --git a/server/lib/credentialRegistry.js b/server/lib/credentialRegistry.js index a915f66227..62092ab84f 100644 --- a/server/lib/credentialRegistry.js +++ b/server/lib/credentialRegistry.js @@ -6,8 +6,8 @@ // // Ordered most-value-first. Adding a credential: // 1. add a descriptor here; -// 2. point `configurePath` at the existing per-integration tab (this page -// never collects a secret); +// 2. point `configurePath` at the integration tab; `privateStore: true` enables +// write-only management here for keys resolved through settings.js; // 3. tag `feature` when an instance-feature id from instanceFeatureRegistry // stays dark without it. // Presence and source only. Never the value, and not a masked prefix either. @@ -17,6 +17,7 @@ export const CREDENTIAL_TIERS = Object.freeze(['free', 'metered', 'none']); export const CREDENTIALS = Object.freeze([ Object.freeze({ id: 'huggingface', + privateStore: true, label: 'Hugging Face', unlocks: 'Authenticated model, LoRA, and 3D-asset downloads (FLUX, LTX, Trellis, and gated Hub repos).', tier: 'free', @@ -25,6 +26,24 @@ export const CREDENTIALS = Object.freeze([ settingsPath: 'imageGen.hfToken', configurePath: '/media/image?settings=1', }), + Object.freeze({ + id: 'artificial-analysis', label: 'Artificial Analysis', privateStore: true, + unlocks: 'Model benchmark and pricing sync.', tier: 'free', + getUrl: 'https://artificialanalysis.ai/', envVars: Object.freeze(['ARTIFICIAL_ANALYSIS_API_KEY']), + settingsPath: 'secrets.artificialAnalysis.apiKey', configurePath: '/settings/credentials', + }), + Object.freeze({ + id: 'fal', label: 'fal.ai', privateStore: true, + unlocks: 'Video generation through fal.ai.', tier: 'metered', + getUrl: 'https://fal.ai/dashboard/keys', envVars: Object.freeze(['FAL_KEY']), + settingsPath: 'videoGen.fal.apiKey', configurePath: '/settings/credentials', + }), + Object.freeze({ + id: 'reactor', label: 'reactor.inc', privateStore: true, + unlocks: 'Video generation through reactor.inc.', tier: 'metered', + getUrl: 'https://reactor.inc/', envVars: Object.freeze(['REACTOR_API_KEY']), + settingsPath: 'videoGen.reactor.apiKey', configurePath: '/settings/credentials', + }), Object.freeze({ id: 'github', label: 'GitHub', @@ -77,6 +96,7 @@ export const CREDENTIALS = Object.freeze([ }), Object.freeze({ id: 'civitai', + privateStore: true, label: 'CivitAI', unlocks: 'LoRA and checkpoint downloads from civitai.com.', tier: 'free', diff --git a/server/lib/validation.js b/server/lib/validation.js index a431333b90..2608afb164 100644 --- a/server/lib/validation.js +++ b/server/lib/validation.js @@ -1,3 +1,4 @@ +import { CREDENTIALS } from './credentialRegistry.js'; import { z } from 'zod'; import { ServerError } from './errorHandler.js'; import { partialWithoutDefaults, emptyToUndefined, emptyToNull, optionalBooleanMap } from './zodCompat.js'; @@ -1917,4 +1918,7 @@ export const modelComparisonImportSchema = z.object({ }); export const modelComparisonDiscoverySchema = z.object({ providerId: z.string().min(1).max(200) }).strict(); +export const privateCredentialParamsSchema = z.object({ id: z.enum(CREDENTIALS.filter(entry => entry.privateStore).map(entry => entry.id)) }); +export const privateCredentialInputSchema = z.object({ value: z.string().trim().max(2000) }).strict(); + export const modelComparisonSyncSchema = z.object({ apiKey: z.string().min(1).max(200).optional() }).strict(); diff --git a/server/routes/modelComparison.test.js b/server/routes/modelComparison.test.js index 8b81c7ec81..6bb35a1b87 100644 --- a/server/routes/modelComparison.test.js +++ b/server/routes/modelComparison.test.js @@ -1,3 +1,4 @@ +vi.mock('../services/settings.js', () => ({ getSettings: vi.fn(async () => ({})), updateSettingsWith: vi.fn() })); import { beforeEach, afterEach, expect, it, vi } from 'vitest'; import express from 'express'; import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; diff --git a/server/routes/settings.js b/server/routes/settings.js index 27266ca450..70d903a437 100644 --- a/server/routes/settings.js +++ b/server/routes/settings.js @@ -1,3 +1,4 @@ +import { PRIVATE_CREDENTIALS, putCredential } from '../services/privateKeyStore.js'; import { Router } from 'express'; import { z } from 'zod'; import { getSettings, updateSettingsWith } from '../services/settings.js'; @@ -21,7 +22,7 @@ import { isPlainObject } from '../lib/objects.js'; import { DEFAULT_UNTRUSTED_CONTENT_POLICY, untrustedContentSettingsSchema } from '../lib/untrustedContent.js'; import { agentContextSettingsSchema } from '../lib/agentContextValidation.js'; import { EFFORT_LEVELS } from '../lib/providerModels.js'; -import { backupConfigSchema, sharingSettingsPatchSchema, featureProviderConfigSchema, autofixerSettingsSchema, codeReviewSettingsSchema, locationSettingsSchema, hideFirstRunCardSchema, settingsEmbeddingsSchema, localLlmSettingsSchema, imessageConfigSchema, signalConfigSchema, spotifyConfigSchema, youtubeConfigSchema, apiAccessSettingsSchema, instanceFeatureSettingsSchema, instanceFeatureIdSchema, instanceFeatureUpdateSchema, loraTrainingConfigSchema, pipelineEditorialChecksSettingsSchema, creativeDirectorSettingsSchema, musicSettingsSchema, federationSettingsSchema, privacySettingsSchema, seriesAutopilotSettingsSchema, layeredIntelligenceSettingsSchema, imageGenGrokSettingsSchema, imageGenAgySettingsSchema, renderDefaultsSettingsSchema, videoGenSettingsSchema, subscriptionCostsMapSchema, usageApiBilledInstanceIdsSchema, namedOrchestrationProfileSchema, orchestrationProfilesSettingsSchema, validateRequest } from '../lib/validation.js'; +import { privateCredentialParamsSchema, privateCredentialInputSchema, backupConfigSchema, sharingSettingsPatchSchema, featureProviderConfigSchema, autofixerSettingsSchema, codeReviewSettingsSchema, locationSettingsSchema, hideFirstRunCardSchema, settingsEmbeddingsSchema, localLlmSettingsSchema, imessageConfigSchema, signalConfigSchema, spotifyConfigSchema, youtubeConfigSchema, apiAccessSettingsSchema, instanceFeatureSettingsSchema, instanceFeatureIdSchema, instanceFeatureUpdateSchema, loraTrainingConfigSchema, pipelineEditorialChecksSettingsSchema, creativeDirectorSettingsSchema, musicSettingsSchema, federationSettingsSchema, privacySettingsSchema, seriesAutopilotSettingsSchema, layeredIntelligenceSettingsSchema, imageGenGrokSettingsSchema, imageGenAgySettingsSchema, renderDefaultsSettingsSchema, videoGenSettingsSchema, subscriptionCostsMapSchema, usageApiBilledInstanceIdsSchema, namedOrchestrationProfileSchema, orchestrationProfilesSettingsSchema, validateRequest } from '../lib/validation.js'; const router = Router(); @@ -173,12 +174,24 @@ router.get('/features', asyncHandler(async (_req, res) => { })); // GET /api/settings/credentials -// Presence + source only. Never a value or masked prefix — the page links out -// to the existing per-integration tab to enter a secret. +// Presence + source only. Supported integrations have a separate write-only setter. router.get('/credentials', asyncHandler(async (_req, res) => { res.json(await getCredentialInventory()); })); +// Write-only, allowlisted integration credentials; no generic secrets access. +router.put('/credentials/:id', asyncHandler(async (req, res) => { + const { id } = validateRequest(privateCredentialParamsSchema, req.params); + const { value } = validateRequest(privateCredentialInputSchema, req.body); + const entry = PRIVATE_CREDENTIALS.find(item => item.id === id); + await updateSettingsWith(current => { + putCredential(current, entry, value); + return current; + }); + const inventory = await getCredentialInventory(); + res.json(inventory.credentials.find(item => item.id === id)); +})); + // POST /api/settings/features/eidoverse/install // Explicit consent boundary: no Eidoverse checkout or dependency install occurs // until the user presses Install in Settings > Features. diff --git a/server/routes/settings.secretsStrip.test.js b/server/routes/settings.secretsStrip.test.js index bf59f6514d..12cfa4814e 100644 --- a/server/routes/settings.secretsStrip.test.js +++ b/server/routes/settings.secretsStrip.test.js @@ -1,6 +1,6 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; import express from 'express'; -import { readFileSync } from 'fs'; +import { readFileSync, rmSync } from 'fs'; import { join } from 'path'; import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js'; import { bindSettingsFile } from '../lib/settingsTestUtil.js'; @@ -29,6 +29,8 @@ const { writeSettingsFile } = bindSettingsFile(tempRoot); const seedSettings = (settings) => writeSettingsFile(settings); +const readPrivateKeys = () => JSON.parse(readFileSync(join(tempRoot, 'private/api-keys.json'), 'utf8')).keys; + const readSettingsFile = () => { const raw = readFileSync(join(tempRoot, 'settings.json'), 'utf8'); return JSON.parse(raw); @@ -44,6 +46,7 @@ const buildApp = async () => { }; beforeEach(async () => { + rmSync(join(tempRoot, 'private'), { recursive: true, force: true }); await seedSettings({}); }); @@ -151,8 +154,10 @@ describe('GET /api/settings — external token redaction', () => { expect(res.body.imageGen?.defaultModel).toBe('flux'); // On-disk values survive the redaction. const persisted = readSettingsFile(); - expect(persisted.imageGen?.hfToken).toBe('hf_secret123'); - expect(persisted.civitai?.apiKey).toBe('civ_secret456'); + expect(persisted.imageGen?.hfToken).toBeUndefined(); + expect(readPrivateKeys().huggingface).toBe('hf_secret123'); + expect(persisted.civitai?.apiKey).toBeUndefined(); + expect(readPrivateKeys().civitai).toBe('civ_secret456'); }); // Because GET no longer returns the tokens, a client that rebuilds a full @@ -168,7 +173,8 @@ describe('GET /api/settings — external token redaction', () => { .send({ imageGen: { mode: 'external', local: { pythonPath: '/usr/bin/python3' } } }); expect(res.status).toBe(200); const persisted = readSettingsFile(); - expect(persisted.imageGen?.hfToken).toBe('hf_keepme'); + expect(persisted.imageGen?.hfToken).toBeUndefined(); + expect(readPrivateKeys().huggingface).toBe('hf_keepme'); expect(persisted.imageGen?.mode).toBe('external'); expect(persisted.imageGen?.local?.pythonPath).toBe('/usr/bin/python3'); // And the save response still doesn't echo the preserved token. @@ -183,7 +189,8 @@ describe('GET /api/settings — external token redaction', () => { .send({ civitai: { autoDownload: false } }); expect(res.status).toBe(200); const persisted = readSettingsFile(); - expect(persisted.civitai?.apiKey).toBe('civ_keepme'); + expect(persisted.civitai?.apiKey).toBeUndefined(); + expect(readPrivateKeys().civitai).toBe('civ_keepme'); expect(persisted.civitai?.autoDownload).toBe(false); }); @@ -193,7 +200,8 @@ describe('GET /api/settings — external token redaction', () => { const res = await request(app).put('/api/settings').send({ timezone: 'America/Los_Angeles' }); expect(res.status).toBe(200); const persisted = readSettingsFile(); - expect(persisted.imageGen?.hfToken).toBe('hf_keepme'); + expect(persisted.imageGen?.hfToken).toBeUndefined(); + expect(readPrivateKeys().huggingface).toBe('hf_keepme'); expect(persisted.timezone).toBe('America/Los_Angeles'); }); }); diff --git a/server/routes/settings.test.js b/server/routes/settings.test.js index 9e06c60c31..ee6ed943b9 100644 --- a/server/routes/settings.test.js +++ b/server/routes/settings.test.js @@ -787,3 +787,20 @@ describe('Settings routes — orchestration profiles (#5992)', () => { }); }); + +it('accepts only registered private keys and never returns the submitted secret', async () => { + const { getCredentialInventory } = await import('../services/credentialInventory.js'); + getCredentialInventory.mockResolvedValue({ credentials: [{ id: 'artificial-analysis', configured: true, source: 'settings', editable: true }] }); + const app = buildApp(); + const result = await request(app).put('/api/settings/credentials/artificial-analysis').send({ value: 'example-private-key' }); + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ configured: true }); + expect(JSON.stringify(result.body)).not.toContain('example-private-key'); + expect(store.secrets.artificialAnalysis.apiKey).toBe('example-private-key'); + const publicSettings = await request(app).get('/api/settings'); + expect(JSON.stringify(publicSettings.body)).not.toContain('example-private-key'); + expect((await request(app).put('/api/settings/credentials/auth').send({ value: 'example' })).status).toBe(400); + expect((await request(app).put('/api/settings/credentials/civitai').send({ value: {}, extra: true })).status).toBe(400); + await request(app).put('/api/settings/credentials/artificial-analysis').send({ value: '' }); + expect(store.secrets.artificialAnalysis.apiKey).toBe(''); +}); diff --git a/server/services/artificialAnalysis.js b/server/services/artificialAnalysis.js index 5fd2964d67..f3c609b4f6 100644 --- a/server/services/artificialAnalysis.js +++ b/server/services/artificialAnalysis.js @@ -1,3 +1,4 @@ +import { getSettings, updateSettingsWith } from './settings.js'; import { ServerError } from '../lib/errorHandler.js'; import { modelComparisonImportSchema } from '../lib/validation.js'; import { importModelComparison } from './modelComparison.js'; @@ -189,8 +190,7 @@ export async function fetchAllArtificialAnalysisModels(apiKey) { headers: { 'x-api-key': apiKey }, }); if (!res.ok) { - const errText = await res.text().catch(() => ''); - throw new ServerError(`Artificial Analysis API failed (${res.status}): ${errText || res.statusText}`, { status: res.status === 401 || res.status === 403 ? 401 : 502 }); + throw new ServerError(`Artificial Analysis API failed (${res.status}): ${res.statusText || 'request rejected'}`, { status: res.status === 401 || res.status === 403 ? 401 : 502 }); } const json = await res.json(); if (!json.data || !Array.isArray(json.data)) break; @@ -202,11 +202,15 @@ export async function fetchAllArtificialAnalysisModels(apiKey) { } export async function syncArtificialAnalysisCatalog({ apiKey } = {}) { - const key = apiKey || process.env.ARTIFICIAL_ANALYSIS_API_KEY; + const settings = await getSettings(); + const key = apiKey?.trim() || settings.secrets?.artificialAnalysis?.apiKey || process.env.ARTIFICIAL_ANALYSIS_API_KEY; if (!key) { throw new ServerError('No Artificial Analysis API key provided or configured in ARTIFICIAL_ANALYSIS_API_KEY', { status: 400 }); } const rawModels = await fetchAllArtificialAnalysisModels(key); + if (apiKey?.trim()) await updateSettingsWith(current => ({ + ...current, secrets: { ...current.secrets, artificialAnalysis: { apiKey: apiKey.trim() } }, + })); const observations = transformAAModelsToObservations(rawModels); const validated = modelComparisonImportSchema.parse({ schemaVersion: 1, diff --git a/server/services/artificialAnalysis.test.js b/server/services/artificialAnalysis.test.js index ce3104340b..2fda37a871 100644 --- a/server/services/artificialAnalysis.test.js +++ b/server/services/artificialAnalysis.test.js @@ -1,3 +1,4 @@ +vi.mock('./settings.js', () => ({ getSettings: vi.fn(async () => ({})), updateSettingsWith: vi.fn() })); import { describe, expect, it, vi } from 'vitest'; import { slugify, diff --git a/server/services/assetMounts.test.js b/server/services/assetMounts.test.js index 80a6a715e6..f2a5e94578 100644 --- a/server/services/assetMounts.test.js +++ b/server/services/assetMounts.test.js @@ -62,6 +62,14 @@ beforeAll(() => { }); describe('the server-owned namespace terminators', () => { + it('never serves the private API key store even when the file exists', async () => { + mkdirSync(join(tempRoot, 'private'), { recursive: true }); + writeFileSync(join(tempRoot, 'private/api-keys.json'), '{"example":"example-secret"}'); + const res = await request(app).get('/data/private/api-keys.json'); + expect(res.status).toBe(404); + expect(res.text).not.toContain('example-secret'); + }); + it('404s an extensionless /data path instead of answering with the SPA index', async () => { const res = await request(app).get('/data/image-to-3d/abc123/model'); expect(res.status).toBe(404); diff --git a/server/services/autonomousJobs.test.js b/server/services/autonomousJobs.test.js index 546496e2c9..d2f8fe11c9 100644 --- a/server/services/autonomousJobs.test.js +++ b/server/services/autonomousJobs.test.js @@ -1,5 +1,8 @@ import { describe, it, expect, beforeEach, vi } from 'vitest' +// This suite owns scheduler time, not settings or credential I/O. +vi.mock('./userTimezone.js', () => ({ getUserTimezone: vi.fn(async () => 'UTC') })) + // Mock modules before import vi.mock('./cosEvents.js', () => ({ cosEvents: { emit: vi.fn() } diff --git a/server/services/credentialInventory.js b/server/services/credentialInventory.js index ad02755d46..f43f72fd89 100644 --- a/server/services/credentialInventory.js +++ b/server/services/credentialInventory.js @@ -153,6 +153,7 @@ const publicRow = (entry, resolution, featuresById) => { : []; return { id: entry.id, + editable: entry.privateStore === true, label: entry.label, unlocks: entry.unlocks, tier: entry.tier, diff --git a/server/services/dataManager.js b/server/services/dataManager.js index 30fb45bda2..3a3d9a6f94 100644 --- a/server/services/dataManager.js +++ b/server/services/dataManager.js @@ -58,6 +58,7 @@ export const CATEGORIES = { 'browser-profile': { label: 'Browser Profile', description: 'Chrome/Chromium browser data', archivable: false, deletable: true, purgeScope: 'category' }, cache: { label: 'Remote API and Reading Cache', description: 'Cached remote API metadata and the author-hosted Accelerando reading source — refetched on demand, safe to purge', archivable: false, deletable: true, purgeScope: 'category' }, 'calendar': { label: 'Calendar', description: 'Calendar sync data', archivable: true, deletable: false }, + 'private': { label: 'Private Keys', description: 'Machine-local integration credentials — managed in Settings > Credentials', archivable: false, deletable: false }, 'certs': { label: 'TLS Certificates', description: 'HTTPS certificate and private key — purging drops the install back to HTTP', archivable: false, deletable: false }, 'commission-feedback': { label: 'Commission Feedback', description: 'Reactions on creative commissions (file mirror of the Postgres store)', archivable: true, deletable: false }, 'conflict-journal': { label: 'Conflict Journal', description: 'Peer-sync conflict history — diagnostics only, safe to purge', archivable: true, deletable: true, purgeScope: 'category' }, diff --git a/server/services/hfToken.js b/server/services/hfToken.js index 4f7e8e7218..b445441697 100644 --- a/server/services/hfToken.js +++ b/server/services/hfToken.js @@ -34,6 +34,7 @@ export async function getHfTokenInfo() { if (stored) return { token: stored, source: 'stored' }; const envToken = ( process.env.HF_TOKEN || + process.env.HUGGINGFACE_TOKEN || process.env.HUGGINGFACE_HUB_TOKEN || process.env.HUGGINGFACEHUB_API_TOKEN || null diff --git a/server/services/huggingFaceCatalog.js b/server/services/huggingFaceCatalog.js index 63e540fa15..77ab5ab381 100644 --- a/server/services/huggingFaceCatalog.js +++ b/server/services/huggingFaceCatalog.js @@ -1,3 +1,4 @@ +import { getHfToken } from './hfToken.js'; import { formatBytes as formatBytesRaw } from '../lib/fileUtils.js' import { fetchWithTimeout } from '../lib/fetchWithTimeout.js' import { readResponseJson } from '../lib/readResponseJson.js' @@ -719,9 +720,9 @@ function toResult(model, backend, requestedCategory, installedIds, installedAudi return result } -function hfHeaders() { +async function hfHeaders() { const headers = { Accept: 'application/json' } - const token = process.env.HUGGINGFACE_TOKEN || process.env.HF_TOKEN + const token = await getHfToken() if (token) headers.Authorization = `Bearer ${token}` return headers } @@ -773,7 +774,7 @@ function hfFetch(url) { return hfGate.run(async () => { const res = await fetchWithTimeout( url, - { headers: hfHeaders() }, + { headers: await hfHeaders() }, HF_TIMEOUT_MS, { retries: 1, retryDelayMs: HF_RETRY_DELAY_MS, shouldRetry: isReplayableConnectionError } // Both attempts lost the connection. undici's own message is a bare `fetch diff --git a/server/services/privateKeyStore.js b/server/services/privateKeyStore.js new file mode 100644 index 0000000000..ac9c395a9a --- /dev/null +++ b/server/services/privateKeyStore.js @@ -0,0 +1,80 @@ +// Machine-local integration keys. Only settings.js writes this store at runtime, +// inside its existing write queue. No public reader returns key material. +import { chmod, mkdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { atomicWrite, safeJSONParse } from '../lib/fileUtils.js'; +import { CREDENTIALS } from '../lib/credentialRegistry.js'; +import { isPlainObject } from '../lib/objects.js'; + +export const PRIVATE_CREDENTIALS = CREDENTIALS.filter(entry => entry.privateStore); + +export function credentialValue(settings, entry) { + return entry.settingsPath.split('.').reduce((value, key) => value?.[key], settings); +} + +export function putCredential(settings, entry, value) { + const parts = entry.settingsPath.split('.'); + const leaf = parts.pop(); + let parent = settings; + for (const key of parts) { + if (!isPlainObject(parent[key])) parent[key] = {}; + parent = parent[key]; + } + parent[leaf] = value; +} + +export async function readPrivateKeys(dataDir) { + const raw = await readFile(join(dataDir, 'private/api-keys.json'), 'utf8').catch(error => { + if (error.code === 'ENOENT') return null; + throw new Error('Private key store could not be read'); + }); + if (raw === null) return {}; + // Never include malformed input (which can contain secrets) in parser errors. + const store = safeJSONParse(raw, null); + if (!isPlainObject(store) || store.schemaVersion !== 1 || !isPlainObject(store.keys) + || Object.values(store.keys).some(value => typeof value !== 'string')) { + throw new Error('Private key store has an unsupported format'); + } + return store.keys; +} + +export async function hydratePrivateKeys(settings, dataDir) { + const keys = await readPrivateKeys(dataDir); + for (const entry of PRIVATE_CREDENTIALS) { + if (Object.hasOwn(keys, entry.id)) putCredential(settings, entry, keys[entry.id]); + } + return settings; +} + +export async function persistPrivateKeys(settings, dataDir, { preserveExisting = false, previousSettings } = {}) { + const keys = await readPrivateKeys(dataDir); + const publicSettings = structuredClone(settings); + let changed = false; + for (const entry of PRIVATE_CREDENTIALS) { + // Existing integration clear actions delete the field rather than writing ''. + // Compare the queued pre-image so omission in an unrelated PATCH is preserved + // by settings' merge, while an intentional removal becomes a tombstone. + const value = credentialValue(settings, entry) + ?? (typeof credentialValue(previousSettings, entry) === 'string' ? '' : undefined); + if (typeof value !== 'string') continue; + if (!preserveExisting || !Object.hasOwn(keys, entry.id)) { + const normalized = value.trim(); // Empty is a tombstone: legacy values cannot reappear. + if (keys[entry.id] !== normalized) changed = true; + keys[entry.id] = normalized; + } + const parts = entry.settingsPath.split('.'); + const leaf = parts.pop(); + const parent = parts.reduce((object, key) => object?.[key], publicSettings); + if (parent) delete parent[leaf]; + } + if (Object.keys(keys).length) { + const directory = join(dataDir, 'private'); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await chmod(directory, 0o700); + const file = join(directory, 'api-keys.json'); + // Directory permissions protect even pre-existing files while tightened. + if (changed) await atomicWrite(file, { schemaVersion: 1, keys }); + await chmod(file, 0o600); + } + return publicSettings; +} diff --git a/server/services/privateKeyStore.test.js b/server/services/privateKeyStore.test.js new file mode 100644 index 0000000000..8f5a059c1c --- /dev/null +++ b/server/services/privateKeyStore.test.js @@ -0,0 +1,77 @@ +import { beforeEach, afterEach, expect, it, vi } from 'vitest'; +import { mkdtemp, readFile, writeFile, rm, stat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +const context = vi.hoisted(() => ({ data: '' })); +vi.mock('../lib/paths.js', async importActual => { + const actual = await importActual(); + return { ...actual, PATHS: { ...actual.PATHS, get data() { return context.data; } } }; +}); +vi.mock('./userActions.js', () => ({ recordUserAction: vi.fn() })); +let settings; +let directory; +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'private-keys-test-')); + context.data = directory; + vi.resetModules(); + settings = await import('./settings.js'); +}); +afterEach(async () => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + await rm(directory, { recursive: true, force: true }); +}); + +it('moves legacy keys out of settings, preserves unrelated data, and resolves after restart and clear', async () => { + await writeFile(join(directory, 'settings.json'), JSON.stringify({ + imageGen: { hfToken: 'hf_example', mode: 'local' }, + civitai: { apiKey: 'example-civitai' }, + videoGen: { fal: { apiKey: 'example-fal', model: 'example' }, reactor: { apiKey: 'example-reactor' } }, + })); + await settings.updateSettings({ theme: 'dark' }); + const disk = await readFile(join(directory, 'settings.json'), 'utf8'); + expect(disk).not.toMatch(/hf_example|example-civitai|example-fal|example-reactor/); + expect(JSON.parse(disk)).toMatchObject({ imageGen: { mode: 'local' }, videoGen: { fal: { model: 'example' } } }); + if (process.platform !== 'win32') { + expect((await stat(join(directory, 'private'))).mode & 0o777).toBe(0o700); + expect((await stat(join(directory, 'private/api-keys.json'))).mode & 0o777).toBe(0o600); + } + settings.__resetSettingsCache(); + expect((await settings.getSettings()).videoGen.reactor.apiKey).toBe('example-reactor'); + await settings.updateSettingsWith(current => ({ ...current, civitai: { apiKey: '' } })); + settings.__resetSettingsCache(); + expect((await settings.getSettings()).civitai.apiKey).toBe(''); + expect((await settings.getSettings()).videoGen.fal.apiKey).toBe('example-fal'); + await settings.updateSettingsWith(current => { + delete current.imageGen.hfToken; + return current; + }); + settings.__resetSettingsCache(); + expect((await settings.getSettings()).imageGen.hfToken).toBe(''); + expect((await settings.getSettings()).videoGen.fal.apiKey).toBe('example-fal'); +}); + +it('persists a successful Artificial Analysis key and reuses it without a request key', async () => { + vi.doMock('./modelComparison.js', () => ({ importModelComparison: vi.fn(async () => ({ observations: [] })) })); + const fetchMock = vi.fn(async () => ({ ok: true, json: async () => ({ data: [{ id: 'example', name: 'Example Model', slug: 'example-model', model_creator: { name: 'Example' }, evaluations: { artificial_analysis_intelligence_index: 40 } }], pagination: { has_more: false } }) })); + vi.stubGlobal('fetch', fetchMock); + vi.stubEnv('ARTIFICIAL_ANALYSIS_API_KEY', 'example-env'); + const { syncArtificialAnalysisCatalog } = await import('./artificialAnalysis.js'); + await syncArtificialAnalysisCatalog({ apiKey: 'example-saved' }); + settings.__resetSettingsCache(); + await syncArtificialAnalysisCatalog(); + expect(fetchMock.mock.calls[1][1].headers['x-api-key']).toBe('example-saved'); + fetchMock.mockResolvedValueOnce({ ok: false, status: 401, text: async () => 'Unauthorized' }); + await expect(syncArtificialAnalysisCatalog({ apiKey: 'example-invalid' })).rejects.toThrow(); + expect((await settings.getSettings()).secrets.artificialAnalysis.apiKey).toBe('example-saved'); +}); + +it('refuses a corrupt private store without overwriting keys or exposing parser input', async () => { + await settings.updateSettings({ civitai: { apiKey: 'example-key' } }); + await writeFile(join(directory, 'private/api-keys.json'), 'example-secret-invalid-json'); + settings.__resetSettingsCache(); + await expect(settings.getSettings()).rejects.toThrow('unsupported format'); + await expect(settings.updateSettings({ theme: 'light' })).rejects.toThrow('unsupported format'); + expect(await readFile(join(directory, 'private/api-keys.json'), 'utf8')).toBe('example-secret-invalid-json'); +}); diff --git a/server/services/settings.js b/server/services/settings.js index 60d3cb4b58..d3f1c3b457 100644 --- a/server/services/settings.js +++ b/server/services/settings.js @@ -1,3 +1,4 @@ +import { hydratePrivateKeys, persistPrivateKeys } from './privateKeyStore.js'; import { join } from 'path'; import { EventEmitter } from 'events'; import { safeJSONParse, PATHS, atomicWrite, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js'; @@ -71,7 +72,7 @@ settingsEvents.setMaxListeners(50); // write resolves. const loadRaw = async () => { const raw = await tryReadFile(SETTINGS_FILE); - return safeJSONParse(raw ?? '{}', {}); + return hydratePrivateKeys(safeJSONParse(raw ?? '{}', {}), PATHS.data); }; /** @@ -171,7 +172,7 @@ export const reloadSettings = async () => { settingsEvents.emit('settings:invalidated'); return {}; } - const cleaned = stripStoreKeys(settings); + const cleaned = stripStoreKeys(await hydratePrivateKeys(settings, PATHS.data)); settingsEvents.emit('settings:updated', cleaned); return cleaned; }; @@ -256,7 +257,8 @@ const save = async (settings, { actor = 'system', skipUserAction = false } = {}) // atomicWrite (temp-file + rename) so a mid-write crash never truncates // settings.json. Pass a pre-stringified string to preserve the trailing // newline; atomicWrite's own JSON.stringify omits it. - await atomicWrite(SETTINGS_FILE, JSON.stringify(cleaned, null, 2) + '\n'); + await atomicWrite(SETTINGS_FILE, JSON.stringify(await persistPrivateKeys(cleaned, PATHS.data, { previousSettings: prev }), null, 2) + '\n'); + await hydratePrivateKeys(cleaned, PATHS.data); // Warn AFTER the successful write so a thrown write never produces // a misleading "stripped" log line for a write that didn't happen. if (isPlainObject(settings)) { @@ -314,7 +316,7 @@ export const getSettingsWithStatus = async () => { // covers BOTH failure modes (malformed content and an unreadable-but-present // file); only a genuinely ABSENT file (fresh install) caches `{}`. const { corrupt, settings } = await readSettingsStrict(); - const loaded = stripStoreKeys(settings); + const loaded = stripStoreKeys(await hydratePrivateKeys(settings, PATHS.data)); // A save()/reloadSettings() may have populated the cache via the // settings:updated listener while this cold read was awaiting the disk read. // Prefer that fresher in-memory value over our (older) on-disk snapshot. diff --git a/server/services/settings.test.js b/server/services/settings.test.js index 10d2a8b5c9..55ecc9ae8f 100644 --- a/server/services/settings.test.js +++ b/server/services/settings.test.js @@ -1,3 +1,8 @@ +// Store I/O is covered through real persisted settings in privateKeyStore.test.js. +vi.mock('./privateKeyStore.js', () => ({ + hydratePrivateKeys: vi.fn(async settings => settings), + persistPrivateKeys: vi.fn(async settings => settings), +})); import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; // settings.js persists via the shared atomicWrite helper and reads via