-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathserver.js
More file actions
1950 lines (1798 loc) · 96.1 KB
/
Copy pathserver.js
File metadata and controls
1950 lines (1798 loc) · 96.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Run with: ~/.bun/bin/bun server.js
import { createPublicKey, verify as cryptoVerify, X509Certificate } from 'crypto';
import { Database } from 'bun:sqlite';
import { unlink } from 'node:fs/promises';
const htmlPath = new URL('./index.html', import.meta.url);
// ── Malware cache (SQLite) ────────────────────────────────────────────────────
const DB_PATH = process.env.MALWARE_DB_PATH || './malware.db';
const db = new Database(DB_PATH);
db.exec('PRAGMA journal_mode = WAL');
db.exec(`
CREATE TABLE IF NOT EXISTS malware (
package_name TEXT NOT NULL,
version TEXT,
scope TEXT,
malid TEXT,
source TEXT,
blocked_at TEXT NOT NULL,
ecosystem TEXT NOT NULL,
reason_json TEXT,
description TEXT,
PRIMARY KEY (ecosystem, package_name, version, malid, blocked_at)
);
CREATE INDEX IF NOT EXISTS idx_malware_blocked_at ON malware(blocked_at DESC);
CREATE INDEX IF NOT EXISTS idx_malware_package ON malware(package_name);
CREATE TABLE IF NOT EXISTS sync_meta (
key TEXT PRIMARY KEY,
value TEXT
);
CREATE TABLE IF NOT EXISTS sync_windows (
ecosystem TEXT NOT NULL,
window_start TEXT NOT NULL,
window_end TEXT NOT NULL,
synced_at TEXT NOT NULL,
PRIMARY KEY (ecosystem, window_start)
);
`);
// Chainguard Libraries VEX (OpenVEX) — backported-fix statements, one row per
// (package build version, vulnerability). Refreshed daily alongside the malware
// mirror; a cold lookup falls back to a live per-package fetch.
db.exec(`
CREATE TABLE IF NOT EXISTS vex (
ecosystem TEXT NOT NULL, -- 'pypi' | 'maven'
package_name TEXT NOT NULL, -- pypi: normalized name; maven: group:artifact
base_version TEXT NOT NULL, -- upstream version (cgr build suffix stripped)
full_version TEXT NOT NULL, -- chainguard build version (from the purl)
vuln_name TEXT, -- advisory id (CGA-…)
cve TEXT,
ghsa TEXT,
aliases_json TEXT,
fixed_at TEXT, -- statement timestamp (when the fix was published)
synced_at TEXT NOT NULL,
PRIMARY KEY (ecosystem, package_name, full_version, vuln_name)
);
CREATE INDEX IF NOT EXISTS idx_vex_pkg ON vex(ecosystem, package_name);
`);
// Migration: add fixed_at column if this DB predates it, then ensure its index.
if (!db.prepare(`SELECT 1 FROM pragma_table_info('vex') WHERE name='fixed_at'`).get()) {
db.exec(`ALTER TABLE vex ADD COLUMN fixed_at TEXT`);
}
db.exec(`CREATE INDEX IF NOT EXISTS idx_vex_fixed_at ON vex(fixed_at)`);
// Migration: add published_at column if not present
{
const hasPubCol = db.prepare(`SELECT 1 FROM pragma_table_info('malware') WHERE name='published_at'`).get();
if (!hasPubCol) {
db.exec(`
ALTER TABLE malware ADD COLUMN published_at TEXT;
CREATE INDEX IF NOT EXISTS idx_malware_published ON malware(published_at);
`);
}
}
// Ensure compound indexes for common TDD query patterns
db.exec(`
CREATE INDEX IF NOT EXISTS idx_malware_eco_pub ON malware(ecosystem, published_at);
CREATE INDEX IF NOT EXISTS idx_malware_src_pub ON malware(source, published_at);
CREATE INDEX IF NOT EXISTS idx_malware_eco_blk ON malware(ecosystem, blocked_at DESC);
`);
const insertMalware = db.prepare(`
INSERT INTO malware
(package_name, version, scope, malid, source, blocked_at, ecosystem, reason_json, description)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(ecosystem, package_name, version, malid, blocked_at) DO UPDATE SET
scope = excluded.scope,
source = excluded.source,
reason_json = excluded.reason_json,
description = excluded.description
`);
// ── Malware enrichment (publish-date fetch from registries) ──────────────────
const enrichState = { running: false, done: 0, total: 0, failed: 0, error: null, startedAt: null, finishedAt: null };
async function fetchNpmTimestamps(packageName) {
const res = await fetch(`https://registry.npmjs.org/${packageName}`);
if (!res.ok) return null;
const data = await res.json();
const time = data.time || {};
const skip = new Set(['created', 'modified', 'unpublished']);
const out = {};
for (const [k, v] of Object.entries(time)) {
if (!skip.has(k)) out[k] = v;
}
out[''] = time.created || null; // for package-wide blocks
return out;
}
async function fetchPypiTimestamps(packageName) {
const res = await fetch(`https://pypi.org/pypi/${encodeURIComponent(packageName)}/json`);
if (!res.ok) return null;
const data = await res.json();
const releases = data.releases || {};
const out = {};
for (const [ver, files] of Object.entries(releases)) {
const uploadTime = files?.[0]?.upload_time;
if (uploadTime) out[ver] = uploadTime.endsWith('Z') ? uploadTime : uploadTime + 'Z';
}
const firstDate = Object.values(out).sort()[0] || null;
out[''] = firstDate;
return out;
}
async function fetchMavenTimestamps(packageName) {
const slashIdx = packageName.indexOf('/');
if (slashIdx < 0) return null;
const group = packageName.slice(0, slashIdx);
const artifact = packageName.slice(slashIdx + 1);
const q = encodeURIComponent(`g:${group} AND a:${artifact}`);
const res = await fetch(`https://search.maven.org/solrsearch/select?q=${q}&core=gav&rows=200&sort=timestamp+asc&wt=json`);
if (!res.ok) return null;
const data = await res.json();
const docs = data.response?.docs || [];
const out = {};
for (const doc of docs) {
if (doc.v && doc.timestamp) out[doc.v] = new Date(doc.timestamp).toISOString();
}
const times = Object.values(out).sort();
out[''] = times[0] || null;
return out;
}
async function runMalwareEnrich() {
if (enrichState.running) throw new Error('Enrichment already in progress');
enrichState.running = true;
enrichState.done = 0;
enrichState.total = 0;
enrichState.failed = 0;
enrichState.error = null;
enrichState.startedAt = new Date().toISOString();
enrichState.finishedAt = null;
try {
const pending = db.prepare(`
SELECT ecosystem, package_name FROM malware
WHERE published_at IS NULL
GROUP BY ecosystem, package_name
ORDER BY MAX(blocked_at) DESC
`).all();
enrichState.total = pending.length;
const updateStmt = db.prepare(`UPDATE malware SET published_at = ? WHERE ecosystem = ? AND package_name = ? AND version = ?`);
const BATCH = 50;
const FETCH_TIMEOUT = 8000;
const withTimeout = (p) => Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), FETCH_TIMEOUT))]);
for (let i = 0; i < pending.length; i += BATCH) {
const batch = pending.slice(i, i + BATCH);
await Promise.allSettled(batch.map(async ({ ecosystem, package_name }) => {
try {
let timeMap;
if (ecosystem === 'npm') timeMap = await withTimeout(fetchNpmTimestamps(package_name));
else if (ecosystem === 'PyPI') timeMap = await withTimeout(fetchPypiTimestamps(package_name));
else timeMap = await withTimeout(fetchMavenTimestamps(package_name));
const versions = db.prepare(
`SELECT version FROM malware WHERE ecosystem = ? AND package_name = ? AND published_at IS NULL`
).all(ecosystem, package_name);
db.transaction(() => {
for (const { version } of versions) {
updateStmt.run(timeMap?.[version] ?? 'NOT_FOUND', ecosystem, package_name, version);
}
})();
} catch {
db.prepare(`UPDATE malware SET published_at = 'ERROR' WHERE ecosystem = ? AND package_name = ? AND published_at IS NULL`)
.run(ecosystem, package_name);
enrichState.failed++;
}
enrichState.done++;
}));
await new Promise(r => setTimeout(r, 0));
}
} catch (err) {
enrichState.error = err.message;
throw err;
} finally {
enrichState.running = false;
enrichState.finishedAt = new Date().toISOString();
}
}
// ── Platform API token (server-side storage + auto-refresh) ───────────────────
// Platform API ecosystem values and their DB names
const PLATFORM_ECOSYSTEMS = [
{ apiName: 'npm', dbName: 'npm' },
{ apiName: 'Maven', dbName: 'Maven' },
{ apiName: 'PyPI', dbName: 'PyPI' },
];
// One-time cleanup: earlier syncs stored rows under the API's inconsistent ecosystem
// casing (e.g. 'pypi'/'maven'), producing orphan rows the read path never queries.
// Drop anything outside the canonical set (derived here so it can't wipe a real ecosystem).
{
const canonical = PLATFORM_ECOSYSTEMS.map(e => e.dbName);
const res = db.prepare(`DELETE FROM malware WHERE ecosystem NOT IN (${canonical.map(() => '?').join(',')})`).run(...canonical);
if (res.changes) console.log(`[migration] removed ${res.changes} orphan malware row(s) with non-canonical ecosystem casing`);
}
let platformToken = process.env.PLATFORM_API_TOKEN || null;
let platformTokenExpiry = null; // Unix timestamp (ms)
let tokenRefreshTimer = null;
// The malware blocklist API lives behind console-api; a token minted for any
// other audience (e.g. the libraries.cgr.dev registry) is rejected with a
// confusing HTTP 500, so we validate the aud claim up front.
const EXPECTED_TOKEN_AUDIENCE = 'https://console-api.enforce.dev';
function decodePlatformTokenPayload(token) {
try {
return JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString());
} catch { return null; }
}
function parsePlatformTokenExpiry(token) {
const payload = decodePlatformTokenPayload(token);
return payload?.exp ? payload.exp * 1000 : null;
}
// Returns the aud claim as an array (JWT aud may be a string or array), or null.
function parsePlatformTokenAudiences(token) {
const aud = decodePlatformTokenPayload(token)?.aud;
if (!aud) return null;
return Array.isArray(aud) ? aud : [aud];
}
function tokenAudienceOk(token) {
const auds = parsePlatformTokenAudiences(token);
// If we can't read an aud claim, don't block — let the API be the judge.
return !auds || auds.includes(EXPECTED_TOKEN_AUDIENCE);
}
function setPlatformToken(token) {
platformToken = token || null;
platformTokenExpiry = token ? parsePlatformTokenExpiry(token) : null;
if (token && !tokenAudienceOk(token)) {
console.warn(`[platform-token] WARNING: token audience is ${JSON.stringify(parsePlatformTokenAudiences(token))}, expected ${EXPECTED_TOKEN_AUDIENCE} — malware sync will fail with HTTP 500`);
}
if (tokenRefreshTimer) { clearTimeout(tokenRefreshTimer); tokenRefreshTimer = null; }
if (platformTokenExpiry) scheduleTokenRefresh();
}
let tokenRefreshInFlight = null;
// Single-flight: concurrent callers (e.g. parallel ecosystem syncs all hitting 401
// at once) share one chainctl mint instead of spawning several.
function refreshPlatformTokenViaChainctl() {
if (tokenRefreshInFlight) return tokenRefreshInFlight;
tokenRefreshInFlight = (async () => {
try {
if (ON_GCP) await ensureChainctlLogin(); // establish an ambient session first
const proc = Bun.spawn(['chainctl', 'auth', 'token', '--audience', 'https://console-api.enforce.dev'], {
stdout: 'pipe', stderr: 'pipe',
});
const text = await new Response(proc.stdout).text();
const code = await proc.exited;
if (code !== 0) throw new Error(`chainctl exited ${code}`);
const token = text.trim();
if (!token) throw new Error('empty token');
setPlatformToken(token);
console.log('Platform token refreshed via chainctl, expires', new Date(platformTokenExpiry).toISOString());
return token;
} catch (err) {
console.error('chainctl token refresh failed:', err.message);
return null;
}
})();
tokenRefreshInFlight.finally(() => { tokenRefreshInFlight = null; });
return tokenRefreshInFlight;
}
// Proactively refresh if the token is missing or within `minMs` of expiry — called
// before a sync so a long page-through doesn't hit a mid-flight 401 (which can
// truncate pagination). Never throws; best-effort.
async function ensurePlatformTokenFresh(minMs = 10 * 60 * 1000) {
const needsRefresh = !platformToken || (platformTokenExpiry && platformTokenExpiry - Date.now() < minMs);
if (needsRefresh) {
console.log('[malware-sync] platform token missing or near expiry — refreshing before sync');
await refreshPlatformTokenViaChainctl();
}
}
function scheduleTokenRefresh() {
if (!platformTokenExpiry) return;
const refreshAt = platformTokenExpiry - 5 * 60 * 1000; // 5 min before expiry
const delay = refreshAt - Date.now();
// If we're already past the refresh point, don't spin — wait 60s before retrying
tokenRefreshTimer = setTimeout(async () => {
await refreshPlatformTokenViaChainctl();
}, Math.max(60000, delay));
}
// (Startup platform-token seeding is relocated below, after the registry-auth
// helpers, so it can use the GCP workload-identity login path without a TDZ.)
// ── Chainguard Libraries registry auth (libraries.cgr.dev) ──────────────────
// Unlike console-api, the registry doesn't want a pre-exchanged token: it takes
// HTTP Basic where username = the assumable identity UIDP and password = the raw
// projected OIDC token (aud=issuer.enforce.dev) — the registry performs the STS
// exchange itself. This is what makes it work under K8s workload identity where
// chainctl can't mint the libraries.cgr.dev audience non-interactively.
// Set CHAINGUARD_IDENTITY (identity UIDP) + mount the projected token at
// SA_TOKEN_PATH. A user-supplied Basic credential from Settings still overrides,
// and CGR_REGISTRY_TOKEN (bearer) / chainctl are kept as fallbacks for other envs.
const CHAINGUARD_IDENTITY = process.env.CHAINGUARD_IDENTITY || '';
const SA_TOKEN_PATH = process.env.SA_TOKEN_PATH || '/var/run/chainguard/oidc/oidc-token';
const REGISTRY_AUDIENCE = 'libraries.cgr.dev';
// Cloud Run (and other GCP compute) has no projected-token file; instead we pull
// a Google-signed OIDC token from the instance metadata server. Detected via
// K_SERVICE (always set by Cloud Run); override with CGR_AMBIENT=gcp elsewhere.
const ON_GCP = !!process.env.K_SERVICE || (process.env.CGR_AMBIENT || '').toLowerCase() === 'gcp';
const METADATA_ID_TOKEN_URL = 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity';
const WORKLOAD_OIDC_AUDIENCE = 'https://issuer.enforce.dev';
let chainctlLoginAt = 0; // last successful ambient login (ms)
let chainctlLoginInFlight = null; // single-flight guard
let registryToken = process.env.CGR_REGISTRY_TOKEN || null;
let registryTokenExpiry = registryToken ? parsePlatformTokenExpiry(registryToken) : null;
let registryHeaderCache = { header: null, refreshAt: 0 }; // Basic(UIDP:oidc), rebuilt every 60s
async function readProjectedToken() {
try { return (await Bun.file(SA_TOKEN_PATH).text()).trim(); }
catch { return null; }
}
// Google-signed OIDC token for the runtime service account (Cloud Run / GCE).
// subject = the SA's numeric ID, which the assumable identity's claim_match
// validates; aud = issuer.enforce.dev (what the registry STS and chainctl want).
async function fetchGcpIdToken(audience = WORKLOAD_OIDC_AUDIENCE) {
try {
const res = await fetch(`${METADATA_ID_TOKEN_URL}?audience=${encodeURIComponent(audience)}&format=full`,
{ headers: { 'Metadata-Flavor': 'Google' }, signal: AbortSignal.timeout(2000) });
if (!res.ok) throw new Error(`metadata HTTP ${res.status}`);
return (await res.text()).trim();
} catch (err) {
console.warn('[cgr] GCP metadata token fetch failed:', err.message);
return null;
}
}
// The OIDC token used as the registry Basic password: the K8s projected token in
// the lab, else the GCP metadata token on Cloud Run. The registry does the STS
// exchange itself, so no chainctl needed for registry reads.
async function getWorkloadOidcToken() {
const projected = await readProjectedToken();
if (projected) return projected;
if (ON_GCP) return await fetchGcpIdToken();
return null;
}
// console-api needs a *Chainguard* bearer token (an STS exchange), which chainctl
// (bundled in the image) performs given the GCP metadata token + identity UIDP.
// We log in once and re-login well before the ~1h expiry; afterwards the existing
// `chainctl auth token` calls succeed. No-op off GCP (the lab uses the file path).
async function ensureChainctlLogin() {
if (!ON_GCP || !CHAINGUARD_IDENTITY) return false;
if (Date.now() - chainctlLoginAt < 45 * 60 * 1000) return true;
if (chainctlLoginInFlight) return chainctlLoginInFlight;
chainctlLoginInFlight = (async () => {
const token = await fetchGcpIdToken();
if (!token) return false;
const tmp = `/tmp/cgr-oidc-${process.pid}`;
try {
await Bun.write(tmp, token);
const proc = Bun.spawn(['chainctl', 'auth', 'login', '--headless',
'--identity-token', tmp, '--identity', CHAINGUARD_IDENTITY,
'--audience', 'https://console-api.enforce.dev', '--audience', REGISTRY_AUDIENCE],
{ stdout: 'pipe', stderr: 'pipe' });
const errText = await new Response(proc.stderr).text();
const code = await proc.exited;
if (code !== 0) throw new Error(`chainctl login exited ${code}: ${errText.trim().slice(0, 300) || '(no stderr)'}`);
chainctlLoginAt = Date.now();
console.log('[cgr] chainctl logged in via GCP workload identity');
return true;
} catch (err) {
console.error('[cgr] chainctl ambient login failed:', err.message);
return false;
} finally {
await unlink(tmp).catch(() => {});
}
})();
chainctlLoginInFlight.finally(() => { chainctlLoginInFlight = null; });
return chainctlLoginInFlight;
}
let registryRefreshInFlight = null;
function refreshRegistryTokenViaChainctl() {
if (registryRefreshInFlight) return registryRefreshInFlight;
registryRefreshInFlight = (async () => {
try {
if (ON_GCP) await ensureChainctlLogin(); // establish an ambient session first
const proc = Bun.spawn(['chainctl', 'auth', 'token', '--audience', REGISTRY_AUDIENCE], { stdout: 'pipe', stderr: 'pipe' });
const [text, errText] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()]);
const code = await proc.exited;
if (code !== 0) throw new Error(`chainctl exited ${code}: ${errText.trim().slice(0, 300) || '(no stderr)'}`);
const token = text.trim();
if (!token) throw new Error('empty token');
registryToken = token;
registryTokenExpiry = parsePlatformTokenExpiry(token);
return token;
} catch (err) {
console.error('chainctl registry token refresh failed:', err.message);
return null;
}
})();
registryRefreshInFlight.finally(() => { registryRefreshInFlight = null; });
return registryRefreshInFlight;
}
async function ensureRegistryTokenFresh(minMs = 5 * 60 * 1000) {
if (!registryToken || (registryTokenExpiry && registryTokenExpiry - Date.now() < minMs)) {
await refreshRegistryTokenViaChainctl();
}
return registryToken;
}
// Authorization header for a libraries.cgr.dev request. Priority:
// 1. user-supplied Basic creds from Settings,
// 2. workload identity: Basic(UIDP : projected OIDC token) — registry does STS,
// 3. bearer from CGR_REGISTRY_TOKEN env or a chainctl-minted registry token.
async function cgrHeaders(req) {
const user = req.headers.get('x-cgr-user') || '';
const pass = req.headers.get('x-cgr-pass') || '';
if (user) return { 'Authorization': 'Basic ' + Buffer.from(`${user}:${pass}`).toString('base64') };
if (CHAINGUARD_IDENTITY) {
const now = Date.now();
if (registryHeaderCache.header && now < registryHeaderCache.refreshAt) return { 'Authorization': registryHeaderCache.header };
const idToken = await getWorkloadOidcToken(); // K8s projected file, or GCP metadata on Cloud Run
if (idToken) {
const header = 'Basic ' + Buffer.from(`${CHAINGUARD_IDENTITY}:${idToken}`).toString('base64');
registryHeaderCache = { header, refreshAt: now + 60000 };
return { 'Authorization': header };
}
console.warn(`[cgr] CHAINGUARD_IDENTITY set but no workload OIDC token (no file at ${SA_TOKEN_PATH}${ON_GCP ? ', and GCP metadata fetch failed' : ''})`);
}
const token = await ensureRegistryTokenFresh();
return token ? { 'Authorization': `Bearer ${token}` } : {};
}
// Seed the platform token: from env, else mint via chainctl. On Cloud Run,
// refreshPlatformTokenViaChainctl establishes the GCP workload-identity session
// (ensureChainctlLogin) before minting. Relocated here so it runs after the
// registry-auth helpers are initialized.
if (platformToken) {
platformTokenExpiry = parsePlatformTokenExpiry(platformToken);
if (platformTokenExpiry) scheduleTokenRefresh();
} else {
refreshPlatformTokenViaChainctl().then(t => {
if (t) console.log('Platform token auto-minted via chainctl on startup');
else console.log('No platform token — paste one in Settings or mount chainctl config');
});
}
// ── Malware sync ──────────────────────────────────────────────────────────────
const syncState = { running: false, fetched: 0, total: 0, error: null, startedAt: null, finishedAt: null, windowsDone: 0, windowsTotal: 0, cancelled: false, expectedTotal: 0 };
// "Warm" once the local mirror holds a usable dataset — either persisted from a
// previous run or filled by a completed sync. While cold (first fill in progress),
// per-package checks are served live so the tool is useful immediately.
let malwareWarm = db.prepare(`SELECT COUNT(*) AS n FROM malware`).get().n > 0;
function malwareStatus() {
const counts = db.prepare(`SELECT ecosystem, COUNT(*) AS n, MAX(blocked_at) AS latest FROM malware GROUP BY ecosystem`).all();
const byEco = Object.fromEntries(counts.map(r => [r.ecosystem, { total: r.n, latest: r.latest }]));
const total = counts.reduce((s, r) => s + r.n, 0);
const lastSync = db.prepare(`SELECT value FROM sync_meta WHERE key = 'last_sync_at'`).get();
const tokenStatus = platformToken
? { set: true, expiresAt: platformTokenExpiry ? new Date(platformTokenExpiry).toISOString() : null, audienceOk: tokenAudienceOk(platformToken) }
: { set: false };
const enrichCounts = db.prepare(`
SELECT
COUNT(*) AS total,
COUNT(CASE WHEN published_at IS NOT NULL AND published_at NOT IN ('NOT_FOUND','ERROR') THEN 1 END) AS enriched,
COUNT(CASE WHEN published_at IS NULL THEN 1 END) AS pending,
COUNT(CASE WHEN published_at IN ('NOT_FOUND','ERROR') THEN 1 END) AS unavailable
FROM malware
`).get();
const vexLastSync = db.prepare(`SELECT value FROM sync_meta WHERE key = 'vex_last_sync_at'`).get();
const vexCounts = db.prepare(`SELECT ecosystem, COUNT(*) AS n, COUNT(DISTINCT package_name) AS pkgs FROM vex GROUP BY ecosystem`).all();
// `cves` = distinct CVE ids fixed (one CVE often spans many packages/builds);
// `fixes` = distinct (package × vulnerability); `total` = raw rows (also spread
// across build versions); `packages` = distinct packages with ≥1 backport.
const vexCves = db.prepare(`SELECT COUNT(DISTINCT cve) AS n FROM vex WHERE cve IS NOT NULL AND cve != ''`).get().n;
const vexFixes = db.prepare(`SELECT COUNT(*) AS n FROM (SELECT 1 FROM vex GROUP BY ecosystem, package_name, vuln_name)`).get().n;
const vexPackages = db.prepare(`SELECT COUNT(*) AS n FROM (SELECT 1 FROM vex GROUP BY ecosystem, package_name)`).get().n;
const vex = {
warm: vexWarm,
cves: vexCves,
fixes: vexFixes,
packages: vexPackages,
total: vexCounts.reduce((s, r) => s + r.n, 0),
byEco: Object.fromEntries(vexCounts.map(r => [r.ecosystem, { statements: r.n, packages: r.pkgs }])),
lastSyncAt: vexLastSync?.value || null,
};
return { total, byEco, lastSyncAt: lastSync?.value || null, sync: { ...syncState }, platformToken: tokenStatus, enrich: { ...enrichCounts, state: { ...enrichState } }, vex };
}
const SCOPE_NORM = { 'MALWARE_SCOPE_VERSION': 'version', 'MALWARE_SCOPE_PACKAGE': 'package', 'MALWARE_SCOPE_UNKNOWN': '' };
function normScope(s) { return SCOPE_NORM[s] ?? s ?? ''; }
function insertItems(items, ecoName) {
const tx = db.transaction(rows => {
for (const it of rows) {
insertMalware.run(
it.package_name ?? it.packageName,
it.version ?? '',
normScope(it.scope),
it.malid ?? '',
it.source ?? null,
it.blocked_at ?? it.blockedAt,
ecoName, // always the queried ecosystem — the API returns inconsistent casing
// (e.g. 'pypi' vs 'PyPI') in the item field, which the read path never matches
JSON.stringify(it.reason || []),
it.description ?? null,
);
}
});
tx(items);
syncState.fetched += items.length;
}
const CONSOLE_API_BASE = 'https://console-api.enforce.dev';
const MALWARE_API_BASE = `${CONSOLE_API_BASE}/libraries/v1/malware/blocklist`;
const MALWARE_EPOCH = '2026-01-01T00:00:00Z';
// Generic console-api GET with 401→chainctl-refresh and transient-5xx/429 retry.
async function consoleApiGet(path, params, ctx = '') {
const qs = params ? `?${params}` : '';
for (let attempt = 1; ; attempt++) {
let res = await fetch(`${CONSOLE_API_BASE}${path}${qs}`, { headers: { Authorization: `Bearer ${platformToken}` } });
if (res.status === 401) {
const refreshed = await refreshPlatformTokenViaChainctl();
if (!refreshed) throw new Error(`HTTP 401 from console-api (${ctx}) — token refresh failed`);
res = await fetch(`${CONSOLE_API_BASE}${path}${qs}`, { headers: { Authorization: `Bearer ${platformToken}` } });
}
if (res.ok) return res.json();
if ((res.status >= 500 || res.status === 429) && attempt <= 3) { await new Promise(r => setTimeout(r, 1000 * attempt)); continue; }
const body = await res.text().catch(() => '');
throw new Error(`HTTP ${res.status} from console-api (${ctx})${body ? `: ${body.slice(0, 200)}` : ''}`);
}
}
// ── Libraries cooldown policy ───────────────────────────────────────────────
// Chainguard withholds package versions younger than a per-ecosystem "cooldown"
// window (age_days < cooldown_days). The window comes from the org's policy
// bindings; we resolve it once and cache it so the UI can badge in-cooldown
// versions. Set LIBRARIES_ORG to the org name or group UIDP to enable this;
// when unset the cooldown feature is simply off (no badges).
const LIBRARIES_ORG = process.env.LIBRARIES_ORG || null;
const POLICY_ECO_MAP = { JAVASCRIPT: 'npm', PYTHON: 'pypi', JAVA: 'maven' };
const POLICY_TTL = 5 * 60 * 1000;
let policyCache = { at: 0, data: null };
// Fallback cooldown windows for when the API returns no policy bindings — e.g. a
// K8s workload identity that can't read Libraries policies. Per-ecosystem
// COOLDOWN_DAYS_NPM/PYPI/MAVEN, else the uniform COOLDOWN_DAYS. (0/unset = none.)
const COOLDOWN_ENV = {
npm: Number(process.env.COOLDOWN_DAYS_NPM) || null,
pypi: Number(process.env.COOLDOWN_DAYS_PYPI) || null,
maven: Number(process.env.COOLDOWN_DAYS_MAVEN) || null,
};
const COOLDOWN_DAYS_DEFAULT = Number(process.env.COOLDOWN_DAYS) || null;
async function resolveLibrariesParent() {
if (/^[0-9a-f]{40}$/i.test(LIBRARIES_ORG)) return LIBRARIES_ORG;
const data = await consoleApiGet('/iam/v1/groups', new URLSearchParams({ name: LIBRARIES_ORG }), 'group resolve');
return (data.items || []).find(g => g.name === LIBRARIES_ORG)?.id || null;
}
async function librariesPolicyData() {
if (policyCache.data && Date.now() - policyCache.at < POLICY_TTL) return policyCache.data;
const result = { enabled: false, org: LIBRARIES_ORG, ecosystems: {} };
if (!LIBRARIES_ORG) return result; // no org configured → feature off
if (!platformToken) return result; // no token → feature off (don't cache)
try {
const parent = await resolveLibrariesParent();
if (!parent) { console.warn(`[libraries-policy] could not resolve org '${LIBRARIES_ORG}'`); return result; }
const [bindings, policies] = await Promise.all([
consoleApiGet('/libraries/v1/policy-bindings', new URLSearchParams({ parent_id: parent }), 'policy-bindings'),
consoleApiGet('/libraries/v1/policies', new URLSearchParams({ parent_id: parent }), 'policies'),
]);
const byId = new Map((policies.items || []).map(p => [p.id, p]));
for (const b of (bindings.items || [])) {
const eco = POLICY_ECO_MAP[b.ecosystem];
if (!eco) continue;
const pol = byId.get(b.policy);
result.ecosystems[eco] = {
cooldownDays: pol?.cooldownDays ?? 7, // Chainguard system default when unset; 0 disables
policyName: pol?.name || null,
mode: (b.mode || '').replace('LIBRARY_POLICY_BINDING_MODE_', ''),
};
}
const fromApi = Object.keys(result.ecosystems);
// Fill any ecosystem the API didn't return (e.g. identity lacks policy-read)
// from the env fallback so the UI can still badge in-cooldown versions.
for (const eco of ['npm', 'pypi', 'maven']) {
if (result.ecosystems[eco]) continue;
const days = COOLDOWN_ENV[eco] ?? COOLDOWN_DAYS_DEFAULT;
if (days != null) result.ecosystems[eco] = { cooldownDays: days, policyName: 'env fallback', mode: 'DEFAULT' };
}
console.log(`[libraries-policy] parent=${parent} bindings=${(bindings.items || []).length} policies=${(policies.items || []).length}; from API: [${fromApi.join(',') || 'none'}]; effective: [${Object.keys(result.ecosystems).join(',') || 'none'}]`);
result.enabled = Object.keys(result.ecosystems).length > 0;
result.parent = parent;
policyCache = { at: Date.now(), data: result };
} catch (err) {
console.error('[libraries-policy] fetch failed:', err.message);
}
return result;
}
// Parse a Package URL (purl) -> { type, name, version }; handles percent-encoded
// scoped npm names and maven namespace/name joined with '/'.
function parsePurl(purl) {
const m = /^pkg:([^/]+)\/(.+)$/.exec(purl || '');
if (!m) return null;
const rest = m[2].split('?')[0];
const at = rest.lastIndexOf('@');
if (at < 0) return null;
return { type: m[1], name: decodeURIComponent(rest.slice(0, at)), version: decodeURIComponent(rest.slice(at + 1)) };
}
// Canonicalise a package name so maven's group:artifact and purl's group/artifact compare equal.
const canonName = n => (n || '').replace(/[:/]+/g, '/');
// Authoritative per-version unblock times from the org's own blocked-pull events
// (cooldown reason only). Sparse - only versions the org actually pulled. 60s memo.
const AUTH_COOLDOWN_TTL = 60 * 1000;
const POLICY_ENUM = { npm: 'JAVASCRIPT', pypi: 'PYTHON', maven: 'JAVA' };
const authCooldownCache = new Map(); // `${eco} ${pkg}` -> { at, map }
async function authoritativeCooldown(eco, pkg) {
if (!LIBRARIES_ORG || !platformToken || !pkg) return {};
const cacheKey = `${eco} ${pkg}`;
const hit = authCooldownCache.get(cacheKey);
if (hit && Date.now() - hit.at < AUTH_COOLDOWN_TTL) return hit.map;
const map = {};
try {
const parent = await resolveLibrariesParent();
if (!parent) return map;
const filterName = eco === 'maven' ? pkg.split(':').pop() : pkg; // purl name = artifactId for maven
const data = await consoleApiGet('/libraries/v1/blocked-packages',
new URLSearchParams({ parent_id: parent, package_name: filterName, page_size: '1000' }), `blocked ${eco} ${pkg}`);
const want = canonName(pkg);
for (const it of (data.items || [])) {
if (it.reason !== 'cooldown' || !it.unblocksAt) continue;
if (POLICY_ENUM[eco] && it.ecosystem && it.ecosystem !== POLICY_ENUM[eco]) continue;
const p = parsePurl(it.purl);
if (!p || canonName(p.name) !== want) continue;
map[p.version] = it.unblocksAt;
}
authCooldownCache.set(cacheKey, { at: Date.now(), map });
} catch (err) {
console.error(`[cooldown] blocked-packages fetch failed for ${eco} ${pkg}: ${err.message}`);
}
return map;
}
// Distinct-source facet cache (see /api/cgr-malware/sources).
const SOURCES_TTL = 60 * 1000;
let sourcesCache = { at: 0, rows: null };
// Identity key matching the malware PRIMARY KEY (normalised the same way insertItems stores).
const malKey = (pkg, ver, malid, blockedAt) => `${pkg}\u0000${ver ?? ''}\u0000${malid ?? ''}\u0000${blockedAt}`;
// GET one blocklist page with 401→chainctl-refresh and transient-5xx/429 retry. Returns parsed JSON.
async function malwareApiGet(params, ctx = '') {
for (let attempt = 1; ; attempt++) {
let res = await fetch(`${MALWARE_API_BASE}?${params}`, { headers: { Authorization: `Bearer ${platformToken}` } });
if (res.status === 401) {
console.log(`Token expired mid-sync${ctx ? ` (${ctx})` : ''}, attempting chainctl refresh…`);
const refreshed = await refreshPlatformTokenViaChainctl();
if (!refreshed) throw new Error('HTTP 401 from Platform API — token expired and chainctl refresh failed');
res = await fetch(`${MALWARE_API_BASE}?${params}`, { headers: { Authorization: `Bearer ${platformToken}` } });
}
if (res.ok) return res.json();
// Non-OK: capture body + context so recurrences are diagnosable.
const bodyText = await res.text().catch(() => '<unreadable body>');
const reqId = res.headers.get('x-request-id') || res.headers.get('x-amzn-requestid') || res.headers.get('cf-ray') || null;
console.error(`[malware-sync] HTTP ${res.status} from Platform API — ${ctx}${reqId ? ` reqId=${reqId}` : ''}\n url: ${MALWARE_API_BASE}?${params}\n body: ${bodyText.slice(0, 1000)}`);
if ((res.status >= 500 || res.status === 429) && attempt <= 3) {
const backoff = 1000 * attempt;
console.warn(`[malware-sync] transient ${res.status}, retrying in ${backoff}ms (attempt ${attempt}/3)…`);
await new Promise(r => setTimeout(r, backoff));
continue;
}
let msg = `HTTP ${res.status} from Platform API`;
if (bodyText && bodyText !== '<unreadable body>') msg += `: ${bodyText.slice(0, 300)}`;
throw new Error(msg);
}
}
// Authoritative count of entries with blocked_at >= since (the API respects `since`, ignores `until`).
async function malwareTotalCountSince(apiName, since) {
const data = await malwareApiGet(new URLSearchParams({ ecosystem: apiName, pageSize: '1', since }), `${apiName} count ${since.slice(0, 10)}`);
return Number(data.totalCount || 0);
}
// Page through every entry with blocked_at >= since (newest-first), invoking onPage(items) per page.
async function malwarePageThrough(apiName, since, onPage) {
let pageToken = null;
while (!syncState.cancelled) {
const params = new URLSearchParams({ ecosystem: apiName, pageSize: '500', since });
if (pageToken) params.set('pageToken', pageToken);
const data = await malwareApiGet(params, `${apiName} since=${since}`);
const items = data.items || [];
onPage(items);
if (!data.nextPageToken || items.length === 0) break;
pageToken = data.nextPageToken;
}
}
// Monthly boundaries [epoch, …, now+1d] used to localise which month(s) changed.
function malwareMonthBoundaries() {
const bs = [];
let cursor = new Date(MALWARE_EPOCH);
const end = new Date(Date.now() + 86400000);
while (cursor < end) { bs.push(cursor.toISOString()); const n = new Date(cursor); n.setUTCMonth(n.getUTCMonth() + 1); cursor = n; }
bs.push(end.toISOString());
return bs;
}
// The delta add-pass only inserts, so upstream REMOVALS (e.g. cleared false positives)
// aren't dropped locally — and the blocklist API has no removal/updated_at feed. We detect
// them by comparing counts within a recent HORIZON window only (removals are recent):
// - Windowing (not global totals) avoids re-triggering forever on ancient drift and on
// totalCount noise, which fluctuates ±tens on the live ~351k npm feed.
// - A small margin absorbs that noise + the add-pass racing the live feed, so only a
// genuine removal batch (local exceeding upstream in-window beyond the noise) reconciles.
// When it does fire, it re-fetches [horizon → now], upserts (preserves published_at), and
// deletes any local row in-window absent upstream. Older drift is a job for a full resync.
async function reconcileMalwareRemovals(apiName, dbName) {
const HORIZON_DAYS = 21;
const horizon = new Date(Date.now() - HORIZON_DAYS * 86400000).toISOString();
const upstreamRecent = await malwareTotalCountSince(apiName, horizon);
const localRecent = db.prepare(`SELECT COUNT(*) AS n FROM malware WHERE ecosystem = ? AND blocked_at >= ?`).get(dbName, horizon).n;
const margin = Math.max(25, Math.round(upstreamRecent * 0.001));
// local exceeding upstream in-window beyond the noise margin ⇒ real removals to reconcile.
// Otherwise it's just new adds / count jitter → nothing to do (add-pass handles adds).
if (localRecent <= upstreamRecent + margin) return;
const repairFrom = horizon;
const expected = upstreamRecent;
console.log(`[${dbName}] reconcile: recent-window local ${localRecent} > upstream ${upstreamRecent} (+${margin} margin) — re-syncing ${repairFrom.slice(0, 10)} → now`);
// Fetch the repair range. CRITICAL: only proceed to the delete step if the page-through
// was COMPLETE — an incomplete fetch (e.g. truncated by a mid-flight token refresh) would
// make us delete rows that actually exist upstream. Verify fetched vs expected; retry on short.
const tolerance = Math.max(20, Math.round(expected * 0.01));
let upstreamKeys = null;
for (let attempt = 1; attempt <= 3 && !syncState.cancelled; attempt++) {
await ensurePlatformTokenFresh(); // avoid a mid-page-through 401 on long ranges
const keys = new Set();
await malwarePageThrough(apiName, repairFrom, (items) => {
for (const it of items) keys.add(malKey(it.packageName, it.version, it.malid, it.blockedAt));
insertItems(items, dbName);
});
if (syncState.cancelled) return;
console.log(`[${dbName}] reconcile: fetched ${keys.size} unique keys (expected ~${expected}, gate ${expected - tolerance}) attempt ${attempt}`);
if (keys.size >= expected - tolerance) { upstreamKeys = keys; break; }
console.warn(`[${dbName}] reconcile: fetched ${keys.size} < expected ${expected} (attempt ${attempt}/3) — retrying before delete`);
}
if (!upstreamKeys) {
// Never delete against an incomplete set — leave the (already-upserted) adds and bail.
console.error(`[${dbName}] reconcile: fetch stayed short after retries — skipping delete to avoid over-removal; will retry next sync`);
return;
}
const localRows = db.prepare(`SELECT package_name, version, malid, blocked_at FROM malware WHERE ecosystem = ? AND blocked_at >= ?`).all(dbName, repairFrom);
const delStmt = db.prepare(`DELETE FROM malware WHERE ecosystem = ? AND package_name = ? AND version = ? AND malid = ? AND blocked_at = ?`);
let removed = 0;
db.transaction(() => {
for (const r of localRows) {
if (!upstreamKeys.has(malKey(r.package_name, r.version, r.malid, r.blocked_at))) {
delStmt.run(dbName, r.package_name, r.version, r.malid, r.blocked_at);
removed++;
}
}
})();
console.log(`[${dbName}] reconcile: removed ${removed} stale record(s)`);
}
// Lazy per-package reconcile. The blocklist API has no removal feed, so a lone
// cleared entry (below the batch-reconcile noise margin) can linger locally until
// a full resync. But the API filters by packageName, so we can cheaply verify one
// package on demand: fetch its upstream entries, upsert them (ON CONFLICT preserves
// published_at), and drop any local row for that package no longer upstream. This
// self-heals the interactive "I'm looking at package X" case without a full sync.
// Best-effort: throws on API failure so the caller can fall back to local rows.
// A short per-(eco,pkg) TTL avoids hammering the API on repeat views.
const PKG_RECONCILE_TTL = 5 * 60 * 1000;
const pkgReconciledAt = new Map(); // `${ecoDbName}\u0000${pkg}` → last-reconciled epoch ms
// Fetch every upstream blocklist entry for one package (paginated).
async function fetchPackageBlocklist(apiName, pkg) {
const items = [];
let pageToken = null;
do {
const params = new URLSearchParams({ ecosystem: apiName, packageName: pkg, pageSize: '500' });
if (pageToken) params.set('pageToken', pageToken);
const data = await malwareApiGet(params, `pkg-check ${apiName} ${pkg}`);
for (const it of (data.items || [])) items.push(it);
pageToken = data.nextPageToken || null;
} while (pageToken);
return items;
}
// Live, read-only per-package lookup used while the mirror is cold (first fill in
// progress) so the UI shows correct results immediately instead of sparse local
// data. Returns rows in the same shape as the DB read path. 60s memo.
const LIVE_LOOKUP_TTL = 60 * 1000;
const liveLookupCache = new Map(); // `${dbName} ${pkg}` -> { at, rows }
async function liveMalwareLookup(dbName, apiName, pkg) {
if (!platformToken || !pkg) return null;
const key = `${dbName} ${pkg}`;
const hit = liveLookupCache.get(key);
if (hit && Date.now() - hit.at < LIVE_LOOKUP_TTL) return hit.rows;
const items = await fetchPackageBlocklist(apiName, pkg);
const rows = items.map(it => ({
package_name: it.packageName, version: it.version ?? '', scope: normScope(it.scope),
malid: it.malid ?? '', source: it.source ?? null, blocked_at: it.blockedAt,
reason: it.reason || [], description: it.description ?? null,
}));
liveLookupCache.set(key, { at: Date.now(), rows });
return rows;
}
async function reconcilePackage(ecoDbName, apiName, pkg) {
// Skip when unauthenticated (avoid a chainctl mint storm on the read path) or
// mid-sync (the running sync is already rewriting this ecosystem).
if (!platformToken || !pkg || syncState.running) return;
const cacheKey = `${ecoDbName}\u0000${pkg}`;
const last = pkgReconciledAt.get(cacheKey);
if (last && Date.now() - last < PKG_RECONCILE_TTL) return;
// Fetch every upstream entry for this package (paginated — single packages are
// usually one page, but long histories can exceed pageSize).
const items = await fetchPackageBlocklist(apiName, pkg);
const upstreamKeys = new Set(items.map(it => malKey(it.packageName, it.version, it.malid, it.blockedAt)));
// Reached only if the full page-through succeeded (malwareApiGet throws otherwise),
// so upstreamKeys is complete and safe to delete against.
const localRows = db.prepare(`SELECT version, malid, blocked_at FROM malware WHERE ecosystem = ? AND package_name = ?`).all(ecoDbName, pkg);
const delStmt = db.prepare(`DELETE FROM malware WHERE ecosystem = ? AND package_name = ? AND version = ? AND malid = ? AND blocked_at = ?`);
let inserted = 0, removed = 0;
db.transaction(() => {
for (const it of items) {
const res = insertMalware.run(
it.packageName, it.version ?? '', normScope(it.scope), it.malid ?? '',
it.source ?? null, it.blockedAt, ecoDbName,
JSON.stringify(it.reason || []), it.description ?? null,
);
if (res.changes) inserted++;
}
for (const r of localRows) {
if (!upstreamKeys.has(malKey(pkg, r.version, r.malid, r.blocked_at))) {
delStmt.run(ecoDbName, pkg, r.version, r.malid, r.blocked_at);
removed++;
}
}
})();
pkgReconciledAt.set(cacheKey, Date.now());
if (removed || inserted) console.log(`[pkg-check] reconciled ${ecoDbName} ${pkg}: +${inserted} upserted, -${removed} stale`);
}
async function runMalwareSync({ token, full = false }) {
if (syncState.running) throw new Error('Sync already in progress');
if (!token) throw new Error('No platform token available');
Object.assign(syncState, {
running: true, fetched: 0, total: 0, error: null,
startedAt: new Date().toISOString(), finishedAt: null,
windowsDone: 0, windowsTotal: 0, cancelled: false, expectedTotal: 0,
});
await ensurePlatformTokenFresh(); // avoid a mid-sync 401 truncating a page-through
// For a full/cold rebuild, probe the upstream total up front so /readyz can
// report a warmup percentage/ETA. Cheap (one pageSize=1 call per ecosystem);
// skipped for delta syncs where it isn't worth the latency.
if (full || !malwareWarm) {
try {
let expected = 0;
for (const { apiName } of PLATFORM_ECOSYSTEMS) expected += await malwareTotalCountSince(apiName, MALWARE_EPOCH);
syncState.expectedTotal = expected;
} catch { /* best-effort; percentage just stays unknown */ }
}
async function syncOneEcosystem({ apiName, dbName }) {
if (syncState.cancelled) return;
let savedPubDates = null;
if (full) {
savedPubDates = db.prepare(`SELECT package_name, version, published_at FROM malware WHERE ecosystem = ? AND published_at IS NOT NULL`).all(dbName);
db.prepare(`DELETE FROM malware WHERE ecosystem = ?`).run(dbName);
}
// Add-pass: first sync (or full) pulls everything since the epoch; later syncs pull
// only entries newer than our latest known blocked_at. (API is newest-first, `since`
// inclusive, `until` ignored.)
const latest = full ? null : db.prepare(`SELECT MAX(blocked_at) AS m FROM malware WHERE ecosystem = ?`).get(dbName)?.m;
syncState.windowsTotal += 1;
await malwarePageThrough(apiName, latest || MALWARE_EPOCH, (items) => insertItems(items, dbName));
syncState.windowsDone += 1;
// Reconcile removals (skipped implicitly on `full` since the add-pass already rebuilt the set).
if (!syncState.cancelled) await reconcileMalwareRemovals(apiName, dbName);
if (full && savedPubDates?.length) {
const restoreStmt = db.prepare(`UPDATE malware SET published_at = ? WHERE ecosystem = ? AND package_name = ? AND version = ?`);
db.transaction(() => { for (const row of savedPubDates) restoreStmt.run(row.published_at, dbName, row.package_name, row.version); })();
}
}
try {
// Ecosystems run sequentially — the blocklist API's pagination cursors proved
// unreliable when several page-throughs ran concurrently (crossed/short reads).
for (const eco of PLATFORM_ECOSYSTEMS) {
if (syncState.cancelled) break;
await syncOneEcosystem(eco);
}
db.prepare(`INSERT OR REPLACE INTO sync_meta (key, value) VALUES ('last_sync_at', ?)`).run(new Date().toISOString());
if (!syncState.cancelled) malwareWarm = true; // mirror is now usable from local
// Always fetch publish dates after a sync — the enrich pass only touches rows
// with published_at IS NULL, so this is a no-op once everything is enriched.
// Fire-and-forget; progress is exposed via /api/cgr-malware/enrich/status.
if (!syncState.cancelled && !enrichState.running) {
runMalwareEnrich().catch(err => console.error('[malware-sync] auto-enrich failed:', err.message));
}
} catch (err) {
syncState.error = err.message;
throw err;
} finally {
syncState.running = false;
syncState.finishedAt = new Date().toISOString();
}
}
// ── Scheduled sync ──────────────────────────────────────────────────────────
// Daily full resync at 03:00 (server-local time; set the TZ env var to control
// the zone). Disable with MALWARE_AUTOSYNC=off.
async function triggerScheduledSync({ full }) {
const kind = full ? 'full' : 'delta';
if (syncState.running) { console.log(`[scheduler] ${kind} sync skipped — a sync is already running`); return; }
await ensurePlatformTokenFresh();
if (!platformToken) { console.warn(`[scheduler] ${kind} sync skipped — no platform token (chainctl/OIDC or paste one)`); return; }
try {
console.log(`[scheduler] starting ${kind} sync`);
await runMalwareSync({ token: platformToken, full }); // auto-enriches on success
console.log(`[scheduler] ${kind} sync finished`);
} catch (err) {
console.error(`[scheduler] ${kind} sync failed:`, err.message);
}
}
function scheduleMalwareJobs() {
if ((process.env.MALWARE_AUTOSYNC || '').toLowerCase() === 'off') {
console.log('[scheduler] auto-sync disabled (MALWARE_AUTOSYNC=off)');
return;
}
// Daily full resync at 03:00 local time; reschedule after each run so it
// survives DST shifts and doesn't drift.
const scheduleDailyFull = () => {
const now = new Date();
const next = new Date(now);
next.setHours(3, 0, 0, 0);
if (next <= now) next.setDate(next.getDate() + 1);
const delay = next - now;
console.log(`[scheduler] next full resync at ${next.toString()} (in ${Math.round(delay / 60000)} min)`);
setTimeout(async () => {
await triggerScheduledSync({ full: true });
await runVexSync(); // refresh the VEX mirror alongside the malware resync
scheduleDailyFull();
}, delay);
};
scheduleDailyFull();
}
scheduleMalwareJobs();
const statsCache = new Map(); // key → { result, expiresAt }
const STATS_TTL = 30000; // 30 seconds