Skip to content

Commit 3656f45

Browse files
tlwillketlwillke
committed
Add quantization timing metrics, prefix selection, and logging schema support.
1 parent c983d5f commit 3656f45

11 files changed

Lines changed: 462 additions & 101 deletions

File tree

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java‎

Lines changed: 220 additions & 42 deletions
Large diffs are not rendered by default.

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/BenchmarkTablePrinter.java‎

Lines changed: 63 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,39 +29,49 @@
2929
public class BenchmarkTablePrinter {
3030
private static final int MIN_COLUMN_WIDTH = 11;
3131
private static final int MIN_HEADER_PADDING = 3;
32+
private static final int MAX_COLUMN_WIDTH = 15; // tune as desired
3233

3334
private String headerFmt;
3435
private String rowFmt;
36+
private int[] colWidths;
3537

3638
public BenchmarkTablePrinter() {
3739
headerFmt = null;
3840
rowFmt = null;
3941
}
4042

41-
4243
private void initializeHeader(List<Metric> cols) {
4344
if (headerFmt != null) {
4445
return;
4546
}
47+
this.colWidths = new int[cols.size() + 1];
4648

4749
// Build the format strings for header & rows
4850
StringBuilder hsb = new StringBuilder();
4951
StringBuilder rsb = new StringBuilder();
5052

5153
// 1) Overquery column width
54+
// Overquery column width
5255
hsb.append("%-12s");
5356
rsb.append("%-12.2f");
57+
colWidths[0] = 12;
5458

5559
// 2) One column per Metric
60+
int i = 0;
5661
for (Metric m : cols) {
5762
String hdr = m.getHeader();
5863
String spec = m.getFmtSpec();
5964
int width = Math.max(MIN_COLUMN_WIDTH, hdr.length() + MIN_HEADER_PADDING);
65+
width = Math.min(width, MAX_COLUMN_WIDTH);
66+
67+
colWidths[i + 1] = width;
6068

6169
// Header: Always a string
6270
hsb.append(" %-").append(width).append("s");
6371
// Row: Use the Metric’s fmtSpec (e.g. ".2f", ".3f")
6472
rsb.append(" %-").append(width).append(spec);
73+
74+
i++;
6575
}
6676

6777
this.headerFmt = hsb.toString();
@@ -85,20 +95,34 @@ public void printConfig(Map<String, ?> params) {
8595
}
8696

8797
private void printHeader(List<Metric> cols) {
88-
// Prepare array: First "Overquery", then each Metric header
89-
Object[] hdrs = new Object[cols.size() + 1];
90-
hdrs[0] = "Overquery";
98+
// Two header lines: split long headers onto a second line when possible
99+
Object[] hdrs1 = new Object[cols.size() + 1];
100+
Object[] hdrs2 = new Object[cols.size() + 1];
101+
102+
hdrs1[0] = "Overquery";
103+
hdrs2[0] = "";
104+
105+
boolean anySecondLine = false;
106+
91107
for (int i = 0; i < cols.size(); i++) {
92-
hdrs[i + 1] = cols.get(i).getHeader();
108+
String hdr = cols.get(i).getHeader();
109+
int width = colWidths[i + 1];
110+
111+
String[] parts = splitHeader2(hdr, width);
112+
hdrs1[i + 1] = parts[0];
113+
hdrs2[i + 1] = parts[1];
114+
if (!parts[1].isEmpty()) anySecondLine = true;
115+
}
116+
117+
String line1 = String.format(Locale.US, headerFmt, hdrs1);
118+
System.out.println(line1);
119+
120+
if (anySecondLine) {
121+
String line2 = String.format(Locale.US, headerFmt, hdrs2);
122+
System.out.println(line2);
93123
}
94124

95-
// Print header line
96-
String line = String.format(Locale.US, headerFmt, hdrs);
97-
System.out.println(line);
98-
// Underline of same length
99-
System.out.println(String.join("",
100-
Collections.nCopies(line.length(), "-")
101-
));
125+
System.out.println(String.join("", Collections.nCopies(line1.length(), "-")));
102126
}
103127

104128
/**
@@ -129,4 +153,31 @@ public void printRow(double overquery,
129153
public void printFooter() {
130154
System.out.println();
131155
}
156+
157+
// Helper for splitting the header into two rows
158+
private static String[] splitHeader2(String hdr, int colWidth) {
159+
if (hdr == null) return new String[] { "", "" };
160+
161+
// Manual break: "Line1\nLine2"
162+
int nl = hdr.indexOf('\n');
163+
if (nl >= 0) {
164+
String a = hdr.substring(0, nl).trim();
165+
String b = hdr.substring(nl + 1).trim();
166+
return new String[] { a, b };
167+
}
168+
169+
// Leave a little slack for padding/alignment
170+
int max = Math.max(1, colWidth - MIN_HEADER_PADDING);
171+
172+
String s = hdr.trim();
173+
if (s.length() <= max) return new String[] { s, "" };
174+
175+
// Find last space before max; if none, hard-split
176+
int cut = s.lastIndexOf(' ', max);
177+
if (cut <= 0) cut = max;
178+
179+
String a = s.substring(0, cut).trim();
180+
String b = s.substring(cut).trim();
181+
return new String[] { a, b };
182+
}
132183
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/benchmarks/ThroughputBenchmark.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ public List<Metric> runBenchmark(
254254
var list = new ArrayList<Metric>();
255255
if (computeAvgQps) {
256256
list.add(Metric.of("search.throughput.avg_qps",
257-
"Avg QPS (of " + numTestRuns + ")",
257+
"Avg QPS\n (of " + numTestRuns + ")",
258258
formatAvgQps,
259259
avgQps));
260260

@@ -270,18 +270,18 @@ public List<Metric> runBenchmark(
270270
}
271271
if (computeMedianQps) {
272272
list.add(Metric.of("search.throughput.median_qps",
273-
"Median QPS (of " + numTestRuns + ")",
273+
"Median QPS\n (of " + numTestRuns + ")",
274274
formatMedianQps,
275275
medianQps));
276276
}
277277
if (computeMaxQps) {
278278
list.add(Metric.of("search.throughput.max_qps",
279-
"Max QPS (of " + numTestRuns + ")",
279+
"Max QPS\n (of " + numTestRuns + ")",
280280
formatMaxQps,
281281
maxQps));
282282

283283
list.add(Metric.of("search.throughput.min_qps",
284-
"Min QPS (of " + numTestRuns + ")",
284+
"Min QPS\n (of " + numTestRuns + ")",
285285
formatMaxQps,
286286
minQps));
287287
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/LoggingSchemaPlanner.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public static List<String> unionLoggingMetricKeys(RunConfig runCfg, List<MultiCo
4949
var ctx = ReportingSelectionResolver.Context.of("topK", Integer.toString(topK));
5050
var resolved = ReportingSelectionResolver.resolve(runCfg.logging, SearchReportingCatalog.catalog(), ctx);
5151
union.addAll(resolved.keys());
52+
expandPrefixes(union, resolved.prefixes(), allConfigs);
5253
}
5354

5455
// 2) Emit keys in canonical order, only if present in union
@@ -103,4 +104,61 @@ private static void addIfPresent(Set<String> union, List<String> ordered, String
103104
ordered.add(key);
104105
}
105106
}
107+
108+
private static void expandPrefixes(Set<String> union,
109+
Set<String> prefixes,
110+
List<MultiConfig> allConfigs) {
111+
if (prefixes == null || prefixes.isEmpty()) return;
112+
113+
// Collect quant types from YAML configs (domain is defined by yaml `compression.type` and `reranking`)
114+
Set<String> indexQuantTypes = new HashSet<>();
115+
Set<String> searchQuantTypes = new HashSet<>();
116+
boolean wantsNVQ = false;
117+
118+
for (MultiConfig cfg : allConfigs) {
119+
if (cfg == null) continue;
120+
121+
// construction compression types (PQ/BQ/None)
122+
if (cfg.construction != null && cfg.construction.compression != null) {
123+
for (var c : cfg.construction.compression) {
124+
if (c != null && c.type != null && !"None".equals(c.type)) {
125+
indexQuantTypes.add(c.type); // "PQ" or "BQ"
126+
}
127+
}
128+
}
129+
130+
// construction reranking types (FP/NVQ)
131+
if (cfg.construction != null && cfg.construction.reranking != null) {
132+
if (cfg.construction.reranking.contains("NVQ")) {
133+
wantsNVQ = true;
134+
}
135+
}
136+
137+
// search compression types (PQ/BQ/None)
138+
if (cfg.search != null && cfg.search.compression != null) {
139+
for (var c : cfg.search.compression) {
140+
if (c != null && c.type != null && !"None".equals(c.type)) {
141+
searchQuantTypes.add(c.type); // "PQ" or "BQ"
142+
}
143+
}
144+
}
145+
}
146+
147+
for (String p : prefixes) {
148+
if ("construction.index_quant_time_s".equals(p)) {
149+
for (String qt : indexQuantTypes) {
150+
union.add(p + "." + qt + ".compute_time_s");
151+
union.add(p + "." + qt + ".encoding_time_s");
152+
}
153+
if (wantsNVQ) {
154+
union.add(p + ".NVQ.compute_time_s"); // we intentionally do not time NVQ encode
155+
}
156+
} else if ("search.search_quant_time_s".equals(p)) {
157+
for (String qt : searchQuantTypes) {
158+
union.add(p + "." + qt + ".compute_time_s");
159+
union.add(p + "." + qt + ".encoding_time_s");
160+
}
161+
}
162+
}
163+
}
106164
}

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/ReportingSelectionResolver.java‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -90,20 +90,45 @@ public Catalog(Map<String, Map<String, List<String>>> benchmarkKeyTemplates,
9090

9191
/** Result: concrete keys + a YAML-ish label per key for warnings/errors. */
9292
public static final class ResolvedSelection {
93-
private final Set<String> keys;
93+
private final Set<String> keys; // exact Metric.key strings
94+
private final Set<String> prefixes; // prefix selectors like "construction.quant."
9495
private final Map<String, String> keyToSelector;
96+
private final Map<String, String> prefixToSelector;
9597

96-
public ResolvedSelection(Set<String> keys, Map<String, String> keyToSelector) {
98+
public ResolvedSelection(Set<String> keys,
99+
Set<String> prefixes,
100+
Map<String, String> keyToSelector,
101+
Map<String, String> prefixToSelector) {
97102
this.keys = Collections.unmodifiableSet(new HashSet<>(keys));
103+
this.prefixes = Collections.unmodifiableSet(new HashSet<>(prefixes));
98104
this.keyToSelector = Collections.unmodifiableMap(new HashMap<>(keyToSelector));
105+
this.prefixToSelector = Collections.unmodifiableMap(new HashMap<>(prefixToSelector));
99106
}
100107

108+
/** Exact selected Metric.key values. */
101109
public Set<String> keys() { return keys; }
102110

103-
/** For warnings/errors: returns something like "search.console.benchmarks.latency.P999". */
111+
/** Prefix selections. */
112+
public Set<String> prefixes() { return prefixes; }
113+
114+
/** True if key is explicitly selected OR matches any selected prefix. */
115+
public boolean matchesKey(String key) {
116+
if (keys.contains(key)) return true;
117+
for (String p : prefixes) {
118+
if (key.startsWith(p)) return true;
119+
}
120+
return false;
121+
}
122+
123+
/** For warnings/errors: returns selector if known, else the key itself. */
104124
public String selectorForKey(String key) {
105125
return keyToSelector.getOrDefault(key, key);
106126
}
127+
128+
/** For warnings/errors: returns selector if known, else the prefix itself. */
129+
public String selectorForPrefix(String prefix) {
130+
return prefixToSelector.getOrDefault(prefix, prefix);
131+
}
107132
}
108133

109134
// -----------------------------
@@ -180,11 +205,13 @@ public static void validateNamedMetricSelectionNames(MetricSelection metricsToSe
180205

181206
public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog catalog, Context ctx) {
182207
if (selection == null) {
183-
return new ResolvedSelection(Set.of(), Map.of());
208+
return new ResolvedSelection(Set.of(), Set.of(), Map.of(), Map.of());
184209
}
185210

186211
Set<String> keys = new HashSet<>();
212+
Set<String> prefixes = new HashSet<>();
187213
Map<String, String> keyToSelector = new HashMap<>();
214+
Map<String, String> prefixToSelector = new HashMap<>();
188215

189216
// Benchmarks (type/stat -> templates -> keys)
190217
if (selection.benchmarks != null && !selection.benchmarks.isEmpty()) {
@@ -207,18 +234,27 @@ public static ResolvedSelection resolve(BenchmarkSelection selection, Catalog ca
207234
if (selection.metrics != null && !selection.metrics.isEmpty()) {
208235
// caller should validate names pre-build, but keep this defensive
209236
validateNamedMetricSelectionNames(selection.metrics, catalog);
210-
237+
// Recognize @prefix
211238
for (var e : selection.metrics.entrySet()) {
212239
String category = e.getKey();
213240
for (String name : e.getValue()) {
214-
String k = substitute(catalog.namedMetricKeys.get(category).get(name), ctx);
215-
keys.add(k);
216-
keyToSelector.putIfAbsent(k, catalog.metricsYamlPrefix + "." + category + "." + name);
241+
String raw = catalog.namedMetricKeys.get(category).get(name);
242+
String k = substitute(raw, ctx);
243+
String selector = catalog.metricsYamlPrefix + "." + category + "." + name;
244+
245+
if (k != null && k.startsWith("@prefix:")) {
246+
String prefix = k.substring("@prefix:".length());
247+
prefixes.add(prefix);
248+
prefixToSelector.putIfAbsent(prefix, selector);
249+
} else {
250+
keys.add(k);
251+
keyToSelector.putIfAbsent(k, selector);
252+
}
217253
}
218254
}
219255
}
220256

221-
return new ResolvedSelection(keys, keyToSelector);
257+
return new ResolvedSelection(keys, prefixes, keyToSelector, prefixToSelector);
222258
}
223259

224260
private static List<String> expandBenchmarkStat(String type, String stat, Catalog catalog, Context ctx) {

‎jvector-examples/src/main/java/io/github/jbellis/jvector/example/reporting/SearchReportingCatalog.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,9 @@ private static Map<String, Map<String, String>> namedMetricKeys() {
9191
"file_count", "search.disk.file_count"
9292
),
9393
"construction", Map.of(
94-
"index_build_time_s", "construction.index_build_time_s",
95-
"encoding_time_s", "construction.encoding_time_s"
94+
"index_build_time_s","construction.index_build_time_s",
95+
"index_quant_time_s","@prefix:construction.index_quant_time_s",
96+
"search_quant_time_s","@prefix:search.search_quant_time_s"
9697
)
9798
);
9899
}

0 commit comments

Comments
 (0)