Skip to content

Commit e7d9b7c

Browse files
RoyLinRoyLin
RoyLin
authored and
RoyLin
committed
feat: Lightpanda integration and browser field rename
Major changes: - Integrate Lightpanda as default headless browser (59MB, <100ms startup) - Rename browser_backend field to browser across all SDKs - Add lightpanda_path configuration option - SDK packages default to enabling lightpanda feature - Add comprehensive architecture documentation to README - Add browser-related fields to Node.js SDK TypeScript types - Add example code for both SDKs
1 parent 197aca7 commit e7d9b7c

22 files changed

Lines changed: 2553 additions & 63 deletions

‎Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ path = "src/main.rs"
3333
[features]
3434
default = []
3535
headless = ["dep:chromiumoxide", "dep:which", "dep:zip"]
36+
# Lightpanda headless backend (Linux/macOS only; Windows unsupported by Lightpanda).
37+
# Depends on `headless` because it reuses chromiumoxide as the CDP client.
38+
lightpanda = ["headless"]
3639

3740
[dependencies]
3841
# Async runtime

‎justfile‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ build:
1919
build-headless:
2020
cargo build -p a3s-search --features headless
2121

22+
# Build with Lightpanda browser support (Linux/macOS only)
23+
build-lightpanda:
24+
cargo build -p a3s-search --features lightpanda
25+
2226
# Build release
2327
release:
2428
cargo build -p a3s-search --release
@@ -123,6 +127,103 @@ test:
123127
fi
124128
echo ""
125129

130+
# Run Lightpanda browser integration tests (requires: --features lightpanda, Linux/macOS)
131+
#
132+
# Prerequisites:
133+
# - Lightpanda binary in PATH, or LIGHTPANDA=/path/to/binary env var set,
134+
# or it will be auto-downloaded from GitHub releases on first run.
135+
#
136+
# Groups (all are --ignored by default, must opt-in):
137+
# binary — ensure_lightpanda() detects/downloads the binary
138+
# pool — BrowserPool lifecycle (start, shutdown, idempotency)
139+
# fetch — real page fetches via Lightpanda CDP
140+
# wait — WaitStrategy variants (Load, Delay, NetworkIdle, Selector)
141+
# ua — custom user-agent header
142+
# concur — concurrent tabs and semaphore limiting
143+
# engine — search engine integration (Google)
144+
# error — error handling (invalid URL, missing binary)
145+
# html — HTML structure validation
146+
#
147+
# Examples:
148+
# just test-lightpanda # run all groups
149+
# just test-lightpanda fetch # run fetch group only
150+
# LIGHTPANDA=/usr/local/bin/lightpanda just test-lightpanda
151+
test-lightpanda GROUP="":
152+
#!/usr/bin/env bash
153+
set -e
154+
BOLD='\033[1m'
155+
GREEN='\033[0;32m'
156+
RED='\033[0;31m'
157+
YELLOW='\033[0;33m'
158+
BLUE='\033[0;34m'
159+
DIM='\033[2m'
160+
RESET='\033[0m'
161+
162+
echo ""
163+
echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
164+
echo -e "${BOLD} 🐼 Lightpanda Browser Integration Tests${RESET}"
165+
echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
166+
167+
if [ -n "$LIGHTPANDA" ]; then
168+
echo -e " ${DIM}Binary: $LIGHTPANDA (from \$LIGHTPANDA)${RESET}"
169+
else
170+
echo -e " ${DIM}Binary: auto-detect / auto-download${RESET}"
171+
fi
172+
173+
# Build filter from group name
174+
if [ -n "{{GROUP}}" ]; then
175+
FILTER="lightpanda_tests::test_lightpanda_{{GROUP}}"
176+
echo -e " ${DIM}Filter: $FILTER${RESET}"
177+
else
178+
FILTER="lightpanda_tests"
179+
echo -e " ${DIM}Filter: all lightpanda tests${RESET}"
180+
fi
181+
echo ""
182+
183+
# Run and capture output
184+
if OUTPUT=$(cargo test \
185+
--features lightpanda \
186+
--test integration \
187+
-- "$FILTER" --ignored --nocapture 2>&1); then
188+
EXIT=0
189+
else
190+
EXIT=1
191+
fi
192+
193+
# Print output (strip ANSI)
194+
echo "$OUTPUT" | sed 's/\x1b\[[0-9;]*m//g' | grep -v "^$" | \
195+
while IFS= read -r line; do
196+
if echo "$line" | grep -qE "^test .*ok$"; then
197+
echo -e " ${GREEN}✓${RESET} ${DIM}$(echo $line | sed 's/^test //')${RESET}"
198+
elif echo "$line" | grep -qE "^test .*FAILED$"; then
199+
echo -e " ${RED}✗${RESET} ${RED}$(echo $line | sed 's/^test //')${RESET}"
200+
elif echo "$line" | grep -qE "^test .*ignored$"; then
201+
echo -e " ${YELLOW}○${RESET} ${DIM}$(echo $line | sed 's/^test //')${RESET}"
202+
elif echo "$line" | grep -qE "^test result:"; then
203+
: # skip — we print our own summary
204+
else
205+
echo " $line"
206+
fi
207+
done
208+
209+
# Summary
210+
RESULT=$(echo "$OUTPUT" | grep "^test result:" | tail -1)
211+
PASSED=$(echo "$RESULT" | grep -oE '[0-9]+ passed' | grep -oE '[0-9]+' || echo 0)
212+
FAILED=$(echo "$RESULT" | grep -oE '[0-9]+ failed' | grep -oE '[0-9]+' || echo 0)
213+
IGNORED=$(echo "$RESULT" | grep -oE '[0-9]+ ignored' | grep -oE '[0-9]+' || echo 0)
214+
215+
echo ""
216+
echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
217+
if [ "$EXIT" -ne 0 ] || [ "$FAILED" -gt 0 ]; then
218+
echo -e " ${RED}${BOLD}✗ FAILED${RESET} ${GREEN}$PASSED passed${RESET} ${RED}$FAILED failed${RESET} ${YELLOW}$IGNORED ignored${RESET}"
219+
else
220+
echo -e " ${GREEN}${BOLD}✓ PASSED${RESET} ${GREEN}$PASSED passed${RESET} ${YELLOW}$IGNORED ignored${RESET}"
221+
fi
222+
echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
223+
echo ""
224+
225+
exit $EXIT
226+
126227
# Run tests with headless feature
127228
test-headless:
128229
#!/usr/bin/env bash

‎sdk/node/Cargo.lock‎

Lines changed: 16 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/node/Cargo.toml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ publish = false
1111
crate-type = ["cdylib"]
1212

1313
[dependencies]
14-
a3s-search = { path = "../..", default-features = false, features = ["headless"] }
14+
a3s-search = { path = "../..", default-features = false, features = ["headless", "lightpanda"] }
1515
napi = { version = "2", features = ["async", "tokio_rt"] }
1616
napi-derive = "2"
1717
tokio = { version = "1", features = ["full"] }
@@ -20,6 +20,11 @@ url = "2"
2020
[build-dependencies]
2121
napi-build = "2"
2222

23+
[features]
24+
default = ["headless", "lightpanda"]
25+
headless = ["a3s-search/headless"]
26+
lightpanda = ["a3s-search/lightpanda", "headless"]
27+
2328
[profile.release]
2429
lto = true
2530
strip = "symbols"

‎sdk/node/examples/test-headless.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
/**
2+
* Example script to test headless browser engines.
3+
*
4+
* This script demonstrates using Google, Baidu, and BingChina engines
5+
* which require a headless browser (Chrome).
6+
*
7+
* On first run, Chrome for Testing will be automatically downloaded
8+
* to ~/.a3s/chromium/ if not already installed.
9+
*/
10+
11+
import { A3SSearch } from '../lib/index.js';
12+
13+
async function testGoogle() {
14+
console.log('\n' + '='.repeat(60));
15+
console.log('Testing Google engine (headless browser)');
16+
console.log('='.repeat(60));
17+
18+
const search = new A3SSearch();
19+
20+
try {
21+
const response = await search.search('rust programming language', {
22+
engines: ['google'],
23+
limit: 5,
24+
});
25+
26+
console.log(`\n✓ Search completed in ${response.durationMs}ms`);
27+
console.log(`✓ Found ${response.count} results`);
28+
29+
if (response.errors.length > 0) {
30+
console.log(`\n⚠ Errors: ${response.errors.length}`);
31+
for (const err of response.errors) {
32+
console.log(` - ${err.engine}: ${err.message}`);
33+
}
34+
}
35+
36+
if (response.results.length > 0) {
37+
console.log('\nTop results:');
38+
for (let i = 0; i < Math.min(3, response.results.length); i++) {
39+
const result = response.results[i];
40+
console.log(`\n${i + 1}. ${result.title}`);
41+
console.log(` URL: ${result.url}`);
42+
console.log(` Score: ${result.score.toFixed(2)}`);
43+
console.log(` Engines: ${result.engines.join(', ')}`);
44+
}
45+
} else {
46+
console.log('\n⚠ No results returned (Google may have served a CAPTCHA)');
47+
}
48+
} catch (error) {
49+
console.log(`\n✗ Error: ${error.message}`);
50+
throw error;
51+
}
52+
}
53+
54+
async function testBaidu() {
55+
console.log('\n' + '='.repeat(60));
56+
console.log('Testing Baidu engine (headless browser)');
57+
console.log('='.repeat(60));
58+
59+
const search = new A3SSearch();
60+
61+
try {
62+
const response = await search.search('人工智能', {
63+
engines: ['baidu'],
64+
limit: 5,
65+
});
66+
67+
console.log(`\n✓ Search completed in ${response.durationMs}ms`);
68+
console.log(`✓ Found ${response.count} results`);
69+
70+
if (response.results.length > 0) {
71+
console.log('\nTop results:');
72+
for (let i = 0; i < Math.min(3, response.results.length); i++) {
73+
const result = response.results[i];
74+
console.log(`\n${i + 1}. ${result.title}`);
75+
console.log(` URL: ${result.url}`);
76+
console.log(` Score: ${result.score.toFixed(2)}`);
77+
}
78+
}
79+
} catch (error) {
80+
console.log(`\n✗ Error: ${error.message}`);
81+
throw error;
82+
}
83+
}
84+
85+
async function testMixedEngines() {
86+
console.log('\n' + '='.repeat(60));
87+
console.log('Testing mixed engines (HTTP + headless)');
88+
console.log('='.repeat(60));
89+
90+
const search = new A3SSearch();
91+
92+
try {
93+
const response = await search.search('web development', {
94+
engines: ['ddg', 'google', 'wiki'],
95+
limit: 10,
96+
});
97+
98+
console.log(`\n✓ Search completed in ${response.durationMs}ms`);
99+
console.log(`✓ Found ${response.count} results`);
100+
101+
// Count results by engine
102+
const engineCounts: Record<string, number> = {};
103+
for (const result of response.results) {
104+
for (const engine of result.engines) {
105+
engineCounts[engine] = (engineCounts[engine] || 0) + 1;
106+
}
107+
}
108+
109+
console.log('\nResults by engine:');
110+
for (const [engine, count] of Object.entries(engineCounts).sort()) {
111+
console.log(` - ${engine}: ${count} results`);
112+
}
113+
} catch (error) {
114+
console.log(`\n✗ Error: ${error.message}`);
115+
throw error;
116+
}
117+
}
118+
119+
async function testNewFeatures() {
120+
console.log('\n' + '='.repeat(60));
121+
console.log('Testing new SearchQuery features');
122+
console.log('='.repeat(60));
123+
124+
const search = new A3SSearch();
125+
126+
try {
127+
const response = await search.search('python tutorial', {
128+
engines: ['ddg'],
129+
language: 'en',
130+
safesearch: 'moderate',
131+
page: 1,
132+
timeRange: 'week',
133+
limit: 5,
134+
});
135+
136+
console.log(`\n✓ Search with filters completed in ${response.durationMs}ms`);
137+
console.log(`✓ Found ${response.count} results`);
138+
console.log(`✓ Suggestions: ${response.suggestions.length}`);
139+
console.log(`✓ Answers: ${response.answers.length}`);
140+
141+
if (response.suggestions.length > 0) {
142+
console.log('\nSearch suggestions:');
143+
for (const suggestion of response.suggestions.slice(0, 5)) {
144+
console.log(` - ${suggestion}`);
145+
}
146+
}
147+
148+
if (response.answers.length > 0) {
149+
console.log('\nInstant answers:');
150+
for (const answer of response.answers.slice(0, 3)) {
151+
console.log(` - ${answer}`);
152+
}
153+
}
154+
} catch (error) {
155+
console.log(`\n✗ Error: ${error.message}`);
156+
throw error;
157+
}
158+
}
159+
160+
async function main() {
161+
console.log('\n' + '='.repeat(60));
162+
console.log('A3S Search - Headless Engine Test Suite');
163+
console.log('='.repeat(60));
164+
console.log('\nNote: On first run, Chrome for Testing will be downloaded');
165+
console.log('to ~/.a3s/chromium/ (approximately 150-200 MB)');
166+
console.log('\nThis may take 1-5 minutes depending on your network speed...');
167+
168+
try {
169+
// Test new features first (faster, no browser needed)
170+
await testNewFeatures();
171+
172+
// Test headless engines
173+
await testGoogle();
174+
await testBaidu();
175+
await testMixedEngines();
176+
177+
console.log('\n' + '='.repeat(60));
178+
console.log('✓ All tests completed successfully!');
179+
console.log('='.repeat(60));
180+
} catch (error) {
181+
console.log('\n' + '='.repeat(60));
182+
console.log(`✗ Test suite failed: ${error}`);
183+
console.log('='.repeat(60));
184+
throw error;
185+
}
186+
}
187+
188+
main().catch(console.error);

0 commit comments

Comments
 (0)