Skip to content

Commit 3203728

Browse files
BorDevTechBorDevTech
committed
feat: add script to fetch and store British Columbia veterinary registrant data
- Implemented a TypeScript script to loop through two-letter firstname combinations to avoid CVBC 500 error. - Extracted GUID from Full Name link as `id` and maintained an empty licenseNumber for schema consistency. - Added logging for per-term and overall progress, and flushed data to disk every 10 terms to manage memory usage. - Ensured deduplication of records by `id` before writing to JSON file. - Saved results to app/api/verify/britishcolumbia/britishcolumbiaVets.json.
1 parent 7689729 commit 3203728

8 files changed

Lines changed: 35897 additions & 67 deletions

File tree

‎app/api/verify/arkansas/logic.ts‎

Lines changed: 96 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -29,77 +29,123 @@ export async function verify({
2929
if (Array.isArray(raw.results)) return raw.results;
3030
return [];
3131
}
32-
async function fetchDetailedReport(entry: RawVetEntry): Promise<VetResult> {
33-
const licenseNumber = entry.license_num?.trim() ?? "";
34-
const url = `https://mip.agri.arkansas.gov/VetLicensingPortal/Guest/Home/Get_Licensee_Info?id=${licenseNumber}`;
35-
let expirationDate: string | null = null;
36-
try {
37-
const res = await fetch(url);
38-
if (res.ok) {
39-
const data = await res.json();
40-
expirationDate = data.lexpdate?.trim() ?? null;
41-
}
42-
else {
43-
console.warn(`⚠️ Failed to fetch detailed report for license ${licenseNumber}: ${res.status} ${res.statusText}`);
44-
45-
}
46-
} catch (error) {
47-
console.warn(`❌ Error fetching expiration for ${licenseNumber}:`, error);
48-
49-
} return {
50-
name: `${entry.first_name} ${entry.last_name}`.trim(),
51-
licenseNumber,
52-
status: entry.license_status?.trim(),
53-
expiration: expirationDate ??
54-
new Date(entry.license_expiration_date).toLocaleDateString("en-US", {
55-
timeZone: "UTC",
56-
year: "numeric",
57-
month: "short",
58-
day: "numeric",
59-
}),
60-
};
6132

62-
}
6333
// 🧠 Internal helper: filter and transform entries
64-
async function filterEntries(entries: RawVetEntry[]): Promise<VetResult[]> {
65-
// 🧹 Normalize and map entries
66-
67-
const filtered = entries
34+
function filterEntries(entries: RawVetEntry[]): VetResult[] {
35+
return entries
6836
.filter((entry) => {
69-
const isVet = entry.license_type?.trim() === "Vet" && entry.license_status?.trim() === "Active";
37+
const isVet = entry.license_type?.toLowerCase().includes("vet");
7038
if (!isVet) return false;
7139

7240
const matchesLicense = licenseNumber
7341
? entry.license_num?.toLowerCase().includes(licenseNumber.toLowerCase())
7442
: true;
7543

76-
let matchesName = true;
77-
if (firstName) {
78-
matchesName =
79-
typeof entry.first_name === "string" &&
80-
entry.first_name.toLowerCase().startsWith(firstName.toLowerCase());
81-
}
82-
if (matchesName && lastName) {
83-
matchesName =
84-
typeof entry.last_name === "string" &&
85-
entry.last_name.toLowerCase().startsWith(lastName.toLowerCase());
86-
}
8744

88-
return matchesLicense && matchesName;
8945

90-
})
46+
const matchesName = firstName || lastName
47+
? (() => {
48+
// Use FIRST_NAME if available, otherwise LAST_NAME
49+
const name = (entry.first_name || entry.last_name || "").trim();
50+
const [dbaFirst, ...dbaRest] = name.split(" ");
51+
const dbaLast = dbaRest.length > 0 ? dbaRest[dbaRest.length - 1] : "";
52+
53+
let firstMatches = true;
54+
let lastMatches = true;
9155

56+
if (firstName) {
57+
firstMatches = dbaFirst?.toLowerCase().startsWith(firstName.toLowerCase());
58+
}
59+
if (lastName) {
60+
lastMatches = dbaLast?.toLowerCase().startsWith(lastName.toLowerCase());
61+
}
9262

93-
return await Promise.all(filtered.map(fetchDetailedReport));
63+
return firstMatches && lastMatches;
64+
})()
65+
: true;
66+
67+
return matchesLicense && matchesName;
68+
})
69+
.map((entry) => ({
70+
name: `${entry.first_name} ${entry.last_name}`.trim(),
71+
licenseNumber: entry.license_num?.trim(),
72+
status: "Active",
73+
license_issue_date: entry.license_issue_date?.trim(),
74+
license_expiration_date: entry.license_expiration_date?.trim()
75+
}));
9476
}
9577

78+
79+
// async function fetchDetailedReport(entry: RawVetEntry): Promise<VetResult> {
80+
// const licenseNumber = entry.license_num?.trim() ?? "";
81+
// const url = `https://mip.agri.arkansas.gov/VetLicensingPortal/Guest/Home/Get_Licensee_Info?id=${licenseNumber}`;
82+
// let expirationDate: string | null = null;
83+
// try {
84+
// const res = await fetch(url);
85+
// if (res.ok) {
86+
// const data = await res.json();
87+
// expirationDate = data.lexpdate?.trim() ?? null;
88+
// }
89+
// else {
90+
// console.warn(`⚠️ Failed to fetch detailed report for license ${licenseNumber}: ${res.status} ${res.statusText}`);
91+
92+
// }
93+
// } catch (error) {
94+
// console.warn(`❌ Error fetching expiration for ${licenseNumber}:`, error);
95+
96+
// } return {
97+
// name: `${entry.first_name} ${entry.last_name}`.trim(),
98+
// licenseNumber,
99+
// status: entry.license_status?.trim(),
100+
// expiration: expirationDate ??
101+
// new Date(entry.license_expiration_date).toLocaleDateString("en-US", {
102+
// timeZone: "UTC",
103+
// year: "numeric",
104+
// month: "short",
105+
// day: "numeric",
106+
// }),
107+
// };
108+
109+
// }
110+
// // 🧠 Internal helper: filter and transform entries
111+
// async function filterEntries(entries: RawVetEntry[]): Promise<VetResult[]> {
112+
// // 🧹 Normalize and map entries
113+
114+
// const filtered = entries
115+
// .filter((entry) => {
116+
// const isVet = entry.license_type?.trim() === "Vet" && entry.license_status?.trim() === "Active";
117+
// if (!isVet) return false;
118+
119+
// const matchesLicense = licenseNumber
120+
// ? entry.license_num?.toLowerCase().includes(licenseNumber.toLowerCase())
121+
// : true;
122+
123+
// let matchesName = true;
124+
// if (firstName) {
125+
// matchesName =
126+
// typeof entry.first_name === "string" &&
127+
// entry.first_name.toLowerCase().startsWith(firstName.toLowerCase());
128+
// }
129+
// if (matchesName && lastName) {
130+
// matchesName =
131+
// typeof entry.last_name === "string" &&
132+
// entry.last_name.toLowerCase().startsWith(lastName.toLowerCase());
133+
// }
134+
135+
// return matchesLicense && matchesName;
136+
137+
// })
138+
139+
140+
// return await Promise.all(filtered.map(fetchDetailedReport));
141+
// }
142+
96143
const res = await fetch(`/api/verify/${key}`, {
97144
method: "GET",
98145
});
99146
if (!res.ok) throw new Error(`Failed to fetch ${key} data`);
100147
const rawData = await res.json();
101148
const parsedData = parseBlob(rawData);
102149
const results = await filterEntries(parsedData);
103-
console.log("parsed results: ", results)
104150
return results;
105151
}

‎app/api/verify/arkansas/route.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ export async function GET(request: NextRequest) {
2525
},
2626
});
2727
const data = await response.text();
28-
return NextResponse.json({
29-
blob: data,
30-
count: Array.isArray(data) ? data.length : 0,
28+
return new Response(data, {
29+
headers: { "Content-Type": "application/json; charset=utf-8" },
30+
status: response.status,
3131
});
3232
} catch (error: unknown) {
3333
console.warn(`⚠️ BlobFetch failed for ${key}, falling back to live parse: ${error}`);

0 commit comments

Comments
 (0)