diff --git a/backend/.gitignore b/backend/.gitignore index e43b503..e1d67da 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -39,3 +39,6 @@ build/ ### upload files ### uploads/ + +# Local run script +run-backend.ps1 diff --git a/backend/service.yaml b/backend/service.yaml index 55434c6..592b09f 100644 --- a/backend/service.yaml +++ b/backend/service.yaml @@ -50,6 +50,16 @@ spec: secretKeyRef: name: gemini-api-key key: latest + - name: GROQ_API_KEY + valueFrom: + secretKeyRef: + name: groq-api-key + key: latest + - name: OPENROUTER_API_KEY + valueFrom: + secretKeyRef: + name: openrouter-api-key + key: latest - name: JWT_SECRET valueFrom: diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/config/StartupInitializer.java b/backend/src/main/java/com/thughari/jobtrackerpro/config/StartupInitializer.java new file mode 100644 index 0000000..cbfbe52 --- /dev/null +++ b/backend/src/main/java/com/thughari/jobtrackerpro/config/StartupInitializer.java @@ -0,0 +1,23 @@ +package com.thughari.jobtrackerpro.config; + +import com.thughari.jobtrackerpro.repo.UserRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class StartupInitializer implements CommandLineRunner { + + private final UserRepository userRepository; + + public StartupInitializer(UserRepository userRepository) { + this.userRepository = userRepository; + } + + @Override + public void run(String... args) throws Exception { + log.info("System Startup: Resetting all active Gmail sync locks."); + userRepository.resetAllSyncLocks(); + } +} diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/controller/GmailIntegrationController.java b/backend/src/main/java/com/thughari/jobtrackerpro/controller/GmailIntegrationController.java index 8148512..4ab6fb6 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/controller/GmailIntegrationController.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/controller/GmailIntegrationController.java @@ -35,6 +35,7 @@ public ResponseEntity connectGmail(@RequestBody Map body try { gmailAutomationService.connectAndSetupPush(authCode, email); + gmailAutomationService.initiateManualSync(email); return ResponseEntity.ok("Gmail Automation enabled successfully."); } catch (Exception e) { log.error("Failed to setup Gmail for user {}: {}", email, e.getMessage()); diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/dto/ChartData.java b/backend/src/main/java/com/thughari/jobtrackerpro/dto/ChartData.java index 00ceced..f5f8e6c 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/dto/ChartData.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/dto/ChartData.java @@ -2,9 +2,12 @@ import lombok.AllArgsConstructor; import lombok.Data; +import java.io.Serializable; + @Data @AllArgsConstructor -public class ChartData { +public class ChartData implements Serializable { + private static final long serialVersionUID = 1L; private String name; private long value; } \ No newline at end of file diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardResponse.java b/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardResponse.java index ab10cdc..58d9dfc 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardResponse.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardResponse.java @@ -2,10 +2,15 @@ import lombok.Data; import java.util.List; +import java.io.Serializable; + @Data -public class DashboardResponse { +public class DashboardResponse implements Serializable { + private static final long serialVersionUID = 1L; private DashboardStatsDTO stats; private List statusChart; private List monthlyChart; private List interviewChart; + private boolean gmailSyncInProgress; + private String gmailSyncStatus; } \ No newline at end of file diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardStatsDTO.java b/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardStatsDTO.java index 49571a3..f49b8e0 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardStatsDTO.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/dto/DashboardStatsDTO.java @@ -3,9 +3,12 @@ import lombok.AllArgsConstructor; import lombok.Data; +import java.io.Serializable; + @Data @AllArgsConstructor -public class DashboardStatsDTO { +public class DashboardStatsDTO implements Serializable { + private static final long serialVersionUID = 1L; private long totalApplications; private long activePipeline; private long interviews; diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/entity/User.java b/backend/src/main/java/com/thughari/jobtrackerpro/entity/User.java index b13d4c5..9c99f84 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/entity/User.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/entity/User.java @@ -53,6 +53,9 @@ public class User { @Column(name = "gmail_sync_started_at") private LocalDateTime gmailSyncStartedAt; + @Column(name = "gmail_sync_status") + private String gmailSyncStatus; + @Column(name = "gmail_connected") private Boolean gmailConnected = false; diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/exception/AiQuotaExceededException.java b/backend/src/main/java/com/thughari/jobtrackerpro/exception/AiQuotaExceededException.java new file mode 100644 index 0000000..067aab4 --- /dev/null +++ b/backend/src/main/java/com/thughari/jobtrackerpro/exception/AiQuotaExceededException.java @@ -0,0 +1,10 @@ +package com.thughari.jobtrackerpro.exception; + +public class AiQuotaExceededException extends RuntimeException { + public AiQuotaExceededException(String message) { + super(message); + } + public AiQuotaExceededException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/interfaces/GeminiService.java b/backend/src/main/java/com/thughari/jobtrackerpro/interfaces/AiExtractionService.java similarity index 88% rename from backend/src/main/java/com/thughari/jobtrackerpro/interfaces/GeminiService.java rename to backend/src/main/java/com/thughari/jobtrackerpro/interfaces/AiExtractionService.java index d0e0b38..87e5ba9 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/interfaces/GeminiService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/interfaces/AiExtractionService.java @@ -5,7 +5,7 @@ import com.thughari.jobtrackerpro.dto.EmailBatchItem; import com.thughari.jobtrackerpro.dto.JobDTO; -public interface GeminiService { +public interface AiExtractionService { JobDTO extractJobFromEmail(String from, String subject, String body); diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/repo/UserRepository.java b/backend/src/main/java/com/thughari/jobtrackerpro/repo/UserRepository.java index f4761b6..f5560c2 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/repo/UserRepository.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/repo/UserRepository.java @@ -30,6 +30,16 @@ public interface UserRepository extends JpaRepository { @Query("UPDATE User u SET u.gmailSyncInProgress = false WHERE u.email = :email") void releaseSyncLock(@Param("email") String email); + @Modifying + @Transactional + @Query("UPDATE User u SET u.gmailSyncInProgress = false, u.gmailSyncStatus = null") + void resetAllSyncLocks(); + + @Modifying + @Transactional + @Query("UPDATE User u SET u.gmailSyncStatus = :status WHERE u.email = :email") + void updateSyncStatus(@Param("email") String email, @Param("status") String status); + List findByGmailConnectedTrue(); @Modifying diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/GeminiExtractionService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/GeminiExtractionService.java index 9776987..86fd10e 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/GeminiExtractionService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/GeminiExtractionService.java @@ -6,7 +6,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.thughari.jobtrackerpro.dto.EmailBatchItem; import com.thughari.jobtrackerpro.dto.JobDTO; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; +import com.thughari.jobtrackerpro.exception.AiQuotaExceededException; import com.thughari.jobtrackerpro.util.UrlParser; import lombok.extern.slf4j.Slf4j; @@ -14,6 +15,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.MediaType; import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClient; import java.time.LocalDateTime; @@ -23,7 +25,7 @@ @Service @Slf4j @ConditionalOnProperty(name = "app.gemini.enabled", havingValue = "true") -public class GeminiExtractionService implements GeminiService { +public class GeminiExtractionService implements AiExtractionService { private final RestClient restClient; private final ObjectMapper objectMapper; @@ -63,6 +65,9 @@ public JobDTO extractJobFromEmail(String from, String subject, String body) { return parseGeminiResponse(response); + } catch (HttpClientErrorException e) { + log.error("Gemini API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("GEMINI_API_ERROR", e); } catch (Exception e) { log.error("AI Extraction failed or timed out", e); return null; @@ -91,364 +96,15 @@ public List extractJobsFromBatch(List items) { return parseBulkGeminiResponse(response); + } catch (HttpClientErrorException e) { + log.error("Gemini API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("GEMINI_API_ERROR", e); } catch (Exception e) { log.error("Bulk AI Extraction failed", e); return List.of(); } } - private String buildBatchPrompt(List items) { - StringBuilder emailListBuilder = new StringBuilder(); - for (int i = 0; i < items.size(); i++) { - EmailBatchItem item = items.get(i); - String rawBody = item.body() == null ? "" : item.body(); - - String trimmed = UrlParser.trimNoise(rawBody); - String safeBody = trimmed.length() > 3000 - ? trimmed.substring(0, 3000 ) - : trimmed; - - List urls = UrlParser.extractAndCleanUrls(rawBody); - - emailListBuilder.append(""" - --- EMAIL INDEX: %d --- - FROM: %s - REPLY-TO: %s - SUBJECT: %s - BODY: %s - - AVAILABLE_URLS: - %s - ----------------------- - """.formatted( - i, - item.from(), - item.replyTo(), - item.subject(), - safeBody, - buildUrlIndexList(urls) - )); - } - - return """ - ##Act as a strict Global Data Extraction System for a Job Application Tracker. - - You analyze emails and extract structured job application data. - - -------------------------------------------------- - - ### TASK - - Analyze the list of emails provided below. - - For EACH email you must determine: - - 1. Is this email related to a REAL hiring interaction? - 2. If YES → extract structured job data. - 3. If NO → completely ignore the email. - - A valid job-related email must indicate interaction with a hiring process. - - Examples of valid hiring interactions: - - Application confirmation - - Recruiter outreach - - Interview invitation - - Coding assessment invitation - - Hiring process update - - Offer - - Rejection - - Emails that are only informational, promotional, or educational MUST be ignored. - - -------------------------------------------------- - - ### HIRING SIGNAL RULE (VERY IMPORTANT) - - A valid job email MUST contain at least ONE hiring process signal such as: - - - "applied" - - "application received" - - "thank you for applying" - - "interview" - - "assessment" - - "coding challenge" - - "recruiter" - - "hiring team" - - "offer" - - "rejected" - - "next steps" - - If NONE of these signals exist, the email MUST be ignored. - - Company name + role alone is NOT sufficient. - - -------------------------------------------------- - - ### HARD EXCLUSION RULES - - DO NOT treat the email as job-related if it is any of the following: - - - Job alerts - - Job recommendation digests - - Emails listing multiple job postings - - Developer events or workshops - - Coding contests - - Newsletters - - Marketing campaigns - - Community announcements - - Learning resources or interview prep - - General job search tips - - Examples that MUST be ignored: - - - "Your job alert" - - "Recommended jobs" - - "Jobs you may like" - - "New jobs for you" - - "Join our developer workshop" - - "LeetCode Weekly Contest" - - "Google Cloud Labs event" - - "Upcoming coding challenge" - - These emails MUST produce NO OUTPUT. - - -------------------------------------------------- - - ### MULTI-COMPANY RULE - - If an email lists multiple different companies or multiple unrelated job listings, - it is a job alert or recommendation email and MUST be ignored. - - Valid job emails normally reference ONE specific job opportunity. - - -------------------------------------------------- - - ### NON-JOB EMAIL EXCLUSIONS - - Strictly ignore emails related to: - - - banking - - OTP - - payment confirmation - - receipts - - subscriptions - - invoices - - shipping notifications - - system alerts - - account security notifications - - -------------------------------------------------- - - ### INDEXING RULE (STRICT) - - Each email is labeled: - - --- EMAIL INDEX: X --- - - You MUST: - - - Return "inputIndex" exactly equal to the EMAIL INDEX value. - - NEVER renumber indexes. - - NEVER invent indexes. - - Only include job-related emails. - - Preserve original index numbers. - - Example: - - If emails are 0,1,2,3 and only 0 and 3 are job-related: - - [ - { "inputIndex": 0, ... }, - { "inputIndex": 3, ... } - ] - - Do NOT return sequential indexes. - - -------------------------------------------------- - - ### LIST OF EMAILS TO ANALYZE - - %s - - -------------------------------------------------- - - ### EXTRACTION RULES - - Apply these only to valid job-related emails. - - 1. COMPANY - - Identify the hiring company. - - Rules: - - - Prefer the company explicitly mentioned in hiring context. - - If multiple companies appear, select the one responsible for the job. - - Fallback: extract from sender domain. - - Examples: - careers@stripe.com → Stripe - talent.wayfair.com → Wayfair - - Emails may be sent by recruiting platforms such as: Naukri, Talent500, LinkedIn, Hired, Wellfound, Indeed, etc. - - These platforms are NOT the hiring company. - - If a recruiting platform is mentioned, identify the actual employer - mentioned in the job description or company section. - - Example: - Email from: Talent500 - Job description mentions: Albertsons Companies - - Correct company: Albertsons - - If the email contains a Reply-To header, and the domain appears to be a company domain, prefer that domain over recruiting platforms. - - Example: - - From: messages.naukri.com - Reply-To: recruiter@yupptv.com - - Company = YuppTV - - Ignore generic domains: - gmail, yahoo, outlook, etc. - - Default: - "Unknown Company" - - -------------------------------------------------- - - 2. ROLE - - Extract the job title. - - Examples: - Software Engineer - Java Developer - Backend Engineer - - If no clear role exists: - Default to **"Software Engineer"** - - -------------------------------------------------- - - 3. STATUS - - Return EXACTLY one of: - - - "Applied" - - "Shortlisted" - - "Interview Scheduled" - - "Offer Received" - - "Rejected" - - Rules: - - Recruiter outreach -> "Applied" - Referral messages -> "Applied" - Assessment invitations -> "Shortlisted" - Interview scheduling -> "Interview Scheduled" - Application Rejected -> "Rejected" - - Never invent new status values. - - -------------------------------------------------- - - 4. LOCATION - - Extract city or country if present. - - Examples: - London - Hyderabad - United States - - If no location exists: - Return **"Remote"** - - -------------------------------------------------- - - 5. NOTES - - Write ONE concise sentence summarizing the email. - - Example: - "Application received for Software Engineer role." - - -------------------------------------------------- - - 6. URL_SELECTION - - Each email contains a list of AVAILABLE_URLS. - - You MUST: - - - choose the index of the most relevant job-related link - - never invent URLs - - ignore footer links - - Ignore links related to: - - - unsubscribe - - help - - privacy - - settings - - If no job-related link exists: - - "urlIndex": -1 - - -------------------------------------------------- - - 7. SALARY - - Extract salary if present. - - If not present: - - salaryMin = 0.0 - salaryMax = 0.0 - - -------------------------------------------------- - - ### MULTILINGUAL RULE - - Emails may be written in any language. - - Extract information normally but return ALL output values in English. - - -------------------------------------------------- - - ### OUTPUT FORMAT - - Return ONLY a raw JSON array. - - No markdown - No explanations - No text before or after JSON. - - Return [] if no job-related emails exist. - - -------------------------------------------------- - - ### Example Output - - [ - { - "inputIndex": 0, - "company": "Stripe", - "role": "Software Engineer", - "location": "Remote", - "status": "Applied", - "urlIndex": 1, - "salaryMin": 0.0, - "salaryMax": 0.0, - "notes": "Application confirmation for Software Engineer role." - } - ]""".formatted(emailListBuilder.toString()); - } private List parseBulkGeminiResponse(String rawResponse) { try { @@ -463,9 +119,14 @@ private List parseBulkGeminiResponse(String rawResponse) { contentText = contentText.replaceAll("```json", "").replaceAll("```", "").trim(); - if (contentText.equals("[]") || contentText.equalsIgnoreCase("null")) { + // Extract just the JSON array between first '[' and last ']' + int start = contentText.indexOf('['); + int end = contentText.lastIndexOf(']'); + if (start == -1 || end == -1 || end <= start) { + log.warn("Gemini bulk response has no JSON array: {}", contentText.substring(0, Math.min(200, contentText.length()))); return List.of(); } + contentText = contentText.substring(start, end + 1); List jobs = objectMapper.readValue(contentText, new TypeReference>() {}); @@ -493,6 +154,7 @@ private List parseBulkGeminiResponse(String rawResponse) { } private String buildUrlIndexList(List urls) { + if (urls.isEmpty()) return "None\n"; StringBuilder sb = new StringBuilder(); for (int i = 0; i < urls.size(); i++) { sb.append(i).append(": ").append(urls.get(i)).append("\n"); @@ -501,84 +163,78 @@ private String buildUrlIndexList(List urls) { } private String buildPrompt(String from, String subject, String body) { - String safeBody = (body != null) ? (body.length() > 8000 ? body.substring(0, 8000 ) : body) : ""; + String safeBody = (body != null) ? (body.length() > 1200 ? body.substring(0, 1200) : body) : ""; return """ - Act as a strict Global Data Extraction System for a Job Application Tracker. - - ### TASK - Analyze the email content below. - Determine whether the email is related to a specific job opportunity, hiring process, or recruiter communication. - - If the email mentions a company and a job role in a hiring context, it MUST be treated as job-related. - Only return `null` if the email is clearly commercial spam, a receipt, or completely unrelated to jobs or careers. - - Valid categories include: - 1. Application Confirmations (ATS). - 2. Interview Invites. - 3. Offers/Rejections. - 4. **Recruiter Outreach / Walk-In Drive Invitations**. - 5. **User sent emails** (e.g. user replying to a recruiter about a role). + Act as a strict ATS Data Extractor. + TASK: Analyze email. Return JSON job object if job-related. Else return null. Skip spam/receipts. - **CRITICAL RULE:** - Only return `null` if the email is strictly commercial spam (selling products), receipts, or completely unrelated to careers. + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link. + 7. SALARY: Extract if present, else 0.0. - ### EMAIL CONTENT - FROM: %s - SUBJECT: %s - BODY: %s - - ### EXTRACTION RULES - 1. **COMPANY**: Identify the hiring company. - - If multiple companies are mentioned, select the one most directly related to the role, interview, assessment, offer, or rejection. - - If no company name is explicitly mentioned in the email content: - - Attempt a fallback extraction from the sender's email domain. - - Derive the company name from the domain (for example, careers@stripe.com -> Stripe). - - Ignore generic email providers such as gmail.com, yahoo.com, outlook.com, and similar. - - If both content-based and domain-based extraction fail: - - Never return null - - Return "Unknown Company" as the default value. - - 2. **ROLE**: Extract the specific job title. - - If it is a Walk-In drive listing multiple roles, pick the one most relevant to "Java" or "Software Engineer", or default to "Software Engineer". - - Default to "Software Engineer" only if no role is clear. - 3. **STATUS**: Map to one of these exact statuses: - - "Applied" (Use this for Walk-in invites, Recruiter outreach, or Sent emails) - - "Shortlisted" (Use this for 'Next steps', Coding Tests, Exams, or HackerRank invites or similar) - - "Interview Scheduled" (for any interview invites) - - "Offer Received" - - "Rejected" - 4. **NOTES**: A 1-sentence summary (e.g., "Walk-in drive invitation", "Replied to recruiter"). - 5. **LOCATION**: Extract City/Country if found, otherwise default to "Remote" but never make it null. - - If not found, default to "Remote". - - 6. **URL**: Hunt for the primary call-to-action link. - - Look for URLs immediately following words like "Apply", "View Job", "Click here", or "Check status". - - If multiple links exist, prioritize ones containing "careers", "jobs", "apply", or "lever.co", "greenhouse.io", "myworkday". - - Return the full raw URL string. - - If no URL is found, return the company's website mentioned in the email. - 7. **SALARY**: Extract salary numbers if present. - - Otherwise return 0.0 for both salaryMin and salaryMax. - - ### MULTILINGUAL RULE - If the email is in Dutch, French, or any other language, you MUST process it normally but provide the JSON output values in English so the user can understand their dashboard. - - ### OUTPUT FORMAT - Return ONLY raw JSON (no markdown blocks, no explanations): - { - "company": "String", - "role": "String", - "location": "String", - "status": "String", - "url": "String", - "salaryMin": 0.0, - "salaryMax": 0.0, - "notes": "String" - } - OR just: null + MULTILINGUAL: Output values in English. + + EMAIL: + From: %s + Subject: %s + Body: %s + + OUTPUT: Raw JSON (no markdown) or null. """.formatted(from, subject, safeBody); } + private String buildBatchPrompt(List items) { + StringBuilder emailListBuilder = new StringBuilder(); + for (int i = 0; i < items.size(); i++) { + EmailBatchItem item = items.get(i); + String rawBody = item.body() == null ? "" : item.body(); + + String trimmed = UrlParser.trimNoise(rawBody); + String safeBody = trimmed.length() > 1200 ? trimmed.substring(0, 1200) : trimmed; + List urls = UrlParser.extractAndCleanUrls(rawBody); + + emailListBuilder.append(""" + [EMAIL %d] + From: %s + Reply-To: %s + Subject: %s + Body: %s + URLs: + %s + """.formatted( + i, item.from(), item.replyTo(), item.subject(), safeBody, buildUrlIndexList(urls) + )); + } + + return """ + Act as a strict ATS Data Extractor. + TASK: Analyze emails. Return JSON array of job objects for job-related emails only. Skip spam/receipts. + + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link from URLs list, else company website. + 7. SALARY: Extract if present, else 0.0. + 8. INPUT INDEX: Must match [EMAIL X] index. + + MULTILINGUAL: Output values in English. + + EMAILS: + %s + + OUTPUT FORMAT: Raw JSON array only (no markdown). + """.formatted(emailListBuilder.toString()); + } + private JobDTO parseGeminiResponse(String rawResponse) { try { JsonNode root = objectMapper.readTree(rawResponse); diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailIntegrationService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailIntegrationService.java index ffb9d30..4303778 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailIntegrationService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailIntegrationService.java @@ -13,7 +13,8 @@ import com.thughari.jobtrackerpro.dto.JobDTO; import com.thughari.jobtrackerpro.entity.User; import com.thughari.jobtrackerpro.exception.ResourceNotFoundException; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.exception.AiQuotaExceededException; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; import com.thughari.jobtrackerpro.repo.UserRepository; import com.thughari.jobtrackerpro.util.CacheEvictService; import com.thughari.jobtrackerpro.util.UrlParser; @@ -24,29 +25,38 @@ import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Caching; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.web.client.RestClient; +import java.lang.Thread; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; @Service @Slf4j public class GmailIntegrationService { private final UserRepository userRepository; - private final GeminiService geminiService; + private final AiExtractionService aiService; private final JobService jobService; private final RestClient restClient; private final CacheEvictService cacheEvictService; + private final Executor taskExecutor; private final String APPLICATION_NAME = "JobTrackerPro"; - + + @Value("${app.gmail.sync-range:30d}") + private String syncRange; + private static final NetHttpTransport HTTP_TRANSPORT; private static final GsonFactory JSON_FACTORY = GsonFactory.getDefaultInstance(); - + static { try { HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); @@ -58,34 +68,36 @@ public class GmailIntegrationService { private static final String ATS_FILTER = "from:(myworkday.com OR greenhouse.io OR lever.co OR smartrecruiters.com OR icims.com OR jobvite.com OR bamboo.hr OR workablemail.com OR successfactors.com OR taleo.net OR avature.net OR jobs2careers.com OR ziprecruiter.com OR monster.com OR careerbuilder.com OR wellfound.com OR lu.ma OR breezy.hr OR jazzhr.com OR comeet.com OR recruitee.com OR teamtailor.com OR applytojob.com OR jobs.github.com OR hackerrankforwork.com OR hackerrank.com OR hackerearth.com OR codility.com OR testgorilla.com OR hirevue.com OR vidcruiter.com OR codemetry.com OR pymetrics.com OR hired.com OR triplebyte.com OR newtonsoftware.com OR jobadder.com OR jobscore.com OR jobsoid.com OR jobplex.com OR jobdroid.com OR jobiak.com OR jobg8.com OR jobisjob.com OR jobrapido.com OR simplyhired.com OR glassdoor.com OR indeed.com OR remoteok.io OR weworkremotely.com OR remote.co OR angel.co OR stackoverflow.com)"; private static final String SUBJECT_FILTER = "subject:(Application OR Applied OR Applying OR Invited OR Candidate OR Thanks OR \"Thank You\" OR Received OR Confirmation OR Interview OR Status OR Sollicitatie OR Engineer OR Developer OR Analyst OR Scientist OR Specialist OR Invitation OR Invite OR Assessment OR Challenge OR Test OR Screened OR Position OR Declaration OR Talent OR Opportunity OR Role OR Job OR Opening OR Opened OR Lead OR Recruiter OR HR OR \"Human Resources\" OR Hiring OR Resume OR CV)"; private static final String EXCLUSIONS = " -\"payment\" -\"invoice\" -\"otp\" -\"transaction\" -\"statement\" -\"bank\" -\"security alert\" -\"verification code\""; - + @Value("${spring.security.oauth2.client.registration.google.client-id}") private String clientId; - + @Value("${spring.security.oauth2.client.registration.google.client-secret}") private String clientSecret; @Value("${app.google.pubsub-topic}") private String pubsubTopic; - public GmailIntegrationService(UserRepository userRepository, GeminiService geminiService, JobService jobService, CacheEvictService cacheEvictService) { + public GmailIntegrationService(UserRepository userRepository, AiExtractionService aiService, JobService jobService, + CacheEvictService cacheEvictService, Executor taskExecutor) { this.userRepository = userRepository; - this.geminiService = geminiService; + this.aiService = aiService; this.jobService = jobService; this.restClient = RestClient.create(); this.cacheEvictService = cacheEvictService; + this.taskExecutor = taskExecutor; } @Transactional @Caching(evict = { - @CacheEvict(value = "users", key = "#email"), + @CacheEvict(value = "users", key = "#email"), @CacheEvict(value = "userEntities", key = "#email") - }) + }) public void connectAndSetupPush(String authCode, String email) throws Exception { - - User user = userRepository.findByEmail(email) + + User user = userRepository.findByEmail(email) .orElseThrow(() -> new RuntimeException("User not found")); - + NetHttpTransport transport = HTTP_TRANSPORT; GsonFactory jsonFactory = JSON_FACTORY; @@ -96,62 +108,89 @@ public void connectAndSetupPush(String authCode, String email) throws Exception String refreshToken = tokenResponse.getRefreshToken(); String accessToken = tokenResponse.getAccessToken(); - Gmail service = new Gmail.Builder(transport, jsonFactory, request -> - request.getHeaders().setAuthorization("Bearer " + accessToken)) + Gmail service = new Gmail.Builder(transport, jsonFactory, + request -> request.getHeaders().setAuthorization("Bearer " + accessToken)) .setApplicationName(APPLICATION_NAME).build(); String labelId = getOrCreateLabel(service); createJobFilter(service, labelId); - WatchRequest watchRequest = new WatchRequest() - .setTopicName(pubsubTopic) - .setLabelIds(List.of(labelId)); - - WatchResponse watchResponse = service.users().watch("me", watchRequest).execute(); + String watchHistoryId = null; + Long watchExpiration = null; + try { + WatchRequest watchRequest = new WatchRequest() + .setTopicName(pubsubTopic) + .setLabelIds(List.of(labelId)); + + WatchResponse watchResponse = service.users().watch("me", watchRequest).execute(); + watchHistoryId = watchResponse.getHistoryId().toString(); + watchExpiration = watchResponse.getExpiration(); + log.info("Gmail Push Watch registered successfully."); + } catch (Exception e) { + log.warn("Failed to set up Gmail Push Watch (OIDC PubSub): {}. Falling back to manual pull-sync mode.", + e.getMessage()); + try { + watchHistoryId = service.users().getProfile("me").execute().getHistoryId().toString(); + } catch (Exception ex) { + log.error("Failed to retrieve initial Gmail profile history ID", ex); + } + } user.setGmailConnected(true); if (refreshToken != null) { user.setGmailRefreshToken(refreshToken); } user.setGmailLabelId(labelId); - user.setGmailHistoryId(watchResponse.getHistoryId().toString()); - user.setGmailWatchExpiration(watchResponse.getExpiration()); + if (watchHistoryId != null) { + user.setGmailHistoryId(watchHistoryId); + } + user.setGmailWatchExpiration(watchExpiration); userRepository.saveAndFlush(user); - + log.info("User {} successfully connected Gmail. Watch set with label ID: {}", email, labelId); } public void initiateManualSync(String email) { - - LocalDateTime now = LocalDateTime.now(); + log.info("Manual sync requested for user: {}", email); + LocalDateTime now = LocalDateTime.now(); LocalDateTime expiryThreshold = now.minusMinutes(15); - - int updatedRows = userRepository.claimSyncLock(email, now, expiryThreshold); - + + int updatedRows = userRepository.claimSyncLock(email, now, expiryThreshold); if (updatedRows == 0) { - return; + log.info("Manual sync already in progress for user: {}. Request ignored.", email); + return; } - cacheEvictService.evictAllForUser(email); + + updateSyncStatus(email, "Initializing sync..."); + taskExecutor.execute(() -> runManualSync(email)); + } + + private void runManualSync(String email) { try { User user = userRepository.findByEmail(email) .orElseThrow(() -> new RuntimeException("User not connected to Gmail")); - + if (!user.getGmailConnected() || user.getGmailRefreshToken() == null) { log.warn("User {} attempted sync without valid Gmail connection", email); return; } String accessToken = getFreshAccessToken(user.getGmailRefreshToken()); - int found = scanInbox(accessToken, email); - + Gmail service = createGmailClient(accessToken); String currentHistoryId = service.users().getProfile("me").execute().getHistoryId().toString(); - + jobService.finalizeManualSync(email, currentHistoryId); + log.info("Manual sync successfully completed for user: {}. New jobs found: {}", email, found); + updateSyncStatus(email, "Sync completed. Found " + found + " new jobs."); + } catch (AiQuotaExceededException e) { + log.error("Manual sync paused: AI API daily quota exceeded."); + updateSyncStatus(email, "Sync paused: AI API daily quota exceeded. Please try again tomorrow."); } catch (Exception e) { log.error("Manual sync failed for {}: {}", email, e.getMessage()); + updateSyncStatus(email, "Sync failed: " + e.getMessage()); } finally { userRepository.releaseSyncLock(email); cacheEvictService.evictAllForUser(email); @@ -182,88 +221,158 @@ public void renewWatch(User user) { } public int scanInbox(String accessToken, String userEmail) { - List batchItems = new ArrayList<>(); - int totalFound = 0; - + updateSyncStatus(userEmail, "Scanning inbox..."); try { Gmail service = createGmailClient(accessToken); - String query = "newer_than:7d (" + ATS_FILTER + " OR " + SUBJECT_FILTER + ")" + EXCLUSIONS; - - String pageToken = null; - do { - ListMessagesResponse response = service.users().messages().list("me") - .setQ(query) - .setPageToken(pageToken) - .execute(); - - if (response.getMessages() != null) { - for (Message msg : response.getMessages()) { - Message fullMsg = service.users().messages().get("me", msg.getId()).setFormat("full").execute(); - long millisecondTimestamp = fullMsg.getInternalDate(); - LocalDateTime emailDate = LocalDateTime.ofInstant( - Instant.ofEpochMilli(millisecondTimestamp), ZoneOffset.UTC); - - String from = "", subj = "", replyTo=""; - if (fullMsg.getPayload().getHeaders() != null) { - for (var h : fullMsg.getPayload().getHeaders()) { - if ("From".equalsIgnoreCase(h.getName())) from = h.getValue(); - if ("Subject".equalsIgnoreCase(h.getName())) subj = h.getValue(); - if ("Reply-To".equalsIgnoreCase(h.getName())) replyTo = h.getValue(); - } - } - - if (!isSystemNoise(subj)) { - String body = extractTextFromBody(fullMsg.getPayload()); - batchItems.add(new EmailBatchItem(from, subj, replyTo, body, emailDate)); - } - } + String query = "newer_than:" + syncRange + " (" + ATS_FILTER + " OR " + SUBJECT_FILTER + ")" + EXCLUSIONS; + log.info("Scanning inbox with Gmail query: {}", query); + + List messageList = fetchMessageList(service, query); + log.info("Found {} emails matching filters.", messageList.size()); + + updateSyncStatus(userEmail, "Downloading details (0/" + messageList.size() + " - 0%)..."); + List fullMessages = fetchMessageDetailsInParallel(service, messageList, userEmail); + List batchItems = processMessagesToBatchItems(fullMessages); + log.info("Processed {} valid job-related batch items.", batchItems.size()); + + return extractAndSaveJobs(batchItems, userEmail); + } catch (AiQuotaExceededException e) { + throw e; + } catch (Exception e) { + log.error("Historical batch scan failed for {}: {}", userEmail, e.getMessage()); + return 0; + } + } + + private List fetchMessageList(Gmail service, String query) throws Exception { + List messages = new ArrayList<>(); + String pageToken = null; + do { + ListMessagesResponse response = service.users().messages().list("me") + .setQ(query) + .setPageToken(pageToken) + .execute(); + if (response.getMessages() != null) { + messages.addAll(response.getMessages()); + } + pageToken = response.getNextPageToken(); + } while (pageToken != null); + return messages; + } + + private List fetchMessageDetailsInParallel(Gmail service, List messageList, String userEmail) { + List fullMessages = new ArrayList<>(); + int chunkSize = 15; + for (int i = 0; i < messageList.size(); i += chunkSize) { + int end = Math.min(i + chunkSize, messageList.size()); + int percent = (messageList.isEmpty()) ? 100 : (int) Math.round(((double) end / messageList.size()) * 100); + String progressMsg = String.format("Downloading details (%d/%d - %d%%)...", end, messageList.size(), percent); + updateSyncStatus(userEmail, progressMsg); + log.info("Fetching details for messages {} to {} of {}...", i + 1, end, messageList.size()); + + List chunk = messageList.subList(i, end); + List> futures = chunk.stream() + .map(msg -> CompletableFuture.supplyAsync(() -> fetchMessageWithRetry(service, msg.getId()), + taskExecutor)) + .toList(); + + fullMessages.addAll(futures.stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList()); + throttleChunkExecution(); + } + return fullMessages; + } + + private Message fetchMessageWithRetry(Gmail service, String messageId) { + try { + return service.users().messages().get("me", messageId).setFormat("full").execute(); + } catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == 429) { + log.warn("Gmail API Rate Limit (429) hit. Retrying message {} after delay...", messageId); + backoffDelay(); + try { + return service.users().messages().get("me", messageId).setFormat("full").execute(); + } catch (Exception ex) { + log.error("Retry failed for message details: id {}", messageId, ex); } - pageToken = response.getNextPageToken(); - } while (pageToken != null); - - if (!batchItems.isEmpty()) { - - List> urlMaps = batchItems.parallelStream() - .map(item -> UrlParser.extractAndCleanUrls(item.body())) - .toList(); - - List extractedJobs = geminiService.extractJobsFromBatch(batchItems); - - for (JobDTO job : extractedJobs) { - hydrateJobUrl(job, urlMaps); - jobService.createOrUpdateJob(job, userEmail); - totalFound++; + } else { + log.error("Failed to fetch message details for id " + messageId, e); + } + } catch (Exception e) { + log.error("Failed to fetch message details for id " + messageId, e); + } + return null; + } + + private void backoffDelay() { + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + + private void throttleChunkExecution() { + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + + private List processMessagesToBatchItems(List fullMessages) { + List batchItems = new ArrayList<>(); + for (Message fullMsg : fullMessages) { + String from = "", subj = "", replyTo = ""; + if (fullMsg.getPayload().getHeaders() != null) { + for (var h : fullMsg.getPayload().getHeaders()) { + if ("From".equalsIgnoreCase(h.getName())) + from = h.getValue(); + if ("Subject".equalsIgnoreCase(h.getName())) + subj = h.getValue(); + if ("Reply-To".equalsIgnoreCase(h.getName())) + replyTo = h.getValue(); } - -// for (JobDTO job : extractedJobs) { -// Integer inputIdx = job.getInputIndex(); -// -// if (inputIdx != null && inputIdx >= 0 && inputIdx < batchUrlLists.size()) { -// List urlsForThisEmail = batchUrlLists.get(inputIdx); -// -// if (job.getUrlIndex() != null && job.getUrlIndex() >= 0 && job.getUrlIndex() < urlsForThisEmail.size()) { -// job.setUrl(urlsForThisEmail.get(job.getUrlIndex())); -// } -// else if (job.getUrl() == null || job.getUrl().isEmpty()) { -// job.setUrl(urlsForThisEmail.stream() -// .filter(u -> u.toLowerCase().contains("career") || u.toLowerCase().contains("job") || u.toLowerCase().contains("apply")) -// .findFirst().orElse("")); -// } -// } -// -// sanitizeUrl(job); -// -// jobService.createOrUpdateJob(job, userEmail); -// totalFound++; -// } } + if (!isSystemNoise(subj)) { + String body = extractTextFromBody(fullMsg.getPayload()); + long millisecondTimestamp = fullMsg.getInternalDate(); + LocalDateTime emailDate = LocalDateTime.ofInstant( + Instant.ofEpochMilli(millisecondTimestamp), ZoneOffset.UTC); + batchItems.add(new EmailBatchItem(from, subj, replyTo, body, emailDate)); + } + } + return batchItems; + } - } catch (Exception e) { - log.error("Historical batch scan failed for {}: {}", userEmail, e.getMessage()); + private int extractAndSaveJobs(List batchItems, String userEmail) { + int totalFound = 0; + int batchSize = 25; + int totalBatches = (int) Math.ceil((double) batchItems.size() / batchSize); + for (int i = 0; i < batchItems.size(); i += batchSize) { + int batchNum = (i / batchSize) + 1; + int percent = (totalBatches == 0) ? 100 : (int) Math.round(((double) batchNum / totalBatches) * 100); + String statusMsg = String.format("Extracting jobs with AI (batch %d/%d - %d%%)...", batchNum, totalBatches, percent); + updateSyncStatus(userEmail, statusMsg); + log.info("Running AI extraction: batch {} of {}...", batchNum, totalBatches); + + List subList = batchItems.subList(i, Math.min(i + batchSize, batchItems.size())); + List> urlMaps = subList.stream() + .map(item -> UrlParser.extractAndCleanUrls(item.body())) + .toList(); + + List extractedJobs = aiService.extractJobsFromBatch(subList); + for (JobDTO job : extractedJobs) { + hydrateJobUrl(job, urlMaps); + jobService.createOrUpdateJob(job, userEmail); + totalFound++; + } } return totalFound; } - + private void hydrateJobUrl(JobDTO job, List> urlMaps) { Integer idx = job.getInputIndex(); if (idx != null && idx >= 0 && idx < urlMaps.size()) { @@ -285,40 +394,44 @@ private void sanitizeUrl(JobDTO job) { } } } - + private String extractTextFromBody(MessagePart part) { if (part.getBody() != null && part.getBody().getData() != null) { byte[] decodedBytes = java.util.Base64.getUrlDecoder().decode(part.getBody().getData()); String content = new String(decodedBytes); - if (part.getMimeType().contains("text/plain")) return content; - if (part.getMimeType().contains("text/html")) return content.replaceAll("<[^>]*>", " "); + if (part.getMimeType().contains("text/plain")) + return content; + if (part.getMimeType().contains("text/html")) + return content.replaceAll("<[^>]*>", " "); } if (part.getParts() != null) { for (MessagePart subPart : part.getParts()) { String text = extractTextFromBody(subPart); - if (text != null && !text.isBlank()) return text; + if (text != null && !text.isBlank()) + return text; } } return ""; } - + private boolean isSystemNoise(String subject) { - if (subject == null) return true; + if (subject == null) + return true; String s = subject.toLowerCase(); return s.contains("security alert") || s.contains("sign-in") || s.contains("verification code") || s.contains("payment") || s.contains("otp"); } private Gmail createGmailClient(String token) throws Exception { - return new Gmail.Builder(HTTP_TRANSPORT, JSON_FACTORY, + return new Gmail.Builder(HTTP_TRANSPORT, JSON_FACTORY, request -> request.getHeaders().setAuthorization("Bearer " + token)) .setApplicationName(APPLICATION_NAME).build(); } @Transactional @Caching(evict = { - @CacheEvict(value = "users", key = "#email"), - @CacheEvict(value = "userEntities", key = "#email") + @CacheEvict(value = "users", key = "#email"), + @CacheEvict(value = "userEntities", key = "#email") }) public void disconnectGmail(String email) { User user = userRepository.findByEmail(email.toLowerCase()) @@ -333,7 +446,7 @@ public void disconnectGmail(String email) { user.setGmailLabelId(null); user.setGmailWatchExpiration(null); user.setGmailSyncInProgress(false); - + userRepository.saveAndFlush(user); cleanupGoogleResourcesAsync(refreshToken, labelId); @@ -342,16 +455,17 @@ public void disconnectGmail(String email) { protected void cleanupGoogleResourcesAsync(String refreshToken, String labelId) { try { String accessToken = getFreshAccessToken(refreshToken); - if (refreshToken == null) return; - + if (refreshToken == null) + return; + Gmail service = createGmailClient(accessToken); service.users().stop("me").execute(); - + restClient.post() .uri("https://oauth2.googleapis.com/revoke?token=" + refreshToken) .retrieve(); - + } catch (Exception e) { log.warn("Non-critical: Google resource cleanup failed: {}", e.getMessage()); } @@ -359,8 +473,8 @@ protected void cleanupGoogleResourcesAsync(String refreshToken, String labelId) public String getFreshAccessToken(String refreshToken) throws Exception { GoogleTokenResponse response = new GoogleRefreshTokenRequest( - HTTP_TRANSPORT, - JSON_FACTORY, + HTTP_TRANSPORT, + JSON_FACTORY, refreshToken, clientId, clientSecret).execute(); return response.getAccessToken(); } @@ -369,7 +483,8 @@ private String getOrCreateLabel(Gmail service) throws Exception { ListLabelsResponse list = service.users().labels().list("me").execute(); if (list.getLabels() != null) { for (Label l : list.getLabels()) { - if ("JobTrackerPro".equalsIgnoreCase(l.getName())) return l.getId(); + if ("JobTrackerPro".equalsIgnoreCase(l.getName())) + return l.getId(); } } Label newLabel = new Label().setName("JobTrackerPro") @@ -379,42 +494,48 @@ private String getOrCreateLabel(Gmail service) throws Exception { } private void createJobFilter(Gmail service, String labelId) throws Exception { - - String finalQuery = "(" + ATS_FILTER + " OR " + SUBJECT_FILTER + ")" + EXCLUSIONS; - - ListFiltersResponse listResponse = service.users().settings().filters().list("me").execute(); + + String finalQuery = "(" + ATS_FILTER + " OR " + SUBJECT_FILTER + ")" + EXCLUSIONS; + + ListFiltersResponse listResponse = service.users().settings().filters().list("me").execute(); if (listResponse != null && listResponse.getFilter() != null) { - - List existingFilters = listResponse.getFilter(); - + + List existingFilters = listResponse.getFilter(); + for (Filter existingFilter : existingFilters) { - if (existingFilter.getAction() != null && - existingFilter.getAction().getAddLabelIds() != null && - existingFilter.getAction().getAddLabelIds().contains(labelId)) { - - log.info("Found outdated JobTrackerPro filter (ID: {}). Deleting for update...", existingFilter.getId()); + if (existingFilter.getAction() != null && + existingFilter.getAction().getAddLabelIds() != null && + existingFilter.getAction().getAddLabelIds().contains(labelId)) { + + log.info("Found outdated JobTrackerPro filter (ID: {}). Deleting for update...", + existingFilter.getId()); service.users().settings().filters().delete("me", existingFilter.getId()).execute(); } } } - - Filter newFilter = new Filter() + + Filter newFilter = new Filter() .setCriteria(new FilterCriteria().setQuery(finalQuery)) .setAction(new FilterAction().setAddLabelIds(List.of(labelId))); - - try { - service.users().settings().filters().create("me", newFilter).execute(); + + try { + service.users().settings().filters().create("me", newFilter).execute(); log.info("Gmail Filter created successfully."); - } catch (GoogleJsonResponseException e) { - if (e.getStatusCode() == 409 || - (e.getStatusCode() == 400 && e.getDetails().getMessage().contains("Filter already exists"))) { + } catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == 409 || + (e.getStatusCode() == 400 && e.getDetails().getMessage().contains("Filter already exists"))) { log.info("Gmail filter already exists, skipping creation."); } else { log.error("Failed to create Gmail filter: {}", e.getDetails().getMessage()); - throw e; + throw e; } } } + + private void updateSyncStatus(String email, String status) { + userRepository.updateSyncStatus(email, status); + cacheEvictService.evictAllForUser(email); + } } \ No newline at end of file diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailWebhookService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailWebhookService.java index ba1995f..a7c264e 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailWebhookService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/GmailWebhookService.java @@ -8,7 +8,7 @@ import com.thughari.jobtrackerpro.dto.EmailBatchItem; import com.thughari.jobtrackerpro.dto.JobDTO; import com.thughari.jobtrackerpro.entity.User; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; import com.thughari.jobtrackerpro.repo.UserRepository; import com.thughari.jobtrackerpro.util.CacheEvictService; @@ -31,7 +31,7 @@ @Slf4j public class GmailWebhookService { - private final GeminiService geminiService; + private final AiExtractionService aiService; private final JobService jobService; private final UserRepository userRepository; private final CacheEvictService cacheEvictService; @@ -42,8 +42,8 @@ public class GmailWebhookService { @Value("${spring.security.oauth2.client.registration.google.client-secret}") private String clientSecret; - public GmailWebhookService(GeminiService geminiService, JobService jobService, UserRepository userRepository, CacheEvictService cacheEvictService) { - this.geminiService = geminiService; + public GmailWebhookService(AiExtractionService aiService, JobService jobService, UserRepository userRepository, CacheEvictService cacheEvictService) { + this.aiService = aiService; this.jobService = jobService; this.userRepository = userRepository; this.cacheEvictService = cacheEvictService; @@ -84,7 +84,7 @@ public void processHistorySync(String userEmail) { if (!batchItems.isEmpty()) { log.info("Ingesting batch of {} emails for {}", batchItems.size(), email); - List extractedJobs = geminiService.extractJobsFromBatch(batchItems); + List extractedJobs = aiService.extractJobsFromBatch(batchItems); jobService.saveBatchResults(email, batchItems, extractedJobs); } } catch (Exception e) { diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/GroqExtractionService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/GroqExtractionService.java new file mode 100644 index 0000000..3e7eb90 --- /dev/null +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/GroqExtractionService.java @@ -0,0 +1,296 @@ +package com.thughari.jobtrackerpro.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.thughari.jobtrackerpro.dto.EmailBatchItem; +import com.thughari.jobtrackerpro.dto.JobDTO; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; +import com.thughari.jobtrackerpro.exception.AiQuotaExceededException; +import com.thughari.jobtrackerpro.util.UrlParser; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClient; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +@ConditionalOnProperty(name = "groq.api.enabled", havingValue = "true") +public class GroqExtractionService implements AiExtractionService { + + private final RestClient restClient; + private final ObjectMapper objectMapper; + + @Value("${groq.api.key}") + private String apiKey; + + @Value("${groq.api.url}") + private String apiUrl; + + @Value("${groq.api.model:llama-3.1-8b-instant}") + private String apiModel; + + public GroqExtractionService() { + this.restClient = RestClient.create(); + this.objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + @Override + public JobDTO extractJobFromEmail(String from, String subject, String body) { + String prompt = buildPrompt(from, subject, body); + + try { + Map requestBody = Map.of( + "model", apiModel, + "messages", List.of( + Map.of( + "role", "user", + "content", prompt + ) + ), + "max_tokens", 800 + ); + + String response = restClient.post() + .uri(apiUrl) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey) + .contentType(MediaType.APPLICATION_JSON) + .body(requestBody) + .retrieve() + .body(String.class); + + return parseOpenAiResponse(response); + + } catch (HttpClientErrorException e) { + log.error("Groq API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("GROQ_API_ERROR", e); + } catch (Exception e) { + log.error("Groq AI Extraction failed or timed out", e); + return null; + } + } + + @Override + public List extractJobsFromBatch(List items) { + if (items == null || items.isEmpty()) return List.of(); + + String prompt = buildBatchPrompt(items); + + try { + Map requestBody = Map.of( + "model", apiModel, + "messages", List.of( + Map.of("role", "user", "content", prompt) + ), + "max_tokens", 2500 + ); + + String response = restClient.post() + .uri(apiUrl) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey) + .contentType(MediaType.APPLICATION_JSON) + .body(requestBody) + .retrieve() + .body(String.class); + + return parseBulkOpenAiResponse(response); + + } catch (HttpClientErrorException e) { + log.error("Groq API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("GROQ_API_ERROR", e); + } catch (Exception e) { + log.error("Groq Bulk AI Extraction failed", e); + return List.of(); + } + } + + private JobDTO parseOpenAiResponse(String rawResponse) { + try { + JsonNode root = objectMapper.readTree(rawResponse); + JsonNode choices = root.path("choices"); + + if (choices.isMissingNode() || choices.isEmpty()) { + return null; + } + + String contentText = choices.get(0).path("message").path("content").asText(); + contentText = contentText.replaceAll("```json", "").replaceAll("```", "").trim(); + + if (contentText.equalsIgnoreCase("null")) { + log.info("Groq AI determined this email is NOT a job application."); + return null; + } + + JobDTO job = objectMapper.readValue(contentText, JobDTO.class); + + LocalDateTime now = LocalDateTime.now(); + job.setAppliedDate(now); + job.setUpdatedAt(now); + + job.setStage(mapStatusToStage(job.getStatus())); + + if ("Rejected".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("failed"); + } else if ("Offer Received".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("passed"); + } else { + job.setStageStatus("active"); + } + + return job; + + } catch (Exception e) { + log.error("Failed to parse Groq AI response: {}", rawResponse); + return null; + } + } + + private List parseBulkOpenAiResponse(String rawResponse) { + try { + JsonNode root = objectMapper.readTree(rawResponse); + JsonNode choices = root.path("choices"); + + if (choices.isMissingNode() || choices.isEmpty()) return List.of(); + + String contentText = choices.get(0).path("message").path("content").asText(); + + // Extract just the JSON array between first '[' and last ']' + int start = contentText.indexOf('['); + int end = contentText.lastIndexOf(']'); + if (start == -1 || end == -1 || end <= start) { + log.warn("Groq bulk response has no JSON array: {}", contentText.substring(0, Math.min(200, contentText.length()))); + return List.of(); + } + contentText = contentText.substring(start, end + 1); + + // Sanitize malformed JSON from LLM (e.g. missing opening quotes on keys) + contentText = contentText.replaceAll(",(?:\\s*)([a-zA-Z_]+)\"\\s*:", ",\"$1\":"); + contentText = contentText.replaceAll("\\{(?:\\s*)([a-zA-Z_]+)\"\\s*:", "{\"$1\":"); + + List jobs = objectMapper.readValue(contentText, new TypeReference>() {}); + + LocalDateTime now = LocalDateTime.now(); + jobs.forEach(job -> { + job.setAppliedDate(now); + job.setUpdatedAt(now); + job.setStage(mapStatusToStage(job.getStatus())); + + if ("Rejected".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("failed"); + } else if ("Offer Received".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("passed"); + } else { + job.setStageStatus("active"); + } + }); + + return jobs; + + } catch (Exception e) { + log.error("Failed to parse Groq Bulk AI response: {}", rawResponse); + return List.of(); + } + } + + private String buildPrompt(String from, String subject, String body) { + String safeBody = (body != null) ? (body.length() > 1200 ? body.substring(0, 1200) : body) : ""; + + return """ + Act as a strict ATS Data Extractor. + TASK: Analyze email. Return JSON job object if job-related. Else return null. Skip spam/receipts. + + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link. + 7. SALARY: Extract if present, else 0.0. + + MULTILINGUAL: Output values in English. + + EMAIL: + From: %s + Subject: %s + Body: %s + + OUTPUT: Raw JSON (no markdown) or null. + """.formatted(from, subject, safeBody); + } + + private String buildBatchPrompt(List items) { + StringBuilder emailListBuilder = new StringBuilder(); + for (int i = 0; i < items.size(); i++) { + EmailBatchItem item = items.get(i); + String rawBody = item.body() == null ? "" : item.body(); + + String trimmed = UrlParser.trimNoise(rawBody); + String safeBody = trimmed.length() > 1200 ? trimmed.substring(0, 1200) : trimmed; + List urls = UrlParser.extractAndCleanUrls(rawBody); + + emailListBuilder.append(""" + [EMAIL %d] + From: %s + Reply-To: %s + Subject: %s + Body: %s + URLs: + %s + """.formatted( + i, item.from(), item.replyTo(), item.subject(), safeBody, buildUrlIndexList(urls) + )); + } + + return """ + Act as a strict ATS Data Extractor. + TASK: Analyze emails. Return JSON array of job objects for job-related emails only. Skip spam/receipts. + + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link from URLs list, else company website. + 7. SALARY: Extract if present, else 0.0. + 8. INPUT INDEX: Must match [EMAIL X] index. + + MULTILINGUAL: Output values in English. + + EMAILS: + %s + + OUTPUT FORMAT: Raw JSON array only (no markdown). + """.formatted(emailListBuilder.toString()); + } + + private String buildUrlIndexList(List urls) { + if (urls.isEmpty()) return "None\\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < urls.size(); i++) { + sb.append(i).append(": ").append(urls.get(i)).append("\\n"); + } + return sb.toString(); + } + + private Integer mapStatusToStage(String status) { + if (status == null) return 1; + if (status.contains("Offer")) return 4; + if (status.contains("Interview")) return 3; + if (status.contains("Shortlisted") || status.contains("exam") || status.contains("test") || status.contains("hackerrank")) return 2; + + return 1; + } +} diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/IngestionService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/IngestionService.java index e199379..0730c2b 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/IngestionService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/IngestionService.java @@ -2,7 +2,7 @@ import com.thughari.jobtrackerpro.dto.JobDTO; import com.thughari.jobtrackerpro.entity.User; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; import com.thughari.jobtrackerpro.repo.UserRepository; import lombok.extern.slf4j.Slf4j; @@ -13,12 +13,12 @@ @Slf4j public class IngestionService { - private final GeminiService geminiService; + private final AiExtractionService aiService; private final JobService jobService; private final UserRepository userRepository; - public IngestionService(GeminiService geminiService, JobService jobService, UserRepository userRepository) { - this.geminiService = geminiService; + public IngestionService(AiExtractionService aiService, JobService jobService, UserRepository userRepository) { + this.aiService = aiService; this.jobService = jobService; this.userRepository = userRepository; } @@ -33,8 +33,8 @@ public void handleManualForward(String from, String subject, String body, String return; } - log.info("Forwarding email to Gemini AI for user: {}", userEmail); - JobDTO job = geminiService.extractJobFromEmail(from, subject, body); + log.info("Forwarding email to AI for user: {}", userEmail); + JobDTO job = aiService.extractJobFromEmail(from, subject, body); if (job != null) { jobService.createOrUpdateJob(job, userEmail); diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/JobService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/JobService.java index 01977dd..f0afdbd 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/JobService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/JobService.java @@ -74,7 +74,26 @@ public DashboardResponse getDashboardData(String email) { List jobs = jobRepository.findByUserEmailOrderByUpdatedAtDesc(email); DashboardResponse response = new DashboardResponse(); + response.setStats(calculateStats(jobs)); + response.setStatusChart(calculateStatusDistribution(jobs)); + response.setMonthlyChart(calculateMonthlyDistribution(jobs)); + response.setInterviewChart(calculateInterviewProgress(jobs)); + + Optional userOpt = userRepository.findByEmail(email); + boolean syncInProgress = userOpt + .map(User::getGmailSyncInProgress) + .orElse(false); + response.setGmailSyncInProgress(syncInProgress); + + String syncStatus = userOpt + .map(User::getGmailSyncStatus) + .orElse(null); + response.setGmailSyncStatus(syncStatus); + + return response; + } + private DashboardStatsDTO calculateStats(List jobs) { long total = jobs.size(); long active = jobs.parallelStream().filter(j -> j.getStatus() != null && !j.getStatus().equals("Rejected") && !j.getStatus().equals("Offer Received")).count(); @@ -85,14 +104,17 @@ public DashboardResponse getDashboardData(String email) { long offers = jobs.parallelStream().filter(j -> "Offer Received".equals(j.getStatus())).count(); long activeInterviews = jobs.parallelStream().filter(j -> "Interview Scheduled".equals(j.getStatus())).count(); - response.setStats(new DashboardStatsDTO(total, active, interviews, activeInterviews, offers)); + return new DashboardStatsDTO(total, active, interviews, activeInterviews, offers); + } + private List calculateStatusDistribution(List jobs) { Map statusMap = jobs.parallelStream() - .collect(Collectors.groupingBy(Job::getStatus, Collectors.counting())); - response.setStatusChart(mapToChartData(statusMap)); + .collect(Collectors.groupingBy(j -> sanitizeStatus(j.getStatus()), Collectors.counting())); + return mapToChartData(statusMap); + } + private List calculateMonthlyDistribution(List jobs) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM yy"); - List last6MonthKeys = new ArrayList<>(); LocalDateTime temp = LocalDateTime.now(); for (int i = 5; i >= 0; i--) { @@ -112,16 +134,18 @@ public DashboardResponse getDashboardData(String email) { } } } - response.setMonthlyChart(mapToChartData(monthMap)); + return mapToChartData(monthMap); + } + private List calculateInterviewProgress(List jobs) { + long total = jobs.size(); long interviewCount = jobs.parallelStream() .filter(j -> j.getStage() != null && j.getStage() >= 3) - .count(); response.setInterviewChart(List.of( + .count(); + return List.of( new ChartData("Interviewed", interviewCount), new ChartData("Not Interviewed", total > 0 ? total - interviewCount : 0) - )); - - return response; + ); } @Caching(evict = { @@ -152,6 +176,7 @@ public JobDTO updateJob(UUID id, JobDTO dto, String email) { LocalDateTime originalAppliedDate = existingJob.getAppliedDate(); BeanUtils.copyProperties(dto, existingJob, "id", "userEmail", "appliedDate", "updatedAt"); + existingJob.setStatus(sanitizeStatus(existingJob.getStatus())); existingJob.setAppliedDate(originalAppliedDate); existingJob.setUpdatedAt(LocalDateTime.now()); return convertToDto(jobRepository.save(existingJob)); @@ -318,8 +343,8 @@ private Job findBestMatch(List existingJobs, JobDTO incoming) { } private void updateExistingJobFromEmail(Job existingJob, JobDTO incoming) { - String currentStatus = (existingJob.getStatus() != null) ? existingJob.getStatus() : ""; - String incomingStatus = (incoming.getStatus() != null) ? incoming.getStatus() : ""; + String currentStatus = sanitizeStatus(existingJob.getStatus()); + String incomingStatus = sanitizeStatus(incoming.getStatus()); LocalDateTime incomingTime = incoming.getUpdatedAt(); LocalDateTime existingTime = existingJob.getUpdatedAt(); @@ -404,15 +429,29 @@ private Set tokenize(String text) { private JobDTO convertToDto(Job job) { JobDTO dto = new JobDTO(); BeanUtils.copyProperties(job, dto); + dto.setStatus(sanitizeStatus(job.getStatus())); return dto; } private Job convertToEntity(JobDTO dto) { Job job = new Job(); BeanUtils.copyProperties(dto, job); + job.setStatus(sanitizeStatus(job.getStatus())); return job; } + private String sanitizeStatus(String status) { + if (status == null || status.isBlank()) return "Applied"; + + String s = status.trim(); + if (s.equalsIgnoreCase("Shortlisted")) return "Shortlisted"; + if (s.equalsIgnoreCase("Interview Scheduled")) return "Interview Scheduled"; + if (s.equalsIgnoreCase("Offer Received")) return "Offer Received"; + if (s.equalsIgnoreCase("Rejected")) return "Rejected"; + + return "Applied"; + } + private List mapToChartData(Map map) { return map.entrySet().parallelStream() .map(e -> new ChartData(e.getKey(), e.getValue())) diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/OpenRouterExtractionService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/OpenRouterExtractionService.java new file mode 100644 index 0000000..0b68bc7 --- /dev/null +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/OpenRouterExtractionService.java @@ -0,0 +1,296 @@ +package com.thughari.jobtrackerpro.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.thughari.jobtrackerpro.dto.EmailBatchItem; +import com.thughari.jobtrackerpro.dto.JobDTO; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; +import com.thughari.jobtrackerpro.exception.AiQuotaExceededException; +import com.thughari.jobtrackerpro.util.UrlParser; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClient; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +@ConditionalOnProperty(name = "openrouter.api.enabled", havingValue = "true") +public class OpenRouterExtractionService implements AiExtractionService { + + private final RestClient restClient; + private final ObjectMapper objectMapper; + + @Value("${openrouter.api.key}") + private String apiKey; + + @Value("${openrouter.api.url}") + private String apiUrl; + + @Value("${openrouter.api.model:meta-llama/llama-3.1-8b-instruct}") + private String apiModel; + + public OpenRouterExtractionService() { + this.restClient = RestClient.create(); + this.objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + @Override + public JobDTO extractJobFromEmail(String from, String subject, String body) { + String prompt = buildPrompt(from, subject, body); + + try { + Map requestBody = Map.of( + "model", apiModel, + "messages", List.of( + Map.of( + "role", "user", + "content", prompt + ) + ), + "max_tokens", 800 + ); + + String response = restClient.post() + .uri(apiUrl) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey) + .contentType(MediaType.APPLICATION_JSON) + .body(requestBody) + .retrieve() + .body(String.class); + + return parseOpenAiResponse(response); + + } catch (HttpClientErrorException e) { + log.error("OpenRouter API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("OPENROUTER_API_ERROR", e); + } catch (Exception e) { + log.error("OpenRouter AI Extraction failed or timed out", e); + return null; + } + } + + @Override + public List extractJobsFromBatch(List items) { + if (items == null || items.isEmpty()) return List.of(); + + String prompt = buildBatchPrompt(items); + + try { + Map requestBody = Map.of( + "model", apiModel, + "messages", List.of( + Map.of("role", "user", "content", prompt) + ), + "max_tokens", 2500 + ); + + String response = restClient.post() + .uri(apiUrl) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey) + .contentType(MediaType.APPLICATION_JSON) + .body(requestBody) + .retrieve() + .body(String.class); + + return parseBulkOpenAiResponse(response); + + } catch (HttpClientErrorException e) { + log.error("OpenRouter API error: {}", e.getResponseBodyAsString()); + throw new AiQuotaExceededException("OPENROUTER_API_ERROR", e); + } catch (Exception e) { + log.error("OpenRouter Bulk AI Extraction failed", e); + return List.of(); + } + } + + private JobDTO parseOpenAiResponse(String rawResponse) { + try { + JsonNode root = objectMapper.readTree(rawResponse); + JsonNode choices = root.path("choices"); + + if (choices.isMissingNode() || choices.isEmpty()) { + return null; + } + + String contentText = choices.get(0).path("message").path("content").asText(); + contentText = contentText.replaceAll("```json", "").replaceAll("```", "").trim(); + + if (contentText.equalsIgnoreCase("null")) { + log.info("OpenRouter AI determined this email is NOT a job application."); + return null; + } + + JobDTO job = objectMapper.readValue(contentText, JobDTO.class); + + LocalDateTime now = LocalDateTime.now(); + job.setAppliedDate(now); + job.setUpdatedAt(now); + + job.setStage(mapStatusToStage(job.getStatus())); + + if ("Rejected".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("failed"); + } else if ("Offer Received".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("passed"); + } else { + job.setStageStatus("active"); + } + + return job; + + } catch (Exception e) { + log.error("Failed to parse OpenRouter AI response: {}", rawResponse); + return null; + } + } + + private List parseBulkOpenAiResponse(String rawResponse) { + try { + JsonNode root = objectMapper.readTree(rawResponse); + JsonNode choices = root.path("choices"); + + if (choices.isMissingNode() || choices.isEmpty()) return List.of(); + + String contentText = choices.get(0).path("message").path("content").asText(); + + // Extract just the JSON array between first '[' and last ']' + int start = contentText.indexOf('['); + int end = contentText.lastIndexOf(']'); + if (start == -1 || end == -1 || end <= start) { + log.warn("OpenRouter bulk response has no JSON array: {}", contentText.substring(0, Math.min(200, contentText.length()))); + return List.of(); + } + contentText = contentText.substring(start, end + 1); + + // Sanitize malformed JSON from LLM (e.g. missing opening quotes on keys) + contentText = contentText.replaceAll(",(?:\\s*)([a-zA-Z_]+)\"\\s*:", ",\"$1\":"); + contentText = contentText.replaceAll("\\{(?:\\s*)([a-zA-Z_]+)\"\\s*:", "{\"$1\":"); + + List jobs = objectMapper.readValue(contentText, new TypeReference>() {}); + + LocalDateTime now = LocalDateTime.now(); + jobs.forEach(job -> { + job.setAppliedDate(now); + job.setUpdatedAt(now); + job.setStage(mapStatusToStage(job.getStatus())); + + if ("Rejected".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("failed"); + } else if ("Offer Received".equalsIgnoreCase(job.getStatus())) { + job.setStageStatus("passed"); + } else { + job.setStageStatus("active"); + } + }); + + return jobs; + + } catch (Exception e) { + log.error("Failed to parse OpenRouter Bulk AI response: {}", rawResponse); + return List.of(); + } + } + + private String buildPrompt(String from, String subject, String body) { + String safeBody = (body != null) ? (body.length() > 1200 ? body.substring(0, 1200) : body) : ""; + + return """ + Act as a strict ATS Data Extractor. + TASK: Analyze email. Return JSON job object if job-related. Else return null. Skip spam/receipts. + + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link. + 7. SALARY: Extract if present, else 0.0. + + MULTILINGUAL: Output values in English. + + EMAIL: + From: %s + Subject: %s + Body: %s + + OUTPUT: Raw JSON (no markdown) or null. + """.formatted(from, subject, safeBody); + } + + private String buildBatchPrompt(List items) { + StringBuilder emailListBuilder = new StringBuilder(); + for (int i = 0; i < items.size(); i++) { + EmailBatchItem item = items.get(i); + String rawBody = item.body() == null ? "" : item.body(); + + String trimmed = UrlParser.trimNoise(rawBody); + String safeBody = trimmed.length() > 1200 ? trimmed.substring(0, 1200) : trimmed; + List urls = UrlParser.extractAndCleanUrls(rawBody); + + emailListBuilder.append(""" + [EMAIL %d] + From: %s + Reply-To: %s + Subject: %s + Body: %s + URLs: + %s + """.formatted( + i, item.from(), item.replyTo(), item.subject(), safeBody, buildUrlIndexList(urls) + )); + } + + return """ + Act as a strict ATS Data Extractor. + TASK: Analyze emails. Return JSON array of job objects for job-related emails only. Skip spam/receipts. + + RULES: + 1. COMPANY: Hiring company. Fallback to email domain. Ignore generic domains. Default: "Unknown Company". + 2. ROLE: Job title. Default: "Software Engineer". + 3. STATUS: "Applied", "Shortlisted", "Interview Scheduled", "Offer Received", or "Rejected". + 4. NOTES: 1-sentence summary. + 5. LOCATION: City/Country. Default: "Remote". + 6. URL: Best call-to-action link from URLs list, else company website. + 7. SALARY: Extract if present, else 0.0. + 8. INPUT INDEX: Must match [EMAIL X] index. + + MULTILINGUAL: Output values in English. + + EMAILS: + %s + + OUTPUT FORMAT: Raw JSON array only (no markdown). + """.formatted(emailListBuilder.toString()); + } + + private String buildUrlIndexList(List urls) { + if (urls.isEmpty()) return "None\\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < urls.size(); i++) { + sb.append(i).append(": ").append(urls.get(i)).append("\\n"); + } + return sb.toString(); + } + + private Integer mapStatusToStage(String status) { + if (status == null) return 1; + if (status.contains("Offer")) return 4; + if (status.contains("Interview")) return 3; + if (status.contains("Shortlisted") || status.contains("exam") || status.contains("test") || status.contains("hackerrank")) return 2; + + return 1; + } +} diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/SmartExtractionService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/SmartExtractionService.java index 16acff1..ef1fe17 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/SmartExtractionService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/SmartExtractionService.java @@ -2,7 +2,8 @@ import com.thughari.jobtrackerpro.dto.EmailBatchItem; import com.thughari.jobtrackerpro.dto.JobDTO; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; +import com.thughari.jobtrackerpro.exception.AiQuotaExceededException; import com.thughari.jobtrackerpro.util.TemplateParser; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; @@ -14,37 +15,49 @@ @Service @Primary @Slf4j -public class SmartExtractionService implements GeminiService { +public class SmartExtractionService implements AiExtractionService { - private final GeminiService activeService; + private final List providers = new ArrayList<>(); - public SmartExtractionService(List services) { - // Find the actual delegate service (GeminiExtractionService or MockGeminiService) - this.activeService = services.stream() - .filter(s -> s != this) - .findFirst() - .orElseThrow(() -> new IllegalStateException("No active GeminiService implementation found!")); - log.info("SmartExtractionService initialized. Delegating non-template extractions to: {}", - this.activeService.getClass().getSimpleName()); + public SmartExtractionService(List services) { + // Order the providers: Gemini -> Groq -> OpenRouter -> Mock + // We filter out 'this' instance to avoid infinite recursion + services.stream().filter(s -> s instanceof GeminiExtractionService).findFirst().ifPresent(providers::add); + services.stream().filter(s -> s instanceof GroqExtractionService).findFirst().ifPresent(providers::add); + services.stream().filter(s -> s instanceof OpenRouterExtractionService).findFirst().ifPresent(providers::add); + services.stream().filter(s -> s.getClass().getSimpleName().contains("Mock")).findFirst().ifPresent(providers::add); + + log.info("SmartExtractionService initialized with fallback chain: {}", + providers.stream().map(p -> p.getClass().getSimpleName()).toList()); } @Override public JobDTO extractJobFromEmail(String from, String subject, String body) { - // log.info("[DEBUG] SmartExtractionService - From: '{}', Subject: '{}'", from, subject); - // log.info("[DEBUG] Body: {}", body); - try { JobDTO manualJob = TemplateParser.parse(from, subject, body); if (manualJob != null) { - // log.info("Successfully matched and manually extracted job for company: {}", manualJob.getCompany()); return manualJob; } } catch (Exception e) { - log.warn("Manual parsing error, falling back to delegation: {}", e.getMessage()); + log.warn("Manual parsing error, falling back to AI delegation: {}", e.getMessage()); } - log.info("Bypassing manual templates. Delegating extraction to: {}", activeService.getClass().getSimpleName()); - return activeService.extractJobFromEmail(from, subject, body); + for (int i = 0; i < providers.size(); i++) { + AiExtractionService provider = providers.get(i); + try { + log.info("Attempting extraction with provider: {}", provider.getClass().getSimpleName()); + return provider.extractJobFromEmail(from, subject, body); + } catch (AiQuotaExceededException e) { + log.warn("Quota exceeded for provider {}: {}", provider.getClass().getSimpleName(), e.getMessage()); + if (i < providers.size() - 1) { + log.info("Falling back to next provider..."); + } else { + log.error("All AI providers exhausted their quotas!"); + throw e; // No more fallbacks + } + } + } + return null; } @Override @@ -57,9 +70,6 @@ public List extractJobsFromBatch(List items) { for (int i = 0; i < items.size(); i++) { EmailBatchItem item = items.get(i); - // log.info("[DEBUG] SmartExtractionService batch item {} - From: '{}', Subject: '{}'", i, item.from(), item.subject()); - // log.info("[DEBUG] Body: {}", item.body()); - JobDTO parsed = null; try { parsed = TemplateParser.parse(item.from(), item.subject(), item.body()); @@ -69,7 +79,6 @@ public List extractJobsFromBatch(List items) { if (parsed != null) { parsed.setInputIndex(i); - // log.info("Successfully matched and manually extracted batch item {} for company: {}", i, parsed.getCompany()); results.add(parsed); } else { remainingItems.add(item); @@ -78,9 +87,23 @@ public List extractJobsFromBatch(List items) { } if (!remainingItems.isEmpty()) { - log.info("Bypassing manual templates for {}/{} items. Delegating to: {}", - remainingItems.size(), items.size(), activeService.getClass().getSimpleName()); - List delegatedJobs = activeService.extractJobsFromBatch(remainingItems); + List delegatedJobs = new ArrayList<>(); + for (int i = 0; i < providers.size(); i++) { + AiExtractionService provider = providers.get(i); + try { + log.info("Attempting batch extraction with provider: {}", provider.getClass().getSimpleName()); + delegatedJobs = provider.extractJobsFromBatch(remainingItems); + break; // Success, stop trying other providers + } catch (AiQuotaExceededException e) { + log.warn("Quota exceeded for provider {}: {}", provider.getClass().getSimpleName(), e.getMessage()); + if (i < providers.size() - 1) { + log.info("Falling back to next provider..."); + } else { + log.error("All AI providers exhausted their quotas!"); + throw e; + } + } + } for (JobDTO job : delegatedJobs) { if (job.getInputIndex() != null && job.getInputIndex() >= 0 && job.getInputIndex() < originalIndexes.size()) { diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockGeminiService.java b/backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionService.java similarity index 96% rename from backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockGeminiService.java rename to backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionService.java index 1c08fc0..cfecde3 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockGeminiService.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionService.java @@ -2,7 +2,7 @@ import com.thughari.jobtrackerpro.dto.EmailBatchItem; import com.thughari.jobtrackerpro.dto.JobDTO; -import com.thughari.jobtrackerpro.interfaces.GeminiService; +import com.thughari.jobtrackerpro.interfaces.AiExtractionService; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -18,7 +18,7 @@ @Service @ConditionalOnProperty(name = "app.gemini.enabled", havingValue = "false", matchIfMissing = true) -public class MockGeminiService implements GeminiService { +public class MockAiExtractionService implements AiExtractionService { private static final Pattern COMPANY_PATTERN = Pattern.compile("(?:to|at)\\s+([A-Z][A-Za-z0-9\\s]+)"); diff --git a/backend/src/main/java/com/thughari/jobtrackerpro/util/UrlParser.java b/backend/src/main/java/com/thughari/jobtrackerpro/util/UrlParser.java index c19db31..9f8d4e3 100644 --- a/backend/src/main/java/com/thughari/jobtrackerpro/util/UrlParser.java +++ b/backend/src/main/java/com/thughari/jobtrackerpro/util/UrlParser.java @@ -73,6 +73,7 @@ public static String trimNoise(String body) { if (index > 0) cleanBody = cleanBody.substring(0, index); } - return cleanBody.length() > 3000 ? cleanBody.substring(0, 3000) : cleanBody; + cleanBody = cleanBody.replaceAll("\\s+", " ").trim(); + return cleanBody.length() > 1200 ? cleanBody.substring(0, 1200) : cleanBody; } } \ No newline at end of file diff --git a/backend/src/main/resources/application-dev.properties b/backend/src/main/resources/application-dev.properties index 03b5be0..2e7d017 100644 --- a/backend/src/main/resources/application-dev.properties +++ b/backend/src/main/resources/application-dev.properties @@ -21,7 +21,20 @@ spring.jpa.show-sql=true # Gemini AI app.gemini.enabled=true gemini.api.key=${GEMINI_API_KEY} -gemini.api.url=https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-2.5-flash-lite:generateContent +gemini.api.url=${GEMINI_API_URL:https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent} + +# Groq AI (Fallback) +groq.api.enabled=true +groq.api.key=${GROQ_API_KEY} +groq.api.url=${GROQ_API_URL:https://api.groq.com/openai/v1/chat/completions} +groq.api.model=${GROQ_MODEL:llama-3.1-8b-instant} + +# OpenRouter AI (Fallback 2) +openrouter.api.enabled=true +openrouter.api.key=${OPENROUTER_API_KEY} +openrouter.api.url=${OPENROUTER_API_URL:https://openrouter.ai/api/v1/chat/completions} +openrouter.api.model=${OPENROUTER_MODEL:meta-llama/llama-3.1-8b-instruct} +app.gmail.sync-range=${APP_GMAIL_SYNC_RANGE:30d} # UI url diff --git a/backend/src/main/resources/application-local.properties b/backend/src/main/resources/application-local.properties index 34b2225..729ac01 100644 --- a/backend/src/main/resources/application-local.properties +++ b/backend/src/main/resources/application-local.properties @@ -13,13 +13,25 @@ spring.mail.port=1025 spring.mail.properties.mail.smtp.auth=false spring.mail.properties.mail.smtp.starttls.enable=false -# Default for open-source -app.gemini.enabled=false - -# app.gemini.enabled=true - -# gemini.api.key=${GEMINI_API_KEY} -# gemini.api.url=https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-2.5-flash-lite:generateContent +# Gemini AI Integration (Google AI Studio) +# To enable, set app.gemini.enabled=true and provide your GEMINI_API_KEY +app.gemini.enabled=${APP_GEMINI_ENABLED:false} +gemini.api.key=${GEMINI_API_KEY:} +# Standard Google AI Studio Gemini API Endpoint +gemini.api.url=${GEMINI_API_URL:https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent} + +# Groq AI Integration (Fallback) +groq.api.enabled=${GROQ_API_ENABLED:true} +groq.api.key=${GROQ_API_KEY:} +groq.api.url=${GROQ_API_URL:https://api.groq.com/openai/v1/chat/completions} +groq.api.model=${GROQ_MODEL:llama-3.1-8b-instant} + +# OpenRouter AI Integration (Fallback 2) +openrouter.api.enabled=true +openrouter.api.key=${OPENROUTER_API_KEY:} +openrouter.api.url=${OPENROUTER_API_URL:https://openrouter.ai/api/v1/chat/completions} +openrouter.api.model=${OPENROUTER_MODEL:meta-llama/llama-3.1-8b-instruct} +app.gmail.sync-range=${APP_GMAIL_SYNC_RANGE:30d} app.storage.type=local diff --git a/backend/src/main/resources/application-prod.properties b/backend/src/main/resources/application-prod.properties index 0fe8f1a..c2da88c 100644 --- a/backend/src/main/resources/application-prod.properties +++ b/backend/src/main/resources/application-prod.properties @@ -36,7 +36,20 @@ logging.level.com.thughari.jobtrackerpro.scheduler=INFO # Gemini AI app.gemini.enabled=true gemini.api.key=${GEMINI_API_KEY} -gemini.api.url=https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-2.5-flash-lite:generateContent +gemini.api.url=${GEMINI_API_URL:https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent} + +# Groq AI (Fallback) +groq.api.enabled=true +groq.api.key=${GROQ_API_KEY} +groq.api.url=${GROQ_API_URL:https://api.groq.com/openai/v1/chat/completions} +groq.api.model=${GROQ_MODEL:llama-3.1-8b-instant} + +# OpenRouter AI (Fallback 2) +openrouter.api.enabled=true +openrouter.api.key=${OPENROUTER_API_KEY} +openrouter.api.url=${OPENROUTER_API_URL:https://openrouter.ai/api/v1/chat/completions} +openrouter.api.model=${OPENROUTER_MODEL:meta-llama/llama-3.1-8b-instruct} +app.gmail.sync-range=${APP_GMAIL_SYNC_RANGE:30d} # UI url app.ui.url=https://jobtrackerpro.in diff --git a/backend/src/test/java/com/thughari/jobtrackerpro/service/JobServiceTest.java b/backend/src/test/java/com/thughari/jobtrackerpro/service/JobServiceTest.java index 99c41a1..a8ec986 100644 --- a/backend/src/test/java/com/thughari/jobtrackerpro/service/JobServiceTest.java +++ b/backend/src/test/java/com/thughari/jobtrackerpro/service/JobServiceTest.java @@ -5,6 +5,7 @@ import com.thughari.jobtrackerpro.entity.Job; import com.thughari.jobtrackerpro.exception.ResourceNotFoundException; import com.thughari.jobtrackerpro.repo.JobRepository; +import com.thughari.jobtrackerpro.repo.UserRepository; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -27,6 +28,9 @@ class JobServiceTest { @Mock private JobRepository jobRepository; + @Mock + private UserRepository userRepository; + @InjectMocks private JobService jobService; @@ -183,6 +187,7 @@ void getDashboardData_calculatesSummaryAndCharts() { offer.setAppliedDate(LocalDateTime.of(2025, 2, 20, 9, 0)); when(jobRepository.findByUserEmailOrderByUpdatedAtDesc(EMAIL)).thenReturn(List.of(applied, interview, offer)); + when(userRepository.findByEmail(EMAIL)).thenReturn(Optional.empty()); DashboardResponse response = jobService.getDashboardData(EMAIL); diff --git a/backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockGeminiServiceTest.java b/backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionServiceTest.java similarity index 72% rename from backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockGeminiServiceTest.java rename to backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionServiceTest.java index 11848d3..ad18715 100644 --- a/backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockGeminiServiceTest.java +++ b/backend/src/test/java/com/thughari/jobtrackerpro/service/mock/MockAiExtractionServiceTest.java @@ -1,14 +1,15 @@ package com.thughari.jobtrackerpro.service.mock; import org.junit.jupiter.api.Test; +import com.thughari.jobtrackerpro.service.mock.MockAiExtractionService; import static org.junit.jupiter.api.Assertions.*; -class MockGeminiServiceTest { +class MockAiExtractionServiceTest { @Test void buildsMockJobFromEmailAndSubject() { - MockGeminiService service = new MockGeminiService(); + MockAiExtractionService service = new MockAiExtractionService(); var result = service.extractJobFromEmail("hr@acme.com", "Backend Engineer", "Body"); diff --git a/docker-compose.yml b/docker-compose.yml index 9600863..d081931 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,8 +8,8 @@ services: ports: - "5432:5432" - mailhog: - image: mailhog/mailhog + mailpit: + image: axllent/mailpit ports: - "1025:1025" - "8025:8025" \ No newline at end of file diff --git a/frontend/src/app/components/application-list/application-list.component.html b/frontend/src/app/components/application-list/application-list.component.html index d3a64fd..91e242e 100644 --- a/frontend/src/app/components/application-list/application-list.component.html +++ b/frontend/src/app/components/application-list/application-list.component.html @@ -20,16 +20,24 @@

- - - - - +
+ @if (syncStatus(); as status) { + + {{ status }} + + } + +
} @@ -61,34 +69,34 @@