Repository navigation
fix(jobs): serialize concurrent job updates and archives with per-ent… #58
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,7 @@ | |
| """ | ||
| from __future__ import annotations | ||
|
|
||
| import asyncio | ||
| import uuid | ||
| from typing import Any, Dict, Optional | ||
|
|
||
|
|
@@ -42,7 +43,7 @@ class JobService: | |
| """Use cases for job postings. | ||
|
|
||
| Constructed per request, like `InterviewService` - it holds only | ||
| references (a repository, an optional agent factory), never state. | ||
| references to collaborators and the container's shared per-job locks. | ||
| """ | ||
|
|
||
| def __init__( | ||
|
|
@@ -52,6 +53,8 @@ def __init__( | |
| jd_analyzer_factory=None, | ||
| rubric_repository: Optional[RubricRepository] = None, | ||
| rubric_generator_factory=None, | ||
| locks: dict[str, asyncio.Lock] | None = None, | ||
| locks_guard: asyncio.Lock | None = None, | ||
| ) -> None: | ||
| self._jobs = job_repository | ||
| # The same wiring-seam pattern as InterviewService's | ||
|
|
@@ -64,6 +67,12 @@ def __init__( | |
| # makes auto-rubric-on-create active in the actual app. | ||
| self._rubrics = rubric_repository | ||
| self._rubric_generator_factory = rubric_generator_factory | ||
| self._locks: dict[str, asyncio.Lock] = locks if locks is not None else {} | ||
| self._locks_guard = locks_guard if locks_guard is not None else asyncio.Lock() | ||
|
|
||
| async def _lock_for(self, job_id: str) -> asyncio.Lock: | ||
| async with self._locks_guard: | ||
| return self._locks.setdefault(job_id, asyncio.Lock()) | ||
|
|
||
| async def create_job( | ||
| self, | ||
|
|
@@ -287,24 +296,26 @@ async def update_job(self, job_id: str, patch: Dict[str, Any]) -> JobRecord: | |
| 1.0), so an edit that breaks that invariant is rejected with the | ||
| same 422 a bad `POST /jobs` payload would get, not silently stored. | ||
| """ | ||
| record = await self.get_job(job_id) | ||
| if not record.is_active: | ||
| raise ConflictError( | ||
| "Cannot update an archived job", | ||
| internal_detail=f"job_id={job_id!r} is archived", | ||
| ) | ||
| lock = await self._lock_for(job_id) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 High · Archived jobs become active When an admin archives a job after Make admin state changes use the same per-job synchronization as job updates. Also at Prompt for AI agentsIn `services/job_service.py` at line 233:
**Archived jobs become active**
When an admin archives a job after `update_job()` reads it but before it saves, `AdminService.set_job_active()` bypasses the per-job lock, so `update_job()` saves its stale active snapshot and reactivates the job.
Make admin state changes use the same per-job synchronization as job updates.
Also at `services/job_service.py:234`, `services/job_service.py:260-261`. |
||
| async with lock: | ||
| record = await self.get_job(job_id) | ||
| if not record.is_active: | ||
| raise ConflictError( | ||
| "Cannot update an archived job", | ||
| internal_detail=f"job_id={job_id!r} is archived", | ||
| ) | ||
|
|
||
| merged = record.job.model_dump() | ||
| merged.update(patch) | ||
| try: | ||
| updated_job = JobDescription(**merged) | ||
| except ValidationError as exc: | ||
| raise JobPatchValidationError(exc) from exc | ||
| merged = record.job.model_dump() | ||
| merged.update(patch) | ||
| try: | ||
| updated_job = JobDescription(**merged) | ||
| except ValidationError as exc: | ||
| raise JobPatchValidationError(exc) from exc | ||
|
|
||
| updated_record = record.model_copy(update={"job": updated_job}) | ||
| stored = await self._jobs.save(updated_record) | ||
| logger.info("job updated", extra=log_context(event="job_updated", job_id=job_id)) | ||
| return stored | ||
| updated_record = record.model_copy(update={"job": updated_job}) | ||
| stored = await self._jobs.save(updated_record) | ||
| logger.info("job updated", extra=log_context(event="job_updated", job_id=job_id)) | ||
| return stored | ||
|
|
||
| async def archive_job(self, job_id: str) -> JobRecord: | ||
| """Soft-delete: mark a job as no longer accepting applications. | ||
|
|
@@ -314,14 +325,16 @@ async def archive_job(self, job_id: str) -> JobRecord: | |
| both. Idempotent: archiving an already-archived job just returns its | ||
| current state rather than erroring. | ||
| """ | ||
| record = await self._jobs.archive(job_id) | ||
| if record is None: | ||
| raise NotFoundError( | ||
| "No job found for the given job_id", | ||
| internal_detail=f"job record missing for job_id={job_id!r}", | ||
| ) | ||
| logger.info("job archived", extra=log_context(event="job_archived", job_id=job_id)) | ||
| return record | ||
| lock = await self._lock_for(job_id) | ||
| async with lock: | ||
| record = await self._jobs.archive(job_id) | ||
| if record is None: | ||
| raise NotFoundError( | ||
| "No job found for the given job_id", | ||
| internal_detail=f"job record missing for job_id={job_id!r}", | ||
| ) | ||
| logger.info("job archived", extra=log_context(event="job_archived", job_id=job_id)) | ||
| return record | ||
|
|
||
|
|
||
| __all__ = ["JobService"] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| """Exercise shared request locks with controlled repository interleavings.""" | ||
| import asyncio | ||
| from types import SimpleNamespace | ||
|
|
||
| import pytest | ||
| from fastapi import Request | ||
|
|
||
| from core.dependencies import get_job_service | ||
| from core.errors import ConflictError | ||
| from repositories.interfaces import JobRecord | ||
| from repositories.memory import InMemoryJobRepository | ||
| from schemas.job import JobDescription | ||
| from services.job_service import JobPatchValidationError | ||
|
|
||
|
|
||
| class PausingJobRepository(InMemoryJobRepository): | ||
| def __init__(self): | ||
| super().__init__() | ||
| self.entered = asyncio.Queue() | ||
| self.release = asyncio.Event() | ||
|
|
||
| async def get(self, job_id): | ||
| record = await super().get(job_id) | ||
| await self.entered.put(("get", job_id)) | ||
| await self.release.wait() | ||
| return record | ||
|
|
||
| async def archive(self, job_id): | ||
| await self.entered.put(("archive", job_id)) | ||
| await self.release.wait() | ||
| return await super().archive(job_id) | ||
|
|
||
|
|
||
| async def request_services(repo, *job_ids): | ||
| for job_id in job_ids: | ||
| await repo.save(JobRecord( | ||
| job_id=job_id, | ||
| job=JobDescription(job_id=job_id, title="Original", description="Sample role"), | ||
| )) | ||
| container = SimpleNamespace( | ||
| job_repository=repo, | ||
| rubric_repository=None, | ||
| jd_analyzer_factory_for=lambda app: None, | ||
| job_locks={}, | ||
| job_locks_guard=asyncio.Lock(), | ||
| ) | ||
| app = SimpleNamespace(state=SimpleNamespace(container=container)) | ||
| # Resolve separately, just as two HTTP requests do. | ||
| return tuple(get_job_service(Request({"type": "http", "app": app})) for _ in range(2)) | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_concurrent_job_updates_preserve_both_fields(): | ||
| repo = PausingJobRepository() | ||
| first, second = await request_services(repo, "job_1") | ||
| async with asyncio.TaskGroup() as tasks: | ||
| tasks.create_task(first.update_job("job_1", {"department": "Platform"})) | ||
| assert await asyncio.wait_for(repo.entered.get(), 1) == ("get", "job_1") | ||
| tasks.create_task(second.update_job("job_1", {"title": "Updated"})) | ||
| try: | ||
| await asyncio.sleep(0) # Let the second request attempt its read. | ||
| assert repo.entered.empty(), "Second request read a stale snapshot" | ||
| finally: | ||
| repo.release.set() | ||
|
|
||
| final = await repo.get("job_1") | ||
| assert final.job.department == "Platform" | ||
| assert final.job.title == "Updated" | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_concurrent_updates_to_different_jobs_do_not_interfere(): | ||
| repo = PausingJobRepository() | ||
| first, second = await request_services(repo, "job_1", "job_2") | ||
| async with asyncio.TaskGroup() as tasks: | ||
| tasks.create_task(first.update_job("job_1", {"title": "Updated Role 1"})) | ||
| tasks.create_task(second.update_job("job_2", {"title": "Updated Role 2"})) | ||
| try: | ||
| entered = [await asyncio.wait_for(repo.entered.get(), 1) for _ in range(2)] | ||
| assert set(entered) == {("get", "job_1"), ("get", "job_2")} | ||
| finally: | ||
| repo.release.set() | ||
|
|
||
| assert (await repo.get("job_1")).job.title == "Updated Role 1" | ||
| assert (await repo.get("job_2")).job.title == "Updated Role 2" | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| @pytest.mark.parametrize("archive_first", [False, True]) | ||
| async def test_update_and_archive_share_the_request_lock(archive_first): | ||
| repo = PausingJobRepository() | ||
| first, second = await request_services(repo, "job_1") | ||
| async with asyncio.TaskGroup() as tasks: | ||
| tasks.create_task( | ||
| first.archive_job("job_1") if archive_first | ||
| else first.update_job("job_1", {"title": "Updated"}) | ||
| ) | ||
| await asyncio.wait_for(repo.entered.get(), 1) | ||
|
|
||
| async def following_request(): | ||
| if archive_first: | ||
| with pytest.raises(ConflictError): | ||
| await second.update_job("job_1", {"title": "Updated"}) | ||
| else: | ||
| await second.archive_job("job_1") | ||
|
|
||
| tasks.create_task(following_request()) | ||
| try: | ||
| await asyncio.sleep(0) | ||
| assert repo.entered.empty(), "Update and archive did not serialize" | ||
| finally: | ||
| repo.release.set() | ||
|
|
||
| final = await repo.get("job_1") | ||
| assert final.is_active is False | ||
| assert final.job.title == ("Original" if archive_first else "Updated") | ||
|
|
||
|
|
||
| @pytest.mark.asyncio | ||
| async def test_validation_failure_releases_lock_for_next_request(): | ||
| repo = PausingJobRepository() | ||
| first, second = await request_services(repo, "job_1") | ||
| repo.release.set() | ||
| with pytest.raises(JobPatchValidationError): | ||
| await first.update_job("job_1", {"openings": None}) | ||
|
|
||
| updated = await asyncio.wait_for(second.update_job("job_1", {"title": "Updated"}), 1) | ||
| assert updated.job.title == "Updated" | ||
| assert updated.job.openings == 1 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🟡 Medium · Missing IDs exhaust process memory
When requests use distinct nonexistent job IDs for update or archive,
_lock_for()inserts a lock before the repository checks whether the job exists and never removes it, so repeated requests leave entries in the lock map and grow memory without bound.Avoid retaining locks for missing jobs or safely reclaim unused lock entries.
Prompt for AI agents