From 766ef84a4dd556610b6f1599b0424850241aa3b8 Mon Sep 17 00:00:00 2001 From: Kanishk Sachdev Date: Wed, 18 Nov 2026 14:20:31 -0500 Subject: [PATCH] Filter the board by classified place and by company, with counted picks The board's location filter was a substring match on the raw text, though every location is classified into country, region and city. Column filters gain any_of and none_of over a JSON list, and a place field that matches locations.places by containment, so a country takes its regions and cities. The list's facets count places and companies under every other condition. --- docs/agents/frontend.md | 13 ++ openapi.json | 41 +++++- src/api/board/column_filters.py | 201 ++++++++++++++++++++--------- src/api/routers/job_board.py | 83 +++++++++++- tests/test_board_column_filters.py | 112 ++++++++++++++++ 5 files changed, 382 insertions(+), 68 deletions(-) diff --git a/docs/agents/frontend.md b/docs/agents/frontend.md index 39c8a24f..e73ff855 100644 --- a/docs/agents/frontend.md +++ b/docs/agents/frontend.md @@ -101,6 +101,19 @@ values; negative text conditions do not include empty values. Private fields such as notes are scoped to the current user's joined row. The field mapping and parameterized compiler live in `api/board/column_filters.py`. +A field whose values are a known set is picked, not typed. `any_of` and +`none_of` take a JSON array in `value`: OR within the condition, AND across +conditions, and `none_of` is the exclusion. Text fields compare them +case-insensitively and whole. The `place` field matches the classified +`locations` rows, never the raw text: a pick is `remote` or +`COUNTRY[/REGION[/CITY]]` (REGION may be empty), and a country takes every +region and city in it, by jsonb containment on `locations.places`. A location +not yet classified matches no pick. Offer picks from the list's +`facets.place` and `facets.company` (on `with_facets=true`), which count the +board under every other condition except the field's own, so a selected pick +never hides its siblings. Labels are the client's: country codes through +`Intl.DisplayNames`. + **A refusal is `{detail: {code, message, ...}}`, built by `api.problem.refuse`.** `code` is what a client branches on and stays stable; `message` is for a person and may be reworded. Extra keys (a `task_id`, the `in_flight` run) ride diff --git a/openapi.json b/openapi.json index eded6b56..f825f18e 100644 --- a/openapi.json +++ b/openapi.json @@ -37346,7 +37346,7 @@ "anyOf": [ { "type": "string", - "maxLength": 1000 + "maxLength": 4000 }, { "type": "null" @@ -39178,6 +39178,24 @@ ], "title": "ExtensionConfig" }, + "FacetValue": { + "properties": { + "value": { + "type": "string", + "title": "Value" + }, + "count": { + "type": "integer", + "title": "Count" + } + }, + "type": "object", + "required": [ + "value", + "count" + ], + "title": "FacetValue" + }, "Facets": { "properties": { "ats": { @@ -39186,6 +39204,22 @@ }, "type": "array", "title": "Ats" + }, + "place": { + "items": { + "$ref": "#/components/schemas/FacetValue" + }, + "type": "array", + "title": "Place", + "default": [] + }, + "company": { + "items": { + "$ref": "#/components/schemas/FacetValue" + }, + "type": "array", + "title": "Company", + "default": [] } }, "type": "object", @@ -39193,7 +39227,7 @@ "ats" ], "title": "Facets", - "description": "Counts under every OTHER filter the page has on. Taken before the ats\nclause joins the rest, because a board-wide \"rippling 4\" beside a lens\nthat holds none of them reads as a lie." + "description": "Counts under every OTHER filter the page has on. Taken before the ats\nclause joins the rest, because a board-wide \"rippling 4\" beside a lens\nthat holds none of them reads as a lie. `place` and `company` likewise\nleave out their own column conditions, so a pick never hides its siblings.\n\nA `place` value is a pick the place column filter takes: \"remote\", or\n\"COUNTRY\", \"COUNTRY/REGION\", \"COUNTRY/REGION/CITY\" (REGION may be empty)." }, "FailedCheck": { "properties": { @@ -39847,7 +39881,8 @@ "text", "date", "number", - "boolean" + "boolean", + "place" ], "title": "Kind" }, diff --git a/src/api/board/column_filters.py b/src/api/board/column_filters.py index 62d7ed38..a09b6f68 100644 --- a/src/api/board/column_filters.py +++ b/src/api/board/column_filters.py @@ -17,10 +17,11 @@ class ColumnFilter(BaseModel): model_config = ConfigDict(extra="forbid") field: str operator: str - value: str | None = Field(default=None, max_length=1000) + # A list operator (any_of, none_of) carries a JSON array here. + value: str | None = Field(default=None, max_length=4000) -FilterKind = Literal["text", "date", "number", "boolean"] +FilterKind = Literal["text", "date", "number", "boolean", "place"] class FilterField(BaseModel): @@ -30,11 +31,22 @@ class FilterField(BaseModel): operators: list[str] +_LIST_OPERATORS = ("any_of", "none_of") _OPERATORS = { - "text": ["contains", "not_contains", "equals", "not_equals", "is_empty", "is_not_empty"], + "text": [ + "contains", + "not_contains", + "equals", + "not_equals", + "any_of", + "none_of", + "is_empty", + "is_not_empty", + ], "date": ["equals", "gte", "lte", "is_empty", "is_not_empty"], "number": ["equals", "gte", "lte", "is_empty", "is_not_empty"], "boolean": ["equals"], + "place": list(_LIST_OPERATORS), } _FIELDS: dict[str, tuple[str, FilterKind, str]] = { # Trimmed so `equals` reads lower(btrim(company)), which idx_jobs_company_key @@ -42,6 +54,7 @@ class FilterField(BaseModel): # primary key for 5 s; trimmed, 0.4 s (2026-10-15). "company": ("Company", "text", "btrim(j.company)"), "title": ("Role", "text", "j.title"), + "place": ("Location", "place", ""), "locations": ("Location text", "text", "array_to_string(j.locations, ', ')"), "terms": ("Employment terms", "text", "array_to_string(j.terms, ', ')"), "source": ("Source", "text", "j.source"), @@ -98,66 +111,128 @@ def compile_filters( if not isinstance(values, list) or len(values) > 32: raise ValueError("Supply at most 32 column filters") rules = [ColumnFilter.model_validate(value) for value in values] - offered = _offered(catalog_only) - clauses, params = [], {} - for index, rule in enumerate(rules): - if rule.field not in offered: - raise ValueError(f"Unknown filter field: {rule.field}") - _, kind, expression = offered[rule.field] - if rule.operator not in _OPERATORS[kind]: - raise ValueError(f"Unsupported operator for {rule.field}") - if rule.operator in ("is_empty", "is_not_empty"): - if rule.value is not None: - raise ValueError("Empty-value operators do not take a value") - empty = ( - f"NULLIF(btrim({expression}), '') IS NULL" - if kind == "text" - else f"{expression} IS NULL" - ) - clauses.append( - f"AND ({empty})" if rule.operator == "is_empty" else f"AND NOT ({empty})" - ) - continue - if rule.value is None or not rule.value.strip(): - raise ValueError(f"A value is required for {rule.field}") - name = f"column_filter_{index}" - value: object = rule.value - if rule.operator in ("not_equals", "not_contains"): - clauses.append(f"AND NULLIF(btrim({expression}), '') IS NOT NULL") - if kind == "date": - value = datetime.date.fromisoformat(rule.value) - elif kind == "number": - value = Decimal(rule.value) - if not value.is_finite(): - raise ValueError("Pay must be a finite number") - if abs(value) > Decimal("1e18"): - raise ValueError("Pay value exceeds the supported numeric range") - units = {r.field: r.value for r in rules if r.operator == "equals"} - if ( - not units.get("comp_currency") - or canonical_compensation_period(units.get("comp_period")) is None - ): - raise ValueError("Pay comparisons require an exact currency and pay period") - elif rule.field == "comp_period" and rule.operator in ("equals", "not_equals"): - value = canonical_compensation_period(rule.value) or rule.value - elif kind == "boolean": - if rule.value not in ("true", "false"): - raise ValueError("Boolean filters accept true or false") - value = rule.value == "true" - if rule.operator in ("contains", "not_contains"): - value = ( - "%" - + rule.value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - + "%" - ) - op = "ILIKE" if rule.operator == "contains" else "NOT ILIKE" - else: - op = {"equals": "=", "not_equals": "<>", "gte": ">=", "lte": "<="}[rule.operator] - if kind == "text" and rule.operator in ("equals", "not_equals"): - clauses.append(f"AND lower({expression}) {op} lower(%({name})s)") - else: - clauses.append(f"AND {expression} {op} %({name})s") - params[name] = value + clauses, params = compile_rules(rules, catalog_only) return rules, clauses, params except (ValueError, ArithmeticError) as error: raise refuse(422, "INVALID_COLUMN_FILTER", str(error)) from error + + +# A place pick is "remote" or "COUNTRY[/REGION[/CITY]]", the levels of a +# `locations.places` entry. The facet (api.routers.job_board) serves the same +# spelling, so a pick is always one the board can count. +REMOTE = "remote" + + +def place_json(token: str) -> str: + """The `places` entry a pick contains, as JSON for jsonb containment: a + country pick matches every region and city in it.""" + country, region, city = ([*token.split("/", 2), "", ""])[:3] + if len(country) != 2 or not country.isalpha(): + raise ValueError(f"Unknown place: {token}") + entry = {"country": country.upper()} + if region: + entry["region"] = region.upper() + if city: + entry["city"] = city + return json.dumps([entry]) + + +def _list_value(rule: ColumnFilter) -> list[str]: + try: + values = json.loads(rule.value or "") + except ValueError: + values = None + if ( + not isinstance(values, list) + or not values + or not all(isinstance(v, str) and v.strip() for v in values) + ): + raise ValueError(f"{rule.field} {rule.operator} takes a JSON list of values") + return [v.strip() for v in values] + + +def _place_clause(rule: ColumnFilter, name: str, params: dict) -> str: + picks = _list_value(rule) + params[f"{name}_remote"] = any(p.lower() == REMOTE for p in picks) + params[f"{name}_places"] = [place_json(p) for p in picks if p.lower() != REMOTE] + # Same join as the board criteria (api.board.criteria): a string not yet + # classified has no row and matches nothing. + match = f"""EXISTS ( + SELECT 1 FROM unnest(j.locations) loc + JOIN locations l ON l.text = btrim(loc) + WHERE (%({name}_remote)s AND l.remote) + OR l.places @> ANY(%({name}_places)s::text[]::jsonb[]))""" + return f"AND {match}" if rule.operator == "any_of" else f"AND NOT {match}" + + +def compile_rules(rules: list[ColumnFilter], catalog_only: bool) -> tuple[list[str], dict]: + """AND of the rules as SQL clauses over `j`, `uj` and `pay`. Raises + ValueError for a rule the endpoint does not offer.""" + offered = _offered(catalog_only) + clauses: list[str] = [] + params: dict = {} + for index, rule in enumerate(rules): + if rule.field not in offered: + raise ValueError(f"Unknown filter field: {rule.field}") + _, kind, expression = offered[rule.field] + if rule.operator not in _OPERATORS[kind]: + raise ValueError(f"Unsupported operator for {rule.field}") + name = f"column_filter_{index}" + if kind == "place": + clauses.append(_place_clause(rule, name, params)) + continue + if rule.operator in _LIST_OPERATORS: + params[name] = [v.lower() for v in _list_value(rule)] + match = f"lower({expression}) = ANY(%({name})s)" + clauses.append(f"AND {match}" if rule.operator == "any_of" else f"AND NOT {match}") + continue + if rule.operator in ("is_empty", "is_not_empty"): + if rule.value is not None: + raise ValueError("Empty-value operators do not take a value") + empty = ( + f"NULLIF(btrim({expression}), '') IS NULL" + if kind == "text" + else f"{expression} IS NULL" + ) + clauses.append( + f"AND ({empty})" if rule.operator == "is_empty" else f"AND NOT ({empty})" + ) + continue + if rule.value is None or not rule.value.strip(): + raise ValueError(f"A value is required for {rule.field}") + value: object = rule.value + if rule.operator in ("not_equals", "not_contains"): + clauses.append(f"AND NULLIF(btrim({expression}), '') IS NOT NULL") + if kind == "date": + value = datetime.date.fromisoformat(rule.value) + elif kind == "number": + value = Decimal(rule.value) + if not value.is_finite(): + raise ValueError("Pay must be a finite number") + if abs(value) > Decimal("1e18"): + raise ValueError("Pay value exceeds the supported numeric range") + units = {r.field: r.value for r in rules if r.operator == "equals"} + if ( + not units.get("comp_currency") + or canonical_compensation_period(units.get("comp_period")) is None + ): + raise ValueError("Pay comparisons require an exact currency and pay period") + elif rule.field == "comp_period" and rule.operator in ("equals", "not_equals"): + value = canonical_compensation_period(rule.value) or rule.value + elif kind == "boolean": + if rule.value not in ("true", "false"): + raise ValueError("Boolean filters accept true or false") + value = rule.value == "true" + if rule.operator in ("contains", "not_contains"): + value = ( + "%" + rule.value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + ) + op = "ILIKE" if rule.operator == "contains" else "NOT ILIKE" + else: + op = {"equals": "=", "not_equals": "<>", "gte": ">=", "lte": "<="}[rule.operator] + if kind == "text" and rule.operator in ("equals", "not_equals"): + clauses.append(f"AND lower({expression}) {op} lower(%({name})s)") + else: + clauses.append(f"AND {expression} {op} %({name})s") + params[name] = value + return clauses, params diff --git a/src/api/routers/job_board.py b/src/api/routers/job_board.py index ffa11389..51312bcf 100644 --- a/src/api/routers/job_board.py +++ b/src/api/routers/job_board.py @@ -187,12 +187,23 @@ class Sort(BaseModel): dir: str +class FacetValue(BaseModel): + value: str + count: int + + class Facets(BaseModel): """Counts under every OTHER filter the page has on. Taken before the ats clause joins the rest, because a board-wide "rippling 4" beside a lens - that holds none of them reads as a lie.""" + that holds none of them reads as a lie. `place` and `company` likewise + leave out their own column conditions, so a pick never hides its siblings. + + A `place` value is a pick the place column filter takes: "remote", or + "COUNTRY", "COUNTRY/REGION", "COUNTRY/REGION/CITY" (REGION may be empty).""" ats: list[AtsCount] + place: list[FacetValue] = [] + company: list[FacetValue] = [] class Board(BaseModel): @@ -237,6 +248,63 @@ def _ats_counts(filter_sql: str, params: dict) -> list[AtsCount]: ) +def _place_counts(filter_sql: str, params: dict) -> list[FacetValue]: + """Rows per place on the board under `filter_sql`, at every level a + classified location names, and the remote rows, most first.""" + board = visibility.FAST.format(columns="j.id, j.locations", extra=filter_sql) + rows = db.query( + f""" + WITH b AS ({board}), + p AS ( + SELECT b.id, l.remote, pl->>'country' AS c, pl->>'region' AS r, pl->>'city' AS ci + FROM b, unnest(b.locations) loc + JOIN locations l ON l.text = btrim(loc) + LEFT JOIN LATERAL jsonb_array_elements(l.places) pl ON TRUE + ) + SELECT c, r, ci, GROUPING(r, ci) AS level, COUNT(DISTINCT id) AS n + FROM p WHERE c IS NOT NULL GROUP BY GROUPING SETS ((c), (c, r), (c, r, ci)) + UNION ALL + SELECT NULL, NULL, NULL, -1, COUNT(DISTINCT id) FROM p WHERE remote + """, + params, + ) + out = [] + for row in rows: + level, n = row["level"], row["n"] + if level == -1: + value = column_filters_.REMOTE if n else None + elif level == 3: + value = row["c"] + elif level == 1: + value = f"{row['c']}/{row['r']}" if row["r"] else None + else: + value = f"{row['c']}/{row['r'] or ''}/{row['ci']}" if row["ci"] else None + if value: + out.append(FacetValue(value=value, count=n)) + return sorted(out, key=lambda f: (-f.count, f.value)) + + +def _company_counts(filter_sql: str, params: dict) -> list[FacetValue]: + return db.query_as( + FacetValue, + visibility.FAST.format( + columns="min(btrim(j.company)) AS value, COUNT(*) AS count", + extra=f"{filter_sql}\nAND btrim(j.company) <> ''\n" + "GROUP BY lower(btrim(j.company)) ORDER BY 2 DESC, 1", + ), + params, + ) + + +def _facets( + rules: list[column_filters_.ColumnFilter], other: list[str], params: dict +) -> tuple[str, dict]: + """`other` and every column condition except those on the facet's field, + so the counts beside a pick are what adding a sibling would add.""" + clauses, column_params = column_filters_.compile_rules(rules, catalog_only=False) + return "\n".join(clauses + other), {**params, **column_params} + + @router.get("/user/jobs/options") def job_options(user: AuthedUser = Depends(require_user)) -> BoardOptions: """Everything the board's filter/edit controls need, generated from data @@ -323,7 +391,18 @@ def list_jobs( extra.append("AND j.source = ANY(%(sources)s)") params["sources"] = wanted_sources # Counted before the ats clause joins `extra`; see Facets (2026-09-07). - facets = Facets(ats=_ats_counts("\n".join(extra), params)) if with_facets else None + facets = None + if with_facets: + other = extra[len(column_clauses) :] + + def without(field: str) -> tuple[str, dict]: + return _facets([r for r in column_rules if r.field != field], other, params) + + facets = Facets( + ats=_ats_counts("\n".join(extra), params), + place=_place_counts(*without("place")), + company=_company_counts(*without("company")), + ) if wanted_ats: extra.append(f"AND ({ATS_SQL}) = ANY(%(ats)s)") params["ats"] = wanted_ats diff --git a/tests/test_board_column_filters.py b/tests/test_board_column_filters.py index 80b6b820..9d674df2 100644 --- a/tests/test_board_column_filters.py +++ b/tests/test_board_column_filters.py @@ -188,3 +188,115 @@ def test_private_column_predicates_cannot_read_another_users_notes(client, user_ ) assert response.status_code == 200 assert response.json()["total"] == count + + +def _board_job(f, company: str, locations: list[str]) -> int: + jid = f.make_job(company=company) + db.execute("UPDATE jobs SET locations=%s WHERE id=%s", (locations, jid)) + db.execute( + "INSERT INTO user_jobs(user_id,job_id,status) VALUES (%s,%s,'saved')", + (user_id_for_subject("test-user"), jid), + ) + return jid + + +def _place(text: str, places: list[dict], remote: bool = False) -> None: + db.execute( + "INSERT INTO locations (text, country, remote, places) VALUES (%s,%s,%s,%s)", + (text, places[0]["country"] if places else None, remote, json.dumps(places)), + ) + + +@pytest.fixture +def placed_board(f): + _place("San Francisco, CA", [{"country": "US", "region": "CA", "city": "San Francisco"}]) + _place("NYC", [{"country": "US", "region": "NY", "city": "New York"}]) + _place("London", [{"country": "GB", "region": None, "city": "London"}]) + _place("Remote (Canada)", [{"country": "CA", "region": None, "city": None}], remote=True) + return { + "sf": _board_job(f, "Acme", ["San Francisco, CA"]), + "ny": _board_job(f, "acme ", ["NYC", "London"]), + "london": _board_job(f, "Globex", ["London"]), + "remote": _board_job(f, "Initech", ["Remote (Canada)"]), + "unclassified": _board_job(f, "Hooli", ["Somewhere new"]), + } + + +def _ids(client, user_headers, rules) -> set[int]: + response = client.get( + "/v1/user/jobs", headers=user_headers, params={"column_filters": json.dumps(rules)} + ) + assert response.status_code == 200, response.text + return {row["job_id"] for row in response.json()["rows"]} + + +@pytest.mark.parametrize( + ("operator", "picks", "expected"), + [ + # A country takes every region and city in it; picks OR together. + ("any_of", ["US"], {"sf", "ny"}), + ("any_of", ["US/CA", "remote"], {"sf", "remote"}), + ("any_of", ["GB//London"], {"ny", "london"}), + ("any_of", ["us/ny/New York"], {"ny"}), + # Excluded: any matching location hides the posting; a location not + # yet classified has nothing to match. + ("none_of", ["GB"], {"sf", "remote", "unclassified"}), + ], +) +def test_place_filter_matches_classified_places( + client, user_headers, placed_board, operator, picks, expected +): + rules = [{"field": "place", "operator": operator, "value": json.dumps(picks)}] + assert _ids(client, user_headers, rules) == {placed_board[k] for k in expected} + + +def test_place_and_company_conditions_combine_with_and(client, user_headers, placed_board): + rules = [ + {"field": "place", "operator": "any_of", "value": json.dumps(["US", "GB"])}, + {"field": "company", "operator": "any_of", "value": json.dumps(["ACME"])}, + ] + assert _ids(client, user_headers, rules) == {placed_board["sf"], placed_board["ny"]} + rules[1]["operator"] = "none_of" + assert _ids(client, user_headers, rules) == {placed_board["london"]} + + +def test_facets_count_places_and_companies_without_their_own_condition( + client, user_headers, placed_board +): + rules = [{"field": "place", "operator": "any_of", "value": json.dumps(["US/CA"])}] + response = client.get( + "/v1/user/jobs", + headers=user_headers, + params={"column_filters": json.dumps(rules), "with_facets": "true"}, + ) + assert response.status_code == 200 + facets = response.json()["facets"] + place = {item["value"]: item["count"] for item in facets["place"]} + # The place pick does not narrow its own facet. + assert place["US"] == 2 + assert place["US/CA"] == 1 + assert place["US/NY/New York"] == 1 + assert place["GB"] == 2 + assert place["GB//London"] == 2 + assert place["remote"] == 1 + assert "GB/" not in place + # It does narrow the company facet: only the SF row is left. + assert {item["value"]: item["count"] for item in facets["company"]} == {"Acme": 1} + + +@pytest.mark.parametrize( + "rule", + [ + {"field": "place", "operator": "any_of", "value": "US"}, + {"field": "place", "operator": "any_of", "value": json.dumps([])}, + {"field": "place", "operator": "any_of", "value": json.dumps(["Atlantis"])}, + {"field": "place", "operator": "contains", "value": "US"}, + {"field": "company", "operator": "any_of", "value": json.dumps([""])}, + ], +) +def test_malformed_list_conditions_are_refused(client, user_headers, rule): + response = client.get( + "/v1/user/jobs", headers=user_headers, params={"column_filters": json.dumps([rule])} + ) + assert response.status_code == 422 + assert response.json()["detail"]["code"] == "INVALID_COLUMN_FILTER"