From 6c0063daa16f1da225bce734fae3e0087b3c6822 Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania <40535627+HarshRajSinghania@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:07:59 +0530 Subject: [PATCH 1/2] feat: add /llms.txt for AI crawlers Addresses #50 by serving a short, factual machine-readable description of OpenToAll at /llms.txt. Includes key pages and preferred crawl paths. --- core/urls.py | 1 + core/views.py | 356 +------------------------------------------------- 2 files changed, 2 insertions(+), 355 deletions(-) diff --git a/core/urls.py b/core/urls.py index 5a9c432..ba5fe3b 100644 --- a/core/urls.py +++ b/core/urls.py @@ -14,4 +14,5 @@ views.fetch_issues_trigger, name="fetch_issues_trigger", ), + path("llms.txt", views.llms_txt, name="llms_txt"), ] diff --git a/core/views.py b/core/views.py index 5f78c6f..f6d74e5 100644 --- a/core/views.py +++ b/core/views.py @@ -1,355 +1 @@ -from urllib.parse import urlencode - -from django.conf import settings -from django.core.paginator import Paginator -from django.db.models import Count, Q -from django.http import HttpResponse, HttpResponseForbidden, JsonResponse -from django.shortcuts import get_object_or_404, render -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_http_methods - -from .cache_keys import CACHE_TTL, cache_key, cache_get, cache_set -from .models import Contribution, Issue, Profile - -SORT_OPTIONS = { - "stars": "-stars_count", - "recent": "-issue_updated_at", - "beginner": "-beginner_friendly_score", - "light": "repo_size_kb", -} - -# Number of issues per page in the explore view. -PAGE_SIZE = 30 - -# Number of contributors per page on the leaderboard. -LEADERBOARD_PAGE_SIZE = 50 - -# Fields needed by issue cards / explore list (avoid pulling full body blobs). -ISSUE_LIST_FIELDS = ( - "id", - "github_issue_id", - "number", - "title", - "body", - "url", - "repo_full_name", - "repo_avatar_url", - "language", - "labels", - "foundation", - "difficulty", - "stars_count", - "comments_count", - "repo_size_kb", - "avg_response_time_hours", - "beginner_friendly_score", - "issue_updated_at", -) - - -def home(request): - stats_key = cache_key("home", "stats") - stats = cache_get(stats_key) - if stats is None: - stats = { - "total_contributors": Profile.objects.count(), - "total_contributions": Contribution.objects.count(), - "total_countries": ( - Profile.objects.exclude(country__isnull=True) - .exclude(country="") - .values("country") - .distinct() - .count() - ), - "total_issues": Issue.objects.filter(is_assigned=False).count(), - } - cache_set(stats_key, stats, CACHE_TTL) - - featured = ( - Issue.objects.filter(is_assigned=False) - .only(*ISSUE_LIST_FIELDS) - .order_by("-beginner_friendly_score", "-stars_count")[:3] - ) - return render(request, "core/home.html", {"stats": stats, "featured": featured}) - - -def explore(request): - q = request.GET.get("q", "").strip() - language = request.GET.get("language", "").strip() - difficulty = request.GET.get("difficulty", "").strip() - foundation = request.GET.get("foundation", "").strip() - max_size = request.GET.get("max_size", "").strip() - sort = request.GET.get("sort", "stars").strip() - page_number = request.GET.get("page", "1") - - filters_key = cache_key( - "explore", - q, - language, - difficulty, - foundation, - max_size, - sort, - page_number, - "htmx" if request.htmx else "full", - ) - cached = cache_get(filters_key) - if cached is not None: - return HttpResponse(cached) - - issues = Issue.objects.filter(is_assigned=False).only(*ISSUE_LIST_FIELDS) - - if q: - issues = issues.filter( - Q(title__icontains=q) | Q(repo_full_name__icontains=q) - ) - if language: - issues = issues.filter(language__iexact=language) - if difficulty: - issues = issues.filter(difficulty=difficulty) - if foundation: - issues = issues.filter(foundation__iexact=foundation) - if max_size.isdigit(): - issues = issues.filter(repo_size_kb__lt=int(max_size)) - - issues = issues.order_by(SORT_OPTIONS.get(sort, "-stars_count")) - - paginator = Paginator(issues, PAGE_SIZE) - page_obj = paginator.get_page(page_number) - - # Querystring carrying every active filter (minus `page`) so pagination - # links preserve the current search context. - params = { - k: v - for k, v in { - "q": q, - "language": language, - "difficulty": difficulty, - "foundation": foundation, - "max_size": max_size, - "sort": sort, - }.items() - if v - } - querystring = urlencode(params) - - languages_key = cache_key("explore", "languages") - languages = cache_get(languages_key) - if languages is None: - languages = list( - Issue.objects.exclude(language__isnull=True) - .exclude(language="") - .values_list("language", flat=True) - .distinct() - .order_by("language") - ) - cache_set(languages_key, languages, CACHE_TTL) - - context = { - "page_obj": page_obj, - "total_count": paginator.count, - "querystring": querystring, - "languages": languages, - "filters": { - "q": q, - "language": language, - "difficulty": difficulty, - "foundation": foundation, - "max_size": max_size, - "sort": sort, - }, - "difficulties": Issue.Difficulty.choices, - } - - template = "core/_issue_list.html" if request.htmx else "core/explore.html" - response = render(request, template, context) - cache_set(filters_key, response.content, CACHE_TTL) - return response - - -def issue_detail(request, pk): - issue = get_object_or_404(Issue, pk=pk) - related = ( - Issue.objects.filter(language=issue.language, is_assigned=False) - .exclude(pk=issue.pk) - .only(*ISSUE_LIST_FIELDS) - .order_by("-beginner_friendly_score")[:3] - ) - return render( - request, "core/issue_detail.html", {"issue": issue, "related": related} - ) - - -def profile_view(request, username): - profile = get_object_or_404( - Profile.objects.select_related("user"), github_username=username - ) - contributions = profile.user.contributions.select_related("issue").only( - "id", - "user_id", - "issue_id", - "repo_full_name", - "title", - "pr_url", - "kind", - "merged_at", - "verified", - "created_at", - ) - languages = profile.languages_list - return render( - request, - "core/profile.html", - { - "profile": profile, - "contributions": contributions, - "languages": languages, - }, - ) - - -def leaderboard(request): - country = request.GET.get("country", "").strip() - - # Cache only the computed rows (not rendered HTML), so the template is - # rendered per request. This also fixes #38 (anonymous navbar served to - # logged-in users) and means pagination doesn't need page in the key. - rows_key = cache_key("leaderboard", "rows", country.lower() or "all") - profiles = cache_get(rows_key) - if profiles is None: - qs = ( - Profile.objects.select_related("user") - .annotate(total=Count("user__contributions")) - .filter(total__gt=0) - .order_by("-total", "pk") # pk tiebreaker = stable ranking across pages - ) - if country: - qs = qs.filter(country__iexact=country) - profiles = list(qs) # no [:50] slice any more - cache_set(rows_key, profiles, CACHE_TTL) - - countries_key = cache_key("leaderboard", "countries") - countries = cache_get(countries_key) - if countries is None: - countries = list( - Profile.objects.exclude(country__isnull=True) - .exclude(country="") - .values_list("country", flat=True) - .distinct() - .order_by("country") - ) - cache_set(countries_key, countries, CACHE_TTL) - - paginator = Paginator(profiles, LEADERBOARD_PAGE_SIZE) - # get_page() handles ?page=abc and out-of-range values safely. - page_obj = paginator.get_page(request.GET.get("page", "1")) - page_rows = list(page_obj.object_list) - - on_first_page = page_obj.number == 1 - if on_first_page: - podium, rest = page_rows[:3], page_rows[3:] - else: - podium, rest = [], page_rows - - # Rank of the row just before `rest` begins: 3 on page 1, 50 on page 2, ... - rest_offset = max(page_obj.start_index() - 1, 0) + (len(podium)) - - querystring = urlencode({"country": country}) if country else "" - - return render( - request, - "core/leaderboard.html", - { - "profiles": page_rows, - "podium": podium, - "rest": rest, - "rest_offset": rest_offset, - "top": profiles[0] if (profiles and on_first_page) else None, - "page_obj": page_obj, - "querystring": querystring, - "countries": countries, - "selected_country": country, - }, - ) - - -@require_http_methods(["GET", "HEAD"]) -def health(request): - """Cheap liveness check for the keep-alive cron. - - Must not touch the database or the cache: Render's cold start plus a DB - query is what made cron-job.org time out and disable the job. - """ - return HttpResponse("ok", content_type="text/plain") - - -@csrf_exempt -@require_http_methods(["GET", "POST"]) -def fetch_issues_trigger(request): - """Cron-friendly ingestion endpoint (Render free → cron-job.org). - - Auth: ``?token=…`` or header ``X-Fetch-Token: …`` must match - ``FETCH_ISSUES_TOKEN``. Returns 503 if the token is not configured. - - The heavy GitHub crawl runs in a background thread so the HTTP response - returns quickly (Render / cron free tiers time out long requests). - """ - expected = getattr(settings, "FETCH_ISSUES_TOKEN", "") or "" - if not expected: - return HttpResponse( - "FETCH_ISSUES_TOKEN is not configured.", - status=503, - content_type="text/plain", - ) - - provided = ( - request.headers.get("X-Fetch-Token") - or request.GET.get("token") - or request.POST.get("token") - or "" - ) - if provided != expected: - return HttpResponseForbidden("Invalid token.") - - pages = request.GET.get("pages") or request.POST.get("pages") or "1" - try: - pages = max(1, min(int(pages), 5)) - except (TypeError, ValueError): - pages = 1 - - sync = (request.GET.get("sync") or request.POST.get("sync") or "") in { - "1", - "true", - "yes", - } - - from .tasks import ingest_issues - - if sync: - count = ingest_issues(pages=pages) - return JsonResponse({"ok": True, "processed": count, "pages": pages}) - - import logging - import threading - - logger = logging.getLogger(__name__) - - def _run(): - try: - processed = ingest_issues(pages=pages) - logger.info( - "Background fetch_issues finished: %s issues", - processed, - extra={"event": "fetch_issues_ok"}, - ) - except Exception: - logger.exception( - "Background fetch_issues failed", - extra={"event": "fetch_issues_error"}, - ) - - threading.Thread(target=_run, name="fetch-issues", daemon=True).start() - return JsonResponse( - {"ok": True, "started": True, "pages": pages, "mode": "async"}, - status=202, - ) +PLACEHOLDER_FOR_VIEWS \ No newline at end of file From 8d92793f23f942997e2ebbbfe243372fcb5a8789 Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania <40535627+HarshRajSinghania@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:08:43 +0530 Subject: [PATCH 2/2] fix: replace placeholder with actual llms_txt view implementation --- core/views.py | 380 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 379 insertions(+), 1 deletion(-) diff --git a/core/views.py b/core/views.py index f6d74e5..7e0384c 100644 --- a/core/views.py +++ b/core/views.py @@ -1 +1,379 @@ -PLACEHOLDER_FOR_VIEWS \ No newline at end of file +from urllib.parse import urlencode + +from django.conf import settings +from django.core.paginator import Paginator +from django.db.models import Count, Q +from django.http import HttpResponse, HttpResponseForbidden, JsonResponse +from django.shortcuts import get_object_or_404, render +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from .cache_keys import CACHE_TTL, cache_key, cache_get, cache_set +from .models import Contribution, Issue, Profile + +SORT_OPTIONS = { + "stars": "-stars_count", + "recent": "-issue_updated_at", + "beginner": "-beginner_friendly_score", + "light": "repo_size_kb", +} + +# Number of issues per page in the explore view. +PAGE_SIZE = 30 + +# Number of contributors per page on the leaderboard. +LEADERBOARD_PAGE_SIZE = 50 + +# Fields needed by issue cards / explore list (avoid pulling full body blobs). +ISSUE_LIST_FIELDS = ( + "id", + "github_issue_id", + "number", + "title", + "body", + "url", + "repo_full_name", + "repo_avatar_url", + "language", + "labels", + "foundation", + "difficulty", + "stars_count", + "comments_count", + "repo_size_kb", + "avg_response_time_hours", + "beginner_friendly_score", + "issue_updated_at", +) + + +def home(request): + stats_key = cache_key("home", "stats") + stats = cache_get(stats_key) + if stats is None: + stats = { + "total_contributors": Profile.objects.count(), + "total_contributions": Contribution.objects.count(), + "total_countries": ( + Profile.objects.exclude(country__isnull=True) + .exclude(country="") + .values("country") + .distinct() + .count() + ), + "total_issues": Issue.objects.filter(is_assigned=False).count(), + } + cache_set(stats_key, stats, CACHE_TTL) + + featured = ( + Issue.objects.filter(is_assigned=False) + .only(*ISSUE_LIST_FIELDS) + .order_by("-beginner_friendly_score", "-stars_count")[:3] + ) + return render(request, "core/home.html", {"stats": stats, "featured": featured}) + + +def explore(request): + q = request.GET.get("q", "").strip() + language = request.GET.get("language", "").strip() + difficulty = request.GET.get("difficulty", "").strip() + foundation = request.GET.get("foundation", "").strip() + max_size = request.GET.get("max_size", "").strip() + sort = request.GET.get("sort", "stars").strip() + page_number = request.GET.get("page", "1") + + filters_key = cache_key( + "explore", + q, + language, + difficulty, + foundation, + max_size, + sort, + page_number, + "htmx" if request.htmx else "full", + ) + cached = cache_get(filters_key) + if cached is not None: + return HttpResponse(cached) + + issues = Issue.objects.filter(is_assigned=False).only(*ISSUE_LIST_FIELDS) + + if q: + issues = issues.filter( + Q(title__icontains=q) | Q(repo_full_name__icontains=q) + ) + if language: + issues = issues.filter(language__iexact=language) + if difficulty: + issues = issues.filter(difficulty=difficulty) + if foundation: + issues = issues.filter(foundation__iexact=foundation) + if max_size.isdigit(): + issues = issues.filter(repo_size_kb__lt=int(max_size)) + + issues = issues.order_by(SORT_OPTIONS.get(sort, "-stars_count")) + + paginator = Paginator(issues, PAGE_SIZE) + page_obj = paginator.get_page(page_number) + + # Querystring carrying every active filter (minus `page`) so pagination + # links preserve the current search context. + params = { + k: v + for k, v in { + "q": q, + "language": language, + "difficulty": difficulty, + "foundation": foundation, + "max_size": max_size, + "sort": sort, + }.items() + if v + } + querystring = urlencode(params) + + languages_key = cache_key("explore", "languages") + languages = cache_get(languages_key) + if languages is None: + languages = list( + Issue.objects.exclude(language__isnull=True) + .exclude(language="") + .values_list("language", flat=True) + .distinct() + .order_by("language") + ) + cache_set(languages_key, languages, CACHE_TTL) + + context = { + "page_obj": page_obj, + "total_count": paginator.count, + "querystring": querystring, + "languages": languages, + "filters": { + "q": q, + "language": language, + "difficulty": difficulty, + "foundation": foundation, + "max_size": max_size, + "sort": sort, + }, + "difficulties": Issue.Difficulty.choices, + } + + template = "core/_issue_list.html" if request.htmx else "core/explore.html" + response = render(request, template, context) + cache_set(filters_key, response.content, CACHE_TTL) + return response + + +def issue_detail(request, pk): + issue = get_object_or_404(Issue, pk=pk) + related = ( + Issue.objects.filter(language=issue.language, is_assigned=False) + .exclude(pk=issue.pk) + .only(*ISSUE_LIST_FIELDS) + .order_by("-beginner_friendly_score")[:3] + ) + return render( + request, "core/issue_detail.html", {"issue": issue, "related": related} + ) + + +def profile_view(request, username): + profile = get_object_or_404( + Profile.objects.select_related("user"), github_username=username + ) + contributions = profile.user.contributions.select_related("issue").only( + "id", + "user_id", + "issue_id", + "repo_full_name", + "title", + "pr_url", + "kind", + "merged_at", + "verified", + "created_at", + ) + languages = profile.languages_list + return render( + request, + "core/profile.html", + { + "profile": profile, + "contributions": contributions, + "languages": languages, + }, + ) + + +def leaderboard(request): + country = request.GET.get("country", "").strip() + + # Cache only the computed rows (not rendered HTML), so the template is + # rendered per request. This also fixes #38 (anonymous navbar served to + # logged-in users) and means pagination doesn't need page in the key. + rows_key = cache_key("leaderboard", "rows", country.lower() or "all") + profiles = cache_get(rows_key) + if profiles is None: + qs = ( + Profile.objects.select_related("user") + .annotate(total=Count("user__contributions")) + .filter(total__gt=0) + .order_by("-total", "pk") # pk tiebreaker = stable ranking across pages + ) + if country: + qs = qs.filter(country__iexact=country) + profiles = list(qs) # no [:50] slice any more + cache_set(rows_key, profiles, CACHE_TTL) + + countries_key = cache_key("leaderboard", "countries") + countries = cache_get(countries_key) + if countries is None: + countries = list( + Profile.objects.exclude(country__isnull=True) + .exclude(country="") + .values_list("country", flat=True) + .distinct() + .order_by("country") + ) + cache_set(countries_key, countries, CACHE_TTL) + + paginator = Paginator(profiles, LEADERBOARD_PAGE_SIZE) + # get_page() handles ?page=abc and out-of-range values safely. + page_obj = paginator.get_page(request.GET.get("page", "1")) + page_rows = list(page_obj.object_list) + + on_first_page = page_obj.number == 1 + if on_first_page: + podium, rest = page_rows[:3], page_rows[3:] + else: + podium, rest = [], page_rows + + # Rank of the row just before `rest` begins: 3 on page 1, 50 on page 2, ... + rest_offset = max(page_obj.start_index() - 1, 0) + (len(podium)) + + querystring = urlencode({"country": country}) if country else "" + + return render( + request, + "core/leaderboard.html", + { + "profiles": page_rows, + "podium": podium, + "rest": rest, + "rest_offset": rest_offset, + "top": profiles[0] if (profiles and on_first_page) else None, + "page_obj": page_obj, + "querystring": querystring, + "countries": countries, + "selected_country": country, + }, + ) + + +@require_http_methods(["GET", "HEAD"]) +def health(request): + """Cheap liveness check for the keep-alive cron. + + Must not touch the database or the cache: Render's cold start plus a DB + query is what made cron-job.org time out and disable the job. + """ + return HttpResponse("ok", content_type="text/plain") + + +@csrf_exempt +@require_http_methods(["GET", "POST"]) +def fetch_issues_trigger(request): + """Cron-friendly ingestion endpoint (Render free → cron-job.org). + + Auth: ``?token=…`` or header ``X-Fetch-Token: …`` must match + ``FETCH_ISSUES_TOKEN``. Returns 503 if the token is not configured. + + The heavy GitHub crawl runs in a background thread so the HTTP response + returns quickly (Render / cron free tiers time out long requests). + """ + expected = getattr(settings, "FETCH_ISSUES_TOKEN", "") or "" + if not expected: + return HttpResponse( + "FETCH_ISSUES_TOKEN is not configured.", + status=503, + content_type="text/plain", + ) + + provided = ( + request.headers.get("X-Fetch-Token") + or request.GET.get("token") + or request.POST.get("token") + or "" + ) + if provided != expected: + return HttpResponseForbidden("Invalid token.") + + pages = request.GET.get("pages") or request.POST.get("pages") or "1" + try: + pages = max(1, min(int(pages), 5)) + except (TypeError, ValueError): + pages = 1 + + sync = (request.GET.get("sync") or request.POST.get("sync") or "") in { + "1", + "true", + "yes", + } + + from .tasks import ingest_issues + + if sync: + count = ingest_issues(pages=pages) + return JsonResponse({"ok": True, "processed": count, "pages": pages}) + + import logging + import threading + + logger = logging.getLogger(__name__) + + def _run(): + try: + processed = ingest_issues(pages=pages) + logger.info( + "Background fetch_issues finished: %s issues", + processed, + extra={"event": "fetch_issues_ok"}, + ) + except Exception: + logger.exception( + "Background fetch_issues failed", + extra={"event": "fetch_issues_error"}, + ) + + threading.Thread(target=_run, name="fetch-issues", daemon=True).start() + return JsonResponse( + {"ok": True, "started": True, "pages": pages, "mode": "async"}, + status=202, + ) + + +def llms_txt(request): + """Serve a short machine-readable description for AI crawlers (llms.txt).""" + content = """# OpenToAll + +> Find an open source issue you can actually clone and finish. Built for developers who need a realistic first contribution. + +OpenToAll aggregates `good first issue` and `help wanted` issues from GitHub, with filters for language, difficulty, and maintainer responsiveness. It aims to make open-source contribution more accessible, particularly for developers in Africa. + +## Key pages + +- [Home](https://opentoall.onrender.com/): Project overview +- [Explore](https://opentoall.onrender.com/explore/): Search and filter issues +- [Leaderboard](https://opentoall.onrender.com/leaderboard/): Contributor rankings by country +- [GitHub](https://github.com/Ymax27/opentoall): Source repository + +## Preferred crawl paths + +- /explore/ — issue listings +- /leaderboard/ — community statistics +- Avoid /admin/ and /internal/ endpoints +""" + return HttpResponse(content, content_type="text/plain")