fetch_all.py

# fetch_all.py — pulls data from all sources, persists critical trial changes

import requests
import json
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta
from config import (
    PUBMED_EMAIL, THERAPEUTIC_AREAS,
    PUBMED_MAX_RESULTS, PUBMED_DAYS_BACK, NEWS_DAYS_BACK
)
from trials_store import upsert_trials

TODAY = datetime.today()
PUBMED_DATE_FROM = (TODAY - timedelta(days=PUBMED_DAYS_BACK)).strftime("%Y/%m/%d")
NEWS_DATE_FROM   = (TODAY - timedelta(days=NEWS_DAYS_BACK)).strftime("%Y-%m-%d")


# ── PubMed ────────────────────────────────────────────────────────────────────
# Fetches recently PUBLISHED trial results — phase 2/3 RCTs and clinical trials
# Uses a wider 7-day window by default since high-impact journals publish weekly

def fetch_pubmed(term: str) -> list[dict]:
    base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"

    # Focus on published results: RCTs + clinical trials, exclude protocols/reviews
    query = (
        f"({term})"
        ' AND ("randomized controlled trial"[pt] OR "clinical trial, phase ii"[pt] OR "clinical trial, phase iii"[pt])'
        ' NOT ("protocol"[ti] OR "study design"[ti] OR "review"[pt])'
        f' AND ("{PUBMED_DATE_FROM}"[pdat] : "3000"[pdat])'  # pdat = publication date (more reliable than edat)
    )

    search_resp = requests.get(f"{base}/esearch.fcgi", params={
        "db": "pubmed", "term": query,
        "retmax": PUBMED_MAX_RESULTS, "retmode": "json",
        "sort": "pub_date",  # most recently published first
        "tool": "pharma_pipeline", "email": PUBMED_EMAIL,
    })
    ids = search_resp.json().get("esearchresult", {}).get("idlist", [])
    if not ids:
        return []

    fetch_resp = requests.get(f"{base}/efetch.fcgi", params={
        "db": "pubmed", "id": ",".join(ids),
        "rettype": "abstract", "retmode": "xml",
        "tool": "pharma_pipeline", "email": PUBMED_EMAIL,
    })
    try:
        root = ET.fromstring(fetch_resp.content)
    except ET.ParseError as e:
        print(f"  [pubmed] XML parse error (likely rate-limited): {e}")
        return []
    articles = []

    for article in root.findall(".//PubmedArticle"):
        title_el    = article.find(".//ArticleTitle")
        pmid_el     = article.find(".//PMID")
        journal_el  = article.find(".//Journal/Title")
        pub_date_el = article.find(".//PubDate/Year")

        # Collect all abstract sections (structured abstracts have Background, Methods, Results, Conclusions)
        abstract_parts = []
        for ab in article.findall(".//AbstractText"):
            label = ab.get("Label", "")
            text  = "".join(ab.itertext()).strip()
            if label:
                abstract_parts.append(f"{label}: {text}")
            else:
                abstract_parts.append(text)
        abstract = " | ".join(abstract_parts)[:1200]  # keep more — results section is critical

        # Extract primary outcome / results mentions if present
        title = "".join(title_el.itertext()) if title_el is not None else "No title"

        articles.append({
            "source":   "PubMed",
            "pmid":     pmid_el.text if pmid_el is not None else "",
            "title":    title,
            "abstract": abstract,
            "journal":  journal_el.text if journal_el is not None else "",
            "year":     pub_date_el.text if pub_date_el is not None else "",
            "url":      f"https://pubmed.ncbi.nlm.nih.gov/{pmid_el.text}/" if pmid_el is not None else "",
        })
    return articles


# ── Google News RSS ───────────────────────────────────────────────────────────
# Free, no API key, real-time, no rate limits
# Targets pharma-specific outlets: STAT, FiercePharma, NEJM, Lancet, etc.

def fetch_news(keywords: str) -> list[dict]:
    import urllib.parse

    # Narrow to high-quality pharma/medical sources
    source_filter = (
        "site:statnews.com OR site:fiercepharma.com OR site:reuters.com OR "
        "site:nejm.org OR site:thelancet.com OR site:nature.com OR "
        "site:medpagetoday.com OR site:biopharmadive.com OR site:endpoints.news"
    )
    query = f"({keywords}) ({source_filter})"
    encoded = urllib.parse.quote(query)
    url = f"https://news.google.com/rss/search?q={encoded}&hl=en-US&gl=US&ceid=US:en"

    try:
        resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"})
        root = ET.fromstring(resp.content)
    except Exception as e:
        print(f"  [news] Google News RSS failed: {e}")
        return []

    articles = []
    for item in root.findall(".//item")[:20]:
        title   = item.findtext("title", "").strip()
        link    = item.findtext("link", "").strip()
        pub     = item.findtext("pubDate", "").strip()
        source  = item.findtext("source", "").strip()
        desc    = item.findtext("description", "").strip()

        if not title or not link:
            continue

        articles.append({
            "source":       source,
            "title":        title,
            "description":  desc[:300],
            "url":          link,
            "published_at": pub,
        })

    return articles


# ── ClinicalTrials.gov ────────────────────────────────────────────────────────
# FIXED: now filters to trials with POSTED RESULTS only
# This gives you actual outcome data, not just newly registered/recruiting trials

def fetch_clinical_trials(condition: str) -> tuple[list[dict], list[dict]]:
    resp = requests.get("https://clinicaltrials.gov/api/v2/studies", params={
        "query.cond": condition,
        "filter.advanced": (
            # Key fix: AREA[ResultsFirstPostDate] = only trials that have posted results
            f"AREA[ResultsFirstPostDate]RANGE[{NEWS_DATE_FROM},{TODAY.strftime('%Y-%m-%d')}]"
        ),
        "fields": (
            "NCTId,BriefTitle,OverallStatus,Phase,BriefSummary,"
            "ResultsFirstPostDate,PrimaryOutcomeMeasure,PrimaryOutcomeDescription"
        ),
        "pageSize": 15,
        "format": "json",
    })

    if resp.status_code != 200:
        print(f"  [trials] API error {resp.status_code}")
        return [], []

    raw_trials = []
    for s in resp.json().get("studies", []):
        proto      = s.get("protocolSection", {})
        results_s  = s.get("resultsSection", {})
        id_mod     = proto.get("identificationModule", {})
        status_mod = proto.get("statusModule", {})
        desc_mod   = proto.get("descriptionModule", {})
        design_mod = proto.get("designModule", {})
        outcomes   = proto.get("outcomesModule", {})

        # Extract primary outcome measure if available
        primary_outcomes = outcomes.get("primaryOutcomes", [])
        primary_outcome  = primary_outcomes[0].get("measure", "") if primary_outcomes else ""

        # Try to get baseline characteristics summary from results section
        outcome_overview = ""
        baseline = results_s.get("baselineCharacteristicsModule", {})
        if baseline:
            outcome_overview = str(baseline)[:300]

        raw_trials.append({
            "nct_id":          id_mod.get("nctId", ""),
            "title":           id_mod.get("briefTitle", ""),
            "status":          status_mod.get("overallStatus", ""),
            "phase":           ", ".join(design_mod.get("phases", [])),
            "summary":         desc_mod.get("briefSummary", "")[:500],
            "primary_outcome": primary_outcome,
            "results_posted":  status_mod.get("resultsFirstPostDateStruct", {}).get("date", ""),
            "url":             f"https://clinicaltrials.gov/study/{id_mod.get('nctId', '')}",
        })

    change_events = upsert_trials(raw_trials)
    return raw_trials, change_events


# ── openFDA ───────────────────────────────────────────────────────────────────

def fetch_fda_approvals() -> list[dict]:
    date_from = (TODAY - timedelta(days=PUBMED_DAYS_BACK)).strftime("%Y%m%d")
    date_to   = TODAY.strftime("%Y%m%d")

    resp = requests.get("https://api.fda.gov/drug/drugsfda.json", params={
        "search": f"submissions.submission_status_date:[{date_from}+TO+{date_to}]",
        "limit": 10,
    })
    if resp.status_code != 200:
        return []

    results = []
    for r in resp.json().get("results", []):
        drug_name = r.get("openfda", {}).get("brand_name", ["Unknown"])[0]
        generic   = r.get("openfda", {}).get("generic_name", [""])[0]
        for sub in r.get("submissions", []):
            if sub.get("submission_type") in ("ORIG", "SUPPL"):
                results.append({
                    "source":  "openFDA",
                    "drug":    f"{drug_name} ({generic})" if generic else drug_name,
                    "action":  sub.get("submission_type", ""),
                    "status":  sub.get("submission_status", ""),
                    "date":    sub.get("submission_status_date", ""),
                    "url":     "https://www.fda.gov/drugs/drug-approvals-and-databases/novel-drug-approvals-fda",
                })
    return results


# ── Main ──────────────────────────────────────────────────────────────────────

def fetch_all() -> dict:
    print(f"\n{'='*50}")
    print(f"Fetching data for {TODAY.strftime('%A, %B %d %Y')}")
    print(f"{'='*50}")

    data = {}

    for ta_name, ta_config in THERAPEUTIC_AREAS.items():
        print(f"\n[{ta_name.upper()}]")
        ta_data = {"pubmed": [], "news": [], "trials": [], "trial_changes": []}

        print(f"  Fetching PubMed (published trial results)...")
        seen_pmids = set()
        for term in ta_config["pubmed_terms"]:
            for a in fetch_pubmed(term):
                if a["pmid"] not in seen_pmids:
                    ta_data["pubmed"].append(a)
                    seen_pmids.add(a["pmid"])
        print(f"  → {len(ta_data['pubmed'])} articles")

        print(f"  Fetching news (Google News RSS)...")
        ta_data["news"] = fetch_news(ta_config["news_keywords"])
        print(f"  → {len(ta_data['news'])} articles")

        print(f"  Fetching ClinicalTrials.gov (results posted only)...")
        trials, changes = fetch_clinical_trials(ta_name)
        ta_data["trials"]        = trials
        ta_data["trial_changes"] = changes
        print(f"  → {len(trials)} trials with results, {len(changes)} change(s)")

        data[ta_name] = ta_data

    print(f"\n[FDA APPROVALS]")
    data["fda"] = fetch_fda_approvals()
    print(f"  → {len(data['fda'])} actions")

    return data


if __name__ == "__main__":
    results = fetch_all()
    print(json.dumps(results, indent=2))