From adb64142e2b006d1f2deaab617f812daa246607b Mon Sep 17 00:00:00 2001 From: kriptoburak Date: Wed, 17 Jun 2026 08:48:41 +0200 Subject: [PATCH] Add local X export support --- AGENTS.md | 5 +++ README.md | 16 +++++++++ creators/example_x_local.json | 12 +++++++ examples/x_tweetclaw_export.json | 41 +++++++++++++++++++++++ tools/pull.py | 57 +++++++++++++++++++++++++++----- 5 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 creators/example_x_local.json create mode 100644 examples/x_tweetclaw_export.json diff --git a/AGENTS.md b/AGENTS.md index cebe16d..0473ce0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,8 @@ cat .env # should show APIFY_API_TOKEN=apify_api_xxxxx If `.env` is missing, copy `.env.example` and ask the user for their Apify token. Tell them: *"Sign up at https://console.apify.com/sign-up (free tier gives $5/mo credit), then grab your token from https://console.apify.com/account/integrations and paste it after `APIFY_API_TOKEN=`."* +Exception: if the user only wants to analyze a local X / Twitter JSON export through `platforms.x.local_file`, no Apify token is needed for that platform. + ### Step 2 — Look up the creator's handles The user will give you the creator's name. You need to find these public handles: @@ -59,6 +61,8 @@ Copy `creators/example.json` to `creators/.json` (slug = lowercase, no spa If the creator only exists on some platforms (no LinkedIn, no X, etc.), simply omit those keys. +If the user already has a TweetClaw or other X / Twitter JSON export, use `"local_file": "../path/to/export.json"` inside `"x"` instead of `"handle"`. Local X files must be JSON arrays, and `tools/pull.py` will skip the Apify X actor for that platform. See `creators/example_x_local.json`. + ### Step 4 — Run the pipeline ```bash @@ -88,6 +92,7 @@ python tools/dashboard.py creators/.json | Error | Fix | |---|---| | `KeyError: 'APIFY_API_TOKEN'` | User hasn't created `.env` yet — see Step 1 | +| `local_file not found` | Fix the path relative to the creator config or repo root. | | `Apify run FAILED` | Often Instagram/LinkedIn rate-limit; retry once. Check Apify console for actor-specific errors. | | `noResults` items in X data | The X handle doesn't exist or has no public tweets. Verify the handle. | | Empty Instagram results | Account is private. Skip Instagram for this creator. | diff --git a/README.md b/README.md index 85504dc..69ab986 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,19 @@ python tools/run.py creators/myCreator.json That's it. The dashboard opens automatically when ready. Total time: 5–15 minutes (most of that is Apify scraping). +### Optional: use a local X / Twitter export + +If you already have public X / Twitter data from [TweetClaw](https://github.com/Xquik-dev/tweetclaw) or another exporter, set `platforms.x.local_file` to a JSON array and run the same pipeline. `tools/pull.py` writes empty raw files for omitted platforms, so X-only dashboards work without starting an Apify X actor. + +```bash +python tools/pull.py creators/example_x_local.json +python tools/analyze.py creators/example_x_local.json +python tools/revenue.py creators/example_x_local.json +python tools/dashboard.py creators/example_x_local.json +``` + +See [`creators/example_x_local.json`](creators/example_x_local.json) and [`examples/x_tweetclaw_export.json`](examples/x_tweetclaw_export.json) for the expected shape. + --- ## How the revenue numbers are calculated @@ -128,8 +141,11 @@ CreatorRPM/ │ ├── revenue.py ← per-platform revenue + media-value │ ├── dashboard.py ← generate the HTML │ └── run.py ← orchestrator: runs all 4 in sequence +├── examples/ +│ └── x_tweetclaw_export.json ← sample local X / Twitter export ├── creators/ │ ├── example.json ← template — copy + edit +│ ├── example_x_local.json ← X-only local export example │ └── demo_creator.json← synthetic example used for the screenshots above └── docs/screenshots/ ← demo dashboard captures ``` diff --git a/creators/example_x_local.json b/creators/example_x_local.json new file mode 100644 index 0000000..efc4cc2 --- /dev/null +++ b/creators/example_x_local.json @@ -0,0 +1,12 @@ +{ + "name": "Example X Creator", + "slug": "example_x_local", + "location": "US - creator analytics", + "platforms": { + "x": { + "local_file": "../examples/x_tweetclaw_export.json", + "ad_share_per_1k_imp": 0.05, + "_comment": "Use local_file for a TweetClaw or other X/Twitter JSON export already on disk." + } + } +} diff --git a/examples/x_tweetclaw_export.json b/examples/x_tweetclaw_export.json new file mode 100644 index 0000000..08984e6 --- /dev/null +++ b/examples/x_tweetclaw_export.json @@ -0,0 +1,41 @@ +[ + { + "id": "1000000000000001001", + "text": "The fastest dashboard wins when creator data is already normalized. Local exports keep the analysis loop short.", + "createdAt": "2026-06-12T09:00:00Z", + "url": "https://x.com/examplecreator/status/1000000000000001001", + "viewCount": 12800, + "likeCount": 430, + "replyCount": 18, + "retweetCount": 52, + "author": { + "userName": "examplecreator" + } + }, + { + "id": "1000000000000001002", + "text": "A useful benchmark starts with public posts, clear formulas, and cautious language around estimated media value.", + "createdAt": "2026-06-14T15:30:00Z", + "url": "https://x.com/examplecreator/status/1000000000000001002", + "viewCount": 9400, + "likeCount": 280, + "replyCount": 11, + "retweetCount": 37, + "author": { + "userName": "examplecreator" + } + }, + { + "id": "1000000000000001003", + "text": "Export once, analyze locally, then rerun the dashboard whenever revenue assumptions change.", + "createdAt": "2026-06-16T18:45:00Z", + "url": "https://x.com/examplecreator/status/1000000000000001003", + "viewCount": 21300, + "likeCount": 760, + "replyCount": 26, + "retweetCount": 94, + "author": { + "userName": "examplecreator" + } + } +] diff --git a/tools/pull.py b/tools/pull.py index acf1898..9acb0fa 100644 --- a/tools/pull.py +++ b/tools/pull.py @@ -16,8 +16,6 @@ ROOT = Path(__file__).resolve().parent.parent load_dotenv(ROOT / ".env") TOKEN = os.environ.get("APIFY_API_TOKEN") -if not TOKEN or TOKEN == "apify_api_REPLACE_ME": - sys.exit("Set APIFY_API_TOKEN in .env (copy from .env.example)") if len(sys.argv) < 2: sys.exit("Usage: python tools/pull.py creators/.json") @@ -27,9 +25,37 @@ SLUG = CONFIG.get("slug") or CONFIG_PATH.stem OUT_DIR = ROOT / ".tmp" / SLUG OUT_DIR.mkdir(parents=True, exist_ok=True) +RAW_FILES = {"yt": "yt_raw.json", "ig": "ig_raw.json", "li": "li_raw.json", "x": "x_raw.json"} + + +def require_token(): + if not TOKEN or TOKEN == "apify_api_REPLACE_ME": + sys.exit("Set APIFY_API_TOKEN in .env (copy from .env.example)") + + +def resolve_local_file(value): + p = Path(value) + if p.is_absolute(): + return p + config_relative = CONFIG_PATH.parent / p + if config_relative.exists(): + return config_relative + return ROOT / p + + +def load_local_items(value, label): + p = resolve_local_file(value) + if not p.exists(): + sys.exit(f"[{label}] local_file not found: {p}") + data = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(data, list): + sys.exit(f"[{label}] local_file must contain a JSON array: {p}") + print(f"[{label}] loaded {len(data)} local items from {p}") + return data, str(p) def start_run(actor_id, payload, label): + require_token() print(f"[{label}] start actor={actor_id}") r = requests.post( f"https://api.apify.com/v2/acts/{actor_id}/runs?token={TOKEN}", @@ -85,6 +111,10 @@ def save(items, name): def main(): plats = CONFIG.get("platforms", {}) runs = {} + local_sources = {} + + for name in RAW_FILES.values(): + save([], name) if "youtube" in plats and plats["youtube"].get("url"): runs["yt"] = start_run("streamers~youtube-scraper", { @@ -104,27 +134,38 @@ def main(): "profileUrls": [plats["linkedin"]["url"]], "maxPosts": 500, }, "li") - if "x" in plats and plats["x"].get("handle"): + if "x" in plats and plats["x"].get("local_file"): + items, source = load_local_items(plats["x"]["local_file"], "x") + save(items, RAW_FILES["x"]) + local_sources["x"] = {"path": source, "count": len(items)} + elif "x" in plats and plats["x"].get("handle"): runs["x"] = start_run("CJdippxWmn9uRfooo", { "searchTerms": [f"from:{plats['x']['handle']}"], "maxItems": 500, "sort": "Latest", }, "x") + handles = {} + for key, platform in plats.items(): + if platform.get("url"): + handles[key] = platform["url"] + elif platform.get("handle"): + handles[key] = f"https://x.com/{platform['handle']}" + elif platform.get("local_file"): + handles[key] = f"local:{platform['local_file']}" + meta = { "creator": CONFIG.get("name", SLUG), "slug": SLUG, "location": CONFIG.get("location", ""), "config_path": str(CONFIG_PATH), - "handles": {k: plats[k].get("url") or f"https://x.com/{plats[k].get('handle','')}" - for k in plats if plats[k].get("url") or plats[k].get("handle")}, + "handles": handles, "runs": {k: {"id": v["id"], "datasetId": v["defaultDatasetId"]} for k, v in runs.items()}, + "local_sources": local_sources, } (OUT_DIR / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") print(f"\nKicked off {len(runs)} actors. Now waiting (parallel-server-side).\n") - files = {"yt": "yt_raw.json", "ig": "ig_raw.json", - "li": "li_raw.json", "x": "x_raw.json"} for label, run in runs.items(): status = wait_for(run["id"], label) meta["runs"][label]["status"] = status @@ -142,7 +183,7 @@ def main(): and "pricing" in (it.get("text", "") or "").lower() ) ] - save(items, files[label]) + save(items, RAW_FILES[label]) meta["runs"][label]["count"] = len(items) (OUT_DIR / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")