From 5b47d46951a6fdaa3b79faa4a2fa66105da6a69c Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:12:48 +0100 Subject: [PATCH 01/68] fix(area): improve error handling for incompatible action-reaction combinations --- backend/automations/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/automations/serializers.py b/backend/automations/serializers.py index 0d0b4e10..b6afa0a9 100755 --- a/backend/automations/serializers.py +++ b/backend/automations/serializers.py @@ -275,7 +275,11 @@ def validate(self, attrs): try: validate_action_reaction_compatibility(action.name, reaction.name) except serializers.ValidationError as e: - raise serializers.ValidationError({"non_field_errors": str(e)}) + # Re-raise with better field targeting for frontend display + error_message = str(e.detail[0]) if hasattr(e, 'detail') and e.detail else str(e) + raise serializers.ValidationError({ + "reaction": f"⚠️ Incompatible combination: {error_message}" + }) # Validation des configurations si elles sont fournies action_config = attrs.get("action_config", {}) From d14e062d27ce455a4846a3029ab1fde1234b727b Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:13:24 +0100 Subject: [PATCH 02/68] fix(validators): update Gmail reaction schemas to require message_id and improve error messaging --- backend/automations/validators.py | 48 ++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/backend/automations/validators.py b/backend/automations/validators.py index 4c0e9c9b..c5cd3084 100755 --- a/backend/automations/validators.py +++ b/backend/automations/validators.py @@ -570,20 +570,20 @@ def validate_email_format(email): "gmail_mark_read": { "type": "object", "properties": { - "email_id": { + "message_id": { "type": "string", - "description": "Gmail message ID to mark as read", + "description": "Gmail message ID to mark as read (automatically provided by Gmail triggers)", }, }, - "required": ["email_id"], + "required": ["message_id"], "additionalProperties": False, }, "gmail_add_label": { "type": "object", "properties": { - "email_id": { + "message_id": { "type": "string", - "description": "Gmail message ID", + "description": "Gmail message ID (automatically provided by Gmail triggers)", }, "label": { "type": "string", @@ -591,7 +591,7 @@ def validate_email_format(email): "description": "Label name to add", }, }, - "required": ["email_id", "label"], + "required": ["message_id", "label"], "additionalProperties": False, }, # Google Calendar Reactions @@ -1172,9 +1172,21 @@ def validate_reaction_config(reaction_name, config): ) except JsonSchemaValidationError as e: - raise serializers.ValidationError( - f"Invalid configuration for reaction '{reaction_name}': {e.message}" - ) + # Provide helpful context for common mistakes + error_message = f"Invalid configuration for reaction '{reaction_name}': {e.message}" + + # Special handling for Gmail reactions that need message_id + if reaction_name in ["gmail_mark_read", "gmail_add_label"]: + if "message_id" in e.message or "email_id" in e.message: + error_message = ( + f"⚠️ The '{reaction_name}' reaction requires an email's message ID, " + "which can only be provided by Gmail trigger actions (like 'New Email', " + "'Email from Sender', etc.). " + "Please use a Gmail trigger action instead of the current trigger. " + f"Technical details: {e.message}" + ) + + raise serializers.ValidationError(error_message) def validate_action_reaction_compatibility(action_name, reaction_name): @@ -1204,9 +1216,23 @@ def validate_action_reaction_compatibility(action_name, reaction_name): # Check if the specific reaction is allowed if reaction_name not in compatible_reactions: + # Provide helpful context for Gmail reactions + context_message = "" + if reaction_name in ["gmail_mark_read", "gmail_add_label"]: + context_message = ( + " These Gmail reactions require an email trigger (gmail_new_email, " + "gmail_new_from_sender, etc.) because they need the email's message ID " + "to work. Timer or other non-Gmail triggers cannot provide this information." + ) + elif reaction_name == "github_create_issue": + context_message = ( + " Note: This reaction works with most triggers, but the issue content " + "will depend on the data provided by the trigger." + ) + raise serializers.ValidationError( - f"Action '{action_name}' is not compatible with reaction '{reaction_name}'. " - f"Compatible reactions: {', '.join(compatible_reactions)}" + f"Action '{action_name}' cannot trigger reaction '{reaction_name}'.{context_message} " + f"Compatible reactions for this action: {', '.join(compatible_reactions) if compatible_reactions else 'none available'}" ) From a8c5d21921b07d3d66c5cdf2384da31759c3b26d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:16 +0100 Subject: [PATCH 03/68] feat(celery): add periodic task for checking YouTube actions every 5 minutes --- backend/area_project/celery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index 188671a2..60c96aa6 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -59,6 +59,10 @@ def get_beat_schedule(): "task": "automations.check_gmail_actions", "schedule": 180.0, # Every 3 minutes (no webhook support yet) }, + "check-youtube-actions": { + "task": "automations.check_youtube_actions", + "schedule": 300.0, # Every 5 minutes (polling + webhook support) + }, "check-weather-actions": { "task": "automations.check_weather_actions", "schedule": 300.0, # Every 5 minutes (no webhook support) From 81332e99e5f8f18bd05f81ddfd59d0a9fa7858e8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:20 +0100 Subject: [PATCH 04/68] feat(oauth2): add YouTube API scopes for read and force-ssl access --- backend/area_project/settings/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/area_project/settings/base.py b/backend/area_project/settings/base.py index 63e9fcb2..05923327 100755 --- a/backend/area_project/settings/base.py +++ b/backend/area_project/settings/base.py @@ -213,6 +213,8 @@ "https://www.googleapis.com/auth/gmail.modify", "https://www.googleapis.com/auth/calendar.readonly", "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/youtube.readonly", + "https://www.googleapis.com/auth/youtube.force-ssl", ], "requires_refresh": True, }, From 5c891a29016727f1c11afd72024f8b4dbaa996ef Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:25 +0100 Subject: [PATCH 05/68] feat(serializers): map YouTube service to Google OAuth --- backend/automations/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/automations/serializers.py b/backend/automations/serializers.py index b6afa0a9..82020829 100755 --- a/backend/automations/serializers.py +++ b/backend/automations/serializers.py @@ -533,6 +533,7 @@ def get_requires_oauth(self, obj): service_oauth_map = { "gmail": "google", "google_calendar": "google", + "youtube": "google", } mapped_oauth = service_oauth_map.get(obj.name) From 3e339e0399b6dd383dcc6714c6ab16dc0d6b615e Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:30 +0100 Subject: [PATCH 06/68] feat(youtube): add YouTube reaction handling and polling task --- backend/automations/tasks.py | 339 +++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index 292a300a..6f0e2b38 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -3540,6 +3540,90 @@ def _execute_reaction_logic( logger.error(f"[REACTION SPOTIFY] Failed to create playlist: {e}") raise ValueError(f"Spotify create_playlist failed: {str(e)}") from e + elif reaction_name == "youtube_post_comment": + # Post a comment on a YouTube video + from users.oauth.manager import OAuthManager + + from .helpers.youtube_helper import post_comment + + access_token = OAuthManager.get_valid_token(area.owner, "google") + if not access_token: + raise ValueError(f"No valid Google token for user {area.owner.username}") + + video_id = reaction_config.get("video_id") or trigger_data.get("video_id") + comment_text = reaction_config.get("comment_text") + + if not video_id or not comment_text: + raise ValueError( + "Video ID and comment text required for youtube_post_comment" + ) + + try: + result = post_comment(access_token, video_id, comment_text) + + logger.info(f"[REACTION YOUTUBE] Posted comment on video {video_id}") + return result + + except Exception as e: + logger.error(f"[REACTION YOUTUBE] Failed to post comment: {e}") + raise ValueError(f"YouTube post_comment failed: {str(e)}") from e + + elif reaction_name == "youtube_add_to_playlist": + # Add video to playlist + from users.oauth.manager import OAuthManager + + from .helpers.youtube_helper import add_video_to_playlist + + access_token = OAuthManager.get_valid_token(area.owner, "google") + if not access_token: + raise ValueError(f"No valid Google token for user {area.owner.username}") + + video_id = reaction_config.get("video_id") or trigger_data.get("video_id") + playlist_id = reaction_config.get("playlist_id") + + if not video_id or not playlist_id: + raise ValueError( + "Video ID and playlist ID required for youtube_add_to_playlist" + ) + + try: + result = add_video_to_playlist(access_token, video_id, playlist_id) + + logger.info( + f"[REACTION YOUTUBE] Added video {video_id} to playlist {playlist_id}" + ) + return result + + except Exception as e: + logger.error(f"[REACTION YOUTUBE] Failed to add to playlist: {e}") + raise ValueError(f"YouTube add_to_playlist failed: {str(e)}") from e + + elif reaction_name == "youtube_rate_video": + # Rate a video (like/dislike) + from users.oauth.manager import OAuthManager + + from .helpers.youtube_helper import rate_video + + access_token = OAuthManager.get_valid_token(area.owner, "google") + if not access_token: + raise ValueError(f"No valid Google token for user {area.owner.username}") + + video_id = reaction_config.get("video_id") or trigger_data.get("video_id") + rating = reaction_config.get("rating", "like") + + if not video_id: + raise ValueError("Video ID required for youtube_rate_video") + + try: + rate_video(access_token, video_id, rating) + + logger.info(f"[REACTION YOUTUBE] Rated video {video_id} as '{rating}'") + return {"success": True, "video_id": video_id, "rating": rating} + + except Exception as e: + logger.error(f"[REACTION YOUTUBE] Failed to rate video: {e}") + raise ValueError(f"YouTube rate_video failed: {str(e)}") from e + else: # Unknown reaction - log and continue logger.warning( @@ -3553,6 +3637,261 @@ def _execute_reaction_logic( } +# ==================== YouTube Polling Task ==================== + + +@shared_task( + name="automations.check_youtube_actions", + bind=True, + max_retries=3, + autoretry_for=RECOVERABLE_EXCEPTIONS, +) +def check_youtube_actions(self): + """ + Poll YouTube for new videos and channel statistics. + + Checks all active Areas with YouTube actions and triggers executions + when matching videos are found or thresholds are exceeded. + + Supported actions: + - youtube_new_video: New video uploaded to channel + - youtube_channel_stats: Channel stats exceed threshold + - youtube_search_videos: New videos matching search query + + Returns: + dict: Summary of polling results + """ + from users.oauth.manager import OAuthManager + + from .helpers.youtube_helper import ( + get_channel_statistics, + get_latest_videos, + search_videos, + ) + + logger.info("Checking YouTube actions...") + + try: + # Get all active Areas with YouTube actions + youtube_areas = get_active_areas( + [ + "youtube_new_video", + "youtube_channel_stats", + "youtube_search_videos", + ] + ) + + if not youtube_areas: + logger.info("No active YouTube areas found") + return {"status": "no_areas", "checked": 0} + + triggered_count = 0 + skipped_count = 0 + no_token_count = 0 + + for area in youtube_areas: + try: + # Get valid Google token (YouTube uses Google OAuth) + access_token = OAuthManager.get_valid_token(area.owner, "google") + + if not access_token: + logger.warning( + f"No valid Google token for user {area.owner.username} " + f"(area #{area.pk})" + ) + no_token_count += 1 + continue + + action_name = area.action.name + action_config = area.action_config or {} + + # ===== YOUTUBE NEW VIDEO ===== + if action_name == "youtube_new_video": + channel_id = action_config.get("channel_id") + + if not channel_id: + logger.warning( + f"No channel_id configured for area #{area.pk}, skipping" + ) + skipped_count += 1 + continue + + # Get published_after from updated_at minus 1 hour or None + published_after = None + if area.updated_at: + from datetime import timedelta + one_hour_ago = area.updated_at - timedelta(hours=1) + published_after = one_hour_ago.isoformat() + + # Fetch latest videos + videos = get_latest_videos( + access_token, channel_id, max_results=5, published_after=published_after + ) + + # Create execution for each new video + for video in videos: + event_id = f"youtube_new_video_{video['video_id']}_{area.pk}" + + trigger_data = { + "video_id": video["video_id"], + "video_title": video["title"], + "video_description": video["description"], + "channel_id": video["channel_id"], + "channel_name": video["channel_title"], + "published_at": video["published_at"], + "thumbnail_url": video["thumbnail_url"], + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_id, + trigger_data=trigger_data, + ) + + if created and execution: + execute_reaction_task.delay(execution.pk) + triggered_count += 1 + logger.info( + f"Triggered area #{area.pk} for new video: {video['title']}" + ) + + # ===== YOUTUBE CHANNEL STATS ===== + elif action_name == "youtube_channel_stats": + channel_id = action_config.get("channel_id") + threshold_type = action_config.get("threshold_type", "subscribers") + threshold_value = int(action_config.get("threshold_value", 1000)) + + if not channel_id: + logger.warning( + f"No channel_id configured for area #{area.pk}, skipping" + ) + skipped_count += 1 + continue + + # Get channel statistics + stats = get_channel_statistics(access_token, channel_id) + + if not stats: + logger.warning(f"Could not fetch stats for channel {channel_id}") + skipped_count += 1 + continue + + # Check threshold + metric_map = { + "subscribers": stats["subscriber_count"], + "views": stats["view_count"], + "videos": stats["video_count"], + } + + current_value = metric_map.get(threshold_type, 0) + + if current_value >= threshold_value: + event_id = f"youtube_channel_stats_{channel_id}_{threshold_type}_{threshold_value}_{area.pk}" + + trigger_data = { + "channel_id": channel_id, + "threshold_type": threshold_type, + "threshold_value": threshold_value, + "current_value": current_value, + "subscriber_count": stats["subscriber_count"], + "view_count": stats["view_count"], + "video_count": stats["video_count"], + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_id, + trigger_data=trigger_data, + ) + + if created and execution: + execute_reaction_task.delay(execution.pk) + triggered_count += 1 + logger.info( + f"Triggered area #{area.pk}: {threshold_type} " + f"reached {current_value} (threshold: {threshold_value})" + ) + + # ===== YOUTUBE SEARCH VIDEOS ===== + elif action_name == "youtube_search_videos": + search_query = action_config.get("search_query") + channel_id = action_config.get("channel_id") # Optional + + if not search_query: + logger.warning( + f"No search_query configured for area #{area.pk}, skipping" + ) + skipped_count += 1 + continue + + # Get published_after from updated_at minus 1 hour or None + published_after = None + if area.updated_at: + from datetime import timedelta + one_hour_ago = area.updated_at - timedelta(hours=1) + published_after = one_hour_ago.isoformat() + + # Search for videos + videos = search_videos( + access_token, + search_query, + max_results=5, + channel_id=channel_id, + published_after=published_after, + ) + + # Create execution for each matching video + for video in videos: + event_id = f"youtube_search_{video['video_id']}_{area.pk}" + + trigger_data = { + "video_id": video["video_id"], + "video_title": video["title"], + "video_description": video["description"], + "channel_id": video["channel_id"], + "channel_name": video["channel_title"], + "published_at": video["published_at"], + "thumbnail_url": video["thumbnail_url"], + "search_query": search_query, + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_id, + trigger_data=trigger_data, + ) + + if created and execution: + execute_reaction_task.delay(execution.pk) + triggered_count += 1 + logger.info( + f"Triggered area #{area.pk} for search result: {video['title']}" + ) + + except Exception as e: + logger.error( + f"Error processing YouTube area #{area.pk}: {e}", exc_info=True + ) + continue + + logger.info( + f"YouTube polling complete: {triggered_count} triggered, " + f"{skipped_count} skipped, {no_token_count} no token" + ) + + return { + "status": "success", + "checked": len(youtube_areas), + "triggered": triggered_count, + "skipped": skipped_count, + "no_token": no_token_count, + } + + except Exception as e: + logger.error(f"Error in check_youtube_actions: {e}", exc_info=True) + raise + + # ==================== Admin/Debug Tasks ==================== From 1742e22935e12f6621021ed33c8c8063c4812e5a Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:35 +0100 Subject: [PATCH 07/68] feat(youtube): implement YouTube webhook signature validation and subscription verification --- backend/automations/webhooks.py | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/backend/automations/webhooks.py b/backend/automations/webhooks.py index 4e46d573..e172e186 100755 --- a/backend/automations/webhooks.py +++ b/backend/automations/webhooks.py @@ -34,6 +34,7 @@ from rest_framework.response import Response from django.conf import settings +from django.http import HttpResponse from django.utils import timezone from django.views.decorators.csrf import csrf_exempt @@ -223,6 +224,10 @@ def validate_webhook_signature( return is_valid elif service_name == "gmail": return True + elif service_name == "youtube": + # YouTube PubSubHubbub doesn't require signature validation + # The hub.challenge verification is handled separately + return True # Unknown service, reject validation logger.warning(f"Webhook signature validation not supported for: {service_name}") return False @@ -281,6 +286,23 @@ def extract_event_id( elif object_id: return f"notion_{object_id}" + elif service_name == "youtube": + # YouTube PubSubHubbub provides video ID in feed entry + # Extract from feed entry (XML parsed to dict) + entry = event_data.get("entry", {}) + video_id = entry.get("yt:videoId") or entry.get("video_id") + published = entry.get("published", "") + + if video_id: + # Include published timestamp for uniqueness + return f"youtube_video_{video_id}_{published}" + + # Fallback: use link if available + link = entry.get("link", {}).get("href", "") + if link and "watch?v=" in link: + vid_id = link.split("watch?v=")[1].split("&")[0] + return f"youtube_video_{vid_id}" + # Fallback: generate ID from timestamp and hash timestamp = timezone.now().isoformat() payload_hash = hashlib.sha256(json.dumps(event_data).encode()).hexdigest()[:16] @@ -503,6 +525,26 @@ def webhook_receiver(request: Request, service: str) -> Response: status=status.HTTP_200_OK, ) + # =================================================================== + # YOUTUBE PUBSUBHUBBUB VERIFICATION CHALLENGE + # =================================================================== + # YouTube/PubSubHubbub sends verification via GET with hub.challenge + # We must echo back hub.challenge to confirm subscription + # Format: GET /webhooks/youtube?hub.mode=subscribe&hub.challenge=XXXX + # =================================================================== + if service == "youtube" and request.method == "GET": + hub_mode = request.GET.get("hub.mode") + hub_challenge = request.GET.get("hub.challenge") + hub_topic = request.GET.get("hub.topic") + + if hub_mode == "subscribe" and hub_challenge: + logger.info(f"✅ YouTube PubSubHubbub subscription verification for topic: {hub_topic}") + # Return challenge as plain text + return HttpResponse(hub_challenge, content_type="text/plain", status=200) + elif hub_mode == "unsubscribe" and hub_challenge: + logger.info(f"✅ YouTube PubSubHubbub unsubscribe verification for topic: {hub_topic}") + return HttpResponse(hub_challenge, content_type="text/plain", status=200) + # Get webhook secret from settings webhook_secrets = getattr(settings, "WEBHOOK_SECRETS", {}) webhook_secret = webhook_secrets.get(service) @@ -533,6 +575,11 @@ def webhook_receiver(request: Request, service: str) -> Response: event_type = request.headers.get("X-GitHub-Event", "unknown") elif service == "gmail": event_type = event_data.get("eventType", "message") + elif service == "youtube": + # YouTube PubSubHubbub sends Atom feed updates + # Event type is always "new_video" for feed notifications + event_type = "new_video" + logger.info("🎥 YouTube webhook: New video notification") elif service == "notion": # Debug: log payload structure to understand Notion's format logger.info(f"🔍 Notion payload keys: {list(event_data.keys())}") From 86ca4a4224c2bb99719dd5b81965ae1ca39013a8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:43 +0100 Subject: [PATCH 08/68] feat(youtube): add helper functions for YouTube Data API interactions --- backend/automations/helpers/youtube_helper.py | 388 ++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 backend/automations/helpers/youtube_helper.py diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py new file mode 100644 index 00000000..17aed56b --- /dev/null +++ b/backend/automations/helpers/youtube_helper.py @@ -0,0 +1,388 @@ +## +## EPITECH PROJECT, 2025 +## Area +## File description: +## youtube_helper +## + +"""YouTube Data API v3 helper functions for actions and reactions.""" + +import logging +from typing import Dict, List, Optional + +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +logger = logging.getLogger(__name__) + + +def get_youtube_service(access_token: str): + """ + Build YouTube Data API v3 service from access token. + + Args: + access_token: Valid Google OAuth2 access token with YouTube scopes + + Returns: + YouTube API service resource + """ + creds = Credentials(token=access_token) + return build("youtube", "v3", credentials=creds) + + +def get_latest_videos( + access_token: str, + channel_id: str, + max_results: int = 10, + published_after: Optional[str] = None, +) -> List[Dict]: + """ + Get latest videos from a YouTube channel. + + Args: + access_token: Valid Google OAuth token + channel_id: YouTube channel ID (e.g., 'UC_x5XG1OV2P6uZZ5FSM9Ttw') + max_results: Max videos to return (default: 10, max: 50) + published_after: ISO 8601 timestamp to filter videos (e.g., '2024-01-01T00:00:00Z') + + Returns: + List of video dicts with id, title, description, publishedAt, thumbnails + + Raises: + HttpError: If YouTube API request fails + """ + try: + service = get_youtube_service(access_token) + + # Build search parameters + search_params = { + "part": "id,snippet", + "channelId": channel_id, + "type": "video", + "order": "date", + "maxResults": min(max_results, 50), + } + + if published_after: + search_params["publishedAfter"] = published_after + + # Execute search + results = service.search().list(**search_params).execute() + + videos = [] + for item in results.get("items", []): + video_data = { + "video_id": item["id"]["videoId"], + "title": item["snippet"]["title"], + "description": item["snippet"]["description"], + "published_at": item["snippet"]["publishedAt"], + "channel_id": item["snippet"]["channelId"], + "channel_title": item["snippet"]["channelTitle"], + "thumbnail_url": item["snippet"]["thumbnails"]["default"]["url"], + } + videos.append(video_data) + + logger.info( + f"Found {len(videos)} videos for channel {channel_id} " + f"(published_after={published_after})" + ) + return videos + + except HttpError as e: + logger.error(f"YouTube get_latest_videos failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in get_latest_videos: {e}") + raise + + +def get_channel_statistics(access_token: str, channel_id: str) -> Dict: + """ + Get channel statistics (subscribers, views, video count). + + Args: + access_token: Valid Google OAuth token + channel_id: YouTube channel ID + + Returns: + Dict with channel stats: + - subscriber_count: Total subscribers + - view_count: Total channel views + - video_count: Total videos uploaded + + Raises: + HttpError: If YouTube API request fails + """ + try: + service = get_youtube_service(access_token) + + results = ( + service.channels() + .list(part="statistics", id=channel_id) + .execute() + ) + + if not results.get("items"): + logger.warning(f"Channel not found: {channel_id}") + return {} + + stats = results["items"][0]["statistics"] + channel_stats = { + "subscriber_count": int(stats.get("subscriberCount", 0)), + "view_count": int(stats.get("viewCount", 0)), + "video_count": int(stats.get("videoCount", 0)), + } + + logger.info(f"Channel {channel_id} stats: {channel_stats}") + return channel_stats + + except HttpError as e: + logger.error(f"YouTube get_channel_statistics failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in get_channel_statistics: {e}") + raise + + +def search_videos( + access_token: str, + query: str, + max_results: int = 10, + channel_id: Optional[str] = None, + published_after: Optional[str] = None, +) -> List[Dict]: + """ + Search for videos matching query. + + Args: + access_token: Valid Google OAuth token + query: Search keywords + max_results: Max videos to return (default: 10, max: 50) + channel_id: Optional channel ID to limit search + published_after: ISO 8601 timestamp to filter videos + + Returns: + List of video dicts with id, title, description, publishedAt + + Raises: + HttpError: If YouTube API request fails + """ + try: + service = get_youtube_service(access_token) + + # Build search parameters + search_params = { + "part": "id,snippet", + "q": query, + "type": "video", + "order": "relevance", + "maxResults": min(max_results, 50), + } + + if channel_id: + search_params["channelId"] = channel_id + if published_after: + search_params["publishedAfter"] = published_after + + # Execute search + results = service.search().list(**search_params).execute() + + videos = [] + for item in results.get("items", []): + video_data = { + "video_id": item["id"]["videoId"], + "title": item["snippet"]["title"], + "description": item["snippet"]["description"], + "published_at": item["snippet"]["publishedAt"], + "channel_id": item["snippet"]["channelId"], + "channel_title": item["snippet"]["channelTitle"], + "thumbnail_url": item["snippet"]["thumbnails"]["default"]["url"], + } + videos.append(video_data) + + logger.info(f"Found {len(videos)} videos for query: '{query}'") + return videos + + except HttpError as e: + logger.error(f"YouTube search_videos failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in search_videos: {e}") + raise + + +def post_comment( + access_token: str, video_id: str, comment_text: str +) -> Dict: + """ + Post a comment on a YouTube video. + + Args: + access_token: Valid Google OAuth token with force-ssl scope + video_id: YouTube video ID + comment_text: Text content of the comment + + Returns: + Dict with comment details (id, text, published_at) + + Raises: + HttpError: If YouTube API request fails + """ + try: + service = get_youtube_service(access_token) + + # Build comment resource + comment_resource = { + "snippet": { + "videoId": video_id, + "topLevelComment": { + "snippet": { + "textOriginal": comment_text + } + } + } + } + + # Insert comment + result = ( + service.commentThreads() + .insert(part="snippet", body=comment_resource) + .execute() + ) + + comment_data = { + "comment_id": result["id"], + "text": result["snippet"]["topLevelComment"]["snippet"]["textDisplay"], + "published_at": result["snippet"]["topLevelComment"]["snippet"]["publishedAt"], + "video_id": video_id, + } + + logger.info(f"Posted comment on video {video_id}: {comment_data['comment_id']}") + return comment_data + + except HttpError as e: + logger.error(f"YouTube post_comment failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in post_comment: {e}") + raise + + +def add_video_to_playlist( + access_token: str, video_id: str, playlist_id: str +) -> Dict: + """ + Add a video to a YouTube playlist. + + Args: + access_token: Valid Google OAuth token with force-ssl scope + video_id: YouTube video ID + playlist_id: YouTube playlist ID + + Returns: + Dict with playlist item details (id, position) + + Raises: + HttpError: If YouTube API request fails + """ + try: + service = get_youtube_service(access_token) + + # Build playlist item resource + playlist_item = { + "snippet": { + "playlistId": playlist_id, + "resourceId": { + "kind": "youtube#video", + "videoId": video_id + } + } + } + + # Insert playlist item + result = ( + service.playlistItems() + .insert(part="snippet", body=playlist_item) + .execute() + ) + + item_data = { + "playlist_item_id": result["id"], + "video_id": video_id, + "playlist_id": playlist_id, + "position": result["snippet"]["position"], + } + + logger.info( + f"Added video {video_id} to playlist {playlist_id} at position {item_data['position']}" + ) + return item_data + + except HttpError as e: + logger.error(f"YouTube add_video_to_playlist failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in add_video_to_playlist: {e}") + raise + + +def rate_video(access_token: str, video_id: str, rating: str) -> bool: + """ + Rate a YouTube video (like, dislike, or remove rating). + + Args: + access_token: Valid Google OAuth token with force-ssl scope + video_id: YouTube video ID + rating: Rating type ('like', 'dislike', or 'none' to remove rating) + + Returns: + True if rating was successful + + Raises: + HttpError: If YouTube API request fails + ValueError: If rating is not 'like', 'dislike', or 'none' + """ + if rating not in ["like", "dislike", "none"]: + raise ValueError(f"Invalid rating: {rating}. Must be 'like', 'dislike', or 'none'") + + try: + service = get_youtube_service(access_token) + + # Rate the video + service.videos().rate(id=video_id, rating=rating).execute() + + logger.info(f"Rated video {video_id} as '{rating}'") + return True + + except HttpError as e: + logger.error(f"YouTube rate_video failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in rate_video: {e}") + raise + + +def parse_atom_feed_entry(feed_entry: Dict) -> Dict: + """ + Parse YouTube PubSubHubbub Atom feed entry into structured data. + + Args: + feed_entry: Parsed Atom feed entry dict + + Returns: + Dict with video details (video_id, title, channel_id, published_at) + """ + # Extract data from feed entry + video_data = { + "video_id": feed_entry.get("yt:videoId", ""), + "title": feed_entry.get("title", ""), + "channel_id": feed_entry.get("yt:channelId", ""), + "channel_title": feed_entry.get("author", {}).get("name", ""), + "published_at": feed_entry.get("published", ""), + "updated_at": feed_entry.get("updated", ""), + "link": feed_entry.get("link", {}).get("@href", ""), + } + + logger.debug(f"Parsed Atom feed entry: {video_data}") + return video_data From 1c48adc6506b42486ce5ec7139cd941c362af5f5 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:48 +0100 Subject: [PATCH 09/68] feat(youtube): add YouTube service with actions and reactions for video monitoring and interaction --- .../management/commands/init_services.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/backend/automations/management/commands/init_services.py b/backend/automations/management/commands/init_services.py index 0e571bad..4e2b6f5f 100755 --- a/backend/automations/management/commands/init_services.py +++ b/backend/automations/management/commands/init_services.py @@ -1122,6 +1122,141 @@ def _create_services(self): }, ], }, + { + "name": "youtube", + "description": "YouTube video integration for monitoring and interacting with videos", + "status": Service.Status.ACTIVE, + "actions": [ + { + "name": "youtube_new_video", + "description": "Triggered when a new video is uploaded to a channel", + "config_schema": { + "channel_id": { + "type": "string", + "label": "Channel ID", + "description": "YouTube channel ID to monitor (e.g., UC_x5XG1OV2P6uZZ5FSM9Ttw)", + "required": True, + "placeholder": "UC_x5XG1OV2P6uZZ5FSM9Ttw", + }, + }, + }, + { + "name": "youtube_channel_stats", + "description": "Triggered when channel statistics change (subscribers, views)", + "config_schema": { + "channel_id": { + "type": "string", + "label": "Channel ID", + "description": "YouTube channel ID to monitor", + "required": True, + "placeholder": "UC_x5XG1OV2P6uZZ5FSM9Ttw", + }, + "threshold_type": { + "type": "string", + "label": "Threshold Type", + "description": "Type of metric to monitor", + "required": True, + "default": "subscribers", + "enum": ["subscribers", "views", "videos"], + }, + "threshold_value": { + "type": "number", + "label": "Threshold Value", + "description": "Trigger when metric exceeds this value", + "required": True, + "default": 1000, + "minimum": 0, + }, + }, + }, + { + "name": "youtube_search_videos", + "description": "Triggered when new videos matching search criteria are found", + "config_schema": { + "search_query": { + "type": "string", + "label": "Search Query", + "description": "Keywords to search for in video titles and descriptions", + "required": True, + "placeholder": "python tutorial", + }, + "channel_id": { + "type": "string", + "label": "Channel ID (Optional)", + "description": "Limit search to specific channel (optional)", + "required": False, + "placeholder": "UC_x5XG1OV2P6uZZ5FSM9Ttw", + }, + }, + }, + ], + "reactions": [ + { + "name": "youtube_post_comment", + "description": "Post a comment on a YouTube video", + "config_schema": { + "video_id": { + "type": "string", + "label": "Video ID", + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + "required": True, + "default": "{video_id}", + "placeholder": "{video_id} or dQw4w9WgXcQ", + }, + "comment_text": { + "type": "text", + "label": "Comment Text", + "description": "Text content of the comment (supports variables: {video_title}, {channel_name})", + "required": True, + "placeholder": "Great video! Thanks for sharing.", + }, + }, + }, + { + "name": "youtube_add_to_playlist", + "description": "Add a video to a YouTube playlist", + "config_schema": { + "video_id": { + "type": "string", + "label": "Video ID", + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + "required": True, + "default": "{video_id}", + "placeholder": "{video_id} or dQw4w9WgXcQ", + }, + "playlist_id": { + "type": "string", + "label": "Playlist ID", + "description": "ID of the playlist to add video to", + "required": True, + "placeholder": "PLrAXtmErZgOeiKm4sgNOknGvNjby9efdf", + }, + }, + }, + { + "name": "youtube_rate_video", + "description": "Like or dislike a YouTube video", + "config_schema": { + "video_id": { + "type": "string", + "label": "Video ID", + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + "required": True, + "default": "{video_id}", + "placeholder": "{video_id} or dQw4w9WgXcQ", + }, + "rating": { + "type": "string", + "label": "Rating", + "description": "Like, dislike, or remove rating", + "required": True, + "default": "like", + "enum": ["like", "dislike", "none"], + }, + }, + }, + ], + }, ] # Create services From fa17db779619410312abc49538279234c1589daa Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:53 +0100 Subject: [PATCH 10/68] feat(youtube): add YouTube to OAuth service name resolution --- frontend/src/pages/ServiceDetail.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/ServiceDetail.tsx b/frontend/src/pages/ServiceDetail.tsx index 09f89a2e..c6df636f 100755 --- a/frontend/src/pages/ServiceDetail.tsx +++ b/frontend/src/pages/ServiceDetail.tsx @@ -165,7 +165,7 @@ const ServiceDetail: React.FC = () => { // For gmail and google_calendar, check if 'google' OAuth is connected const getOAuthServiceName = (serviceName: string) => { const lower = serviceName.toLowerCase(); - if (lower === 'gmail' || lower === 'google_calendar') { + if (lower === 'gmail' || lower === 'google_calendar' || lower === 'youtube') { return 'google'; } return lower; From 8707dfbb9e484a5de205beab05e420be510f1e6a Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:57:57 +0100 Subject: [PATCH 11/68] feat(youtube): add YouTube to token mapping for OAuth service resolution --- mobile/lib/utils/service_token_mapper.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/mobile/lib/utils/service_token_mapper.dart b/mobile/lib/utils/service_token_mapper.dart index cc0c315e..1287b8b9 100644 --- a/mobile/lib/utils/service_token_mapper.dart +++ b/mobile/lib/utils/service_token_mapper.dart @@ -8,6 +8,7 @@ class ServiceTokenMapper { static const Map _tokenMap = { 'gmail': 'google', 'google_calendar': 'google', + 'youtube': 'google', }; /// Get the actual token service name to query in the database From 04e07317196239dfaac5e23112526d7f5c8dbf19 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:58:23 +0100 Subject: [PATCH 12/68] chore(env): clean up comment formatting for webhook secrets section --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 28491b42..179f50b2 100644 --- a/.env.example +++ b/.env.example @@ -150,7 +150,7 @@ NOTION_REDIRECT_URI=http://localhost:8080/auth/oauth/notion/callback/ # JSON dictionary format for webhook secrets # Generate secure random strings: openssl rand -base64 32 # Format: WEBHOOK_SECRETS='{"service_name":"secret_key",...}' -# +# # To enable webhooks for a service: # 1. Generate a secure secret: openssl rand -base64 32 # 2. Add it to this JSON dictionary From e4a3d0fa4d0bd656b1deff168f9d20d4b43042e8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:58:28 +0100 Subject: [PATCH 13/68] fix(validators): remove unnecessary whitespace in error handling for reaction validation --- backend/automations/validators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/automations/validators.py b/backend/automations/validators.py index c5cd3084..bcd1bf9f 100755 --- a/backend/automations/validators.py +++ b/backend/automations/validators.py @@ -1174,7 +1174,7 @@ def validate_reaction_config(reaction_name, config): except JsonSchemaValidationError as e: # Provide helpful context for common mistakes error_message = f"Invalid configuration for reaction '{reaction_name}': {e.message}" - + # Special handling for Gmail reactions that need message_id if reaction_name in ["gmail_mark_read", "gmail_add_label"]: if "message_id" in e.message or "email_id" in e.message: @@ -1185,7 +1185,7 @@ def validate_reaction_config(reaction_name, config): "Please use a Gmail trigger action instead of the current trigger. " f"Technical details: {e.message}" ) - + raise serializers.ValidationError(error_message) @@ -1229,7 +1229,7 @@ def validate_action_reaction_compatibility(action_name, reaction_name): " Note: This reaction works with most triggers, but the issue content " "will depend on the data provided by the trigger." ) - + raise serializers.ValidationError( f"Action '{action_name}' cannot trigger reaction '{reaction_name}'.{context_message} " f"Compatible reactions for this action: {', '.join(compatible_reactions) if compatible_reactions else 'none available'}" From 966936f40f62fe7b12eb3fffbb1e9dde57087ee8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:58:33 +0100 Subject: [PATCH 14/68] fix(webhooks): remove unnecessary whitespace in extract_event_id and webhook_receiver functions --- backend/automations/webhooks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/automations/webhooks.py b/backend/automations/webhooks.py index e172e186..c5f8bb52 100755 --- a/backend/automations/webhooks.py +++ b/backend/automations/webhooks.py @@ -292,11 +292,11 @@ def extract_event_id( entry = event_data.get("entry", {}) video_id = entry.get("yt:videoId") or entry.get("video_id") published = entry.get("published", "") - + if video_id: # Include published timestamp for uniqueness return f"youtube_video_{video_id}_{published}" - + # Fallback: use link if available link = entry.get("link", {}).get("href", "") if link and "watch?v=" in link: @@ -536,7 +536,7 @@ def webhook_receiver(request: Request, service: str) -> Response: hub_mode = request.GET.get("hub.mode") hub_challenge = request.GET.get("hub.challenge") hub_topic = request.GET.get("hub.topic") - + if hub_mode == "subscribe" and hub_challenge: logger.info(f"✅ YouTube PubSubHubbub subscription verification for topic: {hub_topic}") # Return challenge as plain text From ae46258ab386c2404f6cda79b450fb2e12fe26d7 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 03:58:37 +0100 Subject: [PATCH 15/68] fix(youtube_helper): remove unnecessary whitespace in function implementations --- backend/automations/helpers/youtube_helper.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py index 17aed56b..cd1b8073 100644 --- a/backend/automations/helpers/youtube_helper.py +++ b/backend/automations/helpers/youtube_helper.py @@ -54,7 +54,7 @@ def get_latest_videos( """ try: service = get_youtube_service(access_token) - + # Build search parameters search_params = { "part": "id,snippet", @@ -63,7 +63,7 @@ def get_latest_videos( "order": "date", "maxResults": min(max_results, 50), } - + if published_after: search_params["publishedAfter"] = published_after @@ -116,7 +116,7 @@ def get_channel_statistics(access_token: str, channel_id: str) -> Dict: """ try: service = get_youtube_service(access_token) - + results = ( service.channels() .list(part="statistics", id=channel_id) @@ -170,7 +170,7 @@ def search_videos( """ try: service = get_youtube_service(access_token) - + # Build search parameters search_params = { "part": "id,snippet", @@ -179,7 +179,7 @@ def search_videos( "order": "relevance", "maxResults": min(max_results, 50), } - + if channel_id: search_params["channelId"] = channel_id if published_after: @@ -231,7 +231,7 @@ def post_comment( """ try: service = get_youtube_service(access_token) - + # Build comment resource comment_resource = { "snippet": { @@ -288,7 +288,7 @@ def add_video_to_playlist( """ try: service = get_youtube_service(access_token) - + # Build playlist item resource playlist_item = { "snippet": { @@ -348,7 +348,7 @@ def rate_video(access_token: str, video_id: str, rating: str) -> bool: try: service = get_youtube_service(access_token) - + # Rate the video service.videos().rate(id=video_id, rating=rating).execute() From fb0a6c5395720ce0b561d0721b78efc6d1bbf4b8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 04:27:51 +0100 Subject: [PATCH 16/68] feat(calendar): add Google Calendar polling task for event actions --- backend/automations/tasks.py | 242 +++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index 6f0e2b38..e9d3f0ee 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -798,6 +798,248 @@ def check_gmail_actions(self): raise self.retry(exc=exc, countdown=300) from None +@shared_task( + name="automations.check_google_calendar_actions", + bind=True, + max_retries=3, + autoretry_for=RECOVERABLE_EXCEPTIONS, +) +def check_google_calendar_actions(self): + """ + Poll Google Calendar for events matching user's action criteria. + + Checks all active Areas with Calendar actions and triggers executions + when matching events are found. + + Supported actions: + - calendar_new_event: New event created + - calendar_event_starting_soon: Event starting within X minutes + + Returns: + dict: Summary of polling results + """ + from datetime import datetime, timedelta + + from users.oauth.manager import OAuthManager + + from .helpers.calendar_helper import list_upcoming_events + + logger.info("Checking Google Calendar actions...") + + try: + # Get all active Areas with Calendar actions + calendar_areas = get_active_areas( + [ + "calendar_new_event", + "calendar_event_starting_soon", + ] + ) + + if not calendar_areas: + logger.info("No active Calendar areas found") + return {"status": "no_areas", "checked": 0} + + triggered_count = 0 + skipped_count = 0 + no_token_count = 0 + + for area in calendar_areas: + try: + # Get valid Google token + access_token = OAuthManager.get_valid_token(area.owner, "google") + + if not access_token: + logger.warning( + f"No valid Google token for user {area.owner.username}, " + f"area '{area.name}'" + ) + no_token_count += 1 + continue + + action_name = area.action.name + action_config = area.action_config or {} + + # Get last checked state + state, _ = ActionState.objects.get_or_create(area=area) + + # ===== CALENDAR NEW EVENT ===== + if action_name == "calendar_new_event": + # Fetch events created since last check (or last 1 hour) + time_min = state.last_checked_at or ( + timezone.now() - timedelta(hours=1) + ) + time_min_str = time_min.isoformat() + + events = list_upcoming_events( + access_token, max_results=10, time_min=time_min_str + ) + + # Filter for events created recently (check created timestamp) + for event in events: + event_id = event.get("id") + created_time = event.get("created") + + if not event_id or not created_time: + continue + + # Parse created timestamp + created_dt = datetime.fromisoformat( + created_time.replace("Z", "+00:00") + ) + + # Only trigger for events created after last check + if state.last_checked_at and created_dt <= state.last_checked_at: + continue + + # Create unique event ID + event_external_id = f"calendar_new_event_{event_id}_{area.pk}" + + # Prepare trigger data + start_time = event.get("start", {}).get( + "dateTime", event.get("start", {}).get("date", "") + ) + end_time = event.get("end", {}).get( + "dateTime", event.get("end", {}).get("date", "") + ) + + trigger_data = { + "service": "google_calendar", + "action": action_name, + "event_id": event_id, + "event_title": event.get("summary", "Untitled"), + "event_description": event.get("description", ""), + "event_location": event.get("location", ""), + "start_time": start_time, + "end_time": end_time, + "created": created_time, + "organizer": event.get("organizer", {}).get("email", ""), + "attendees": [ + attendee.get("email", "") + for attendee in event.get("attendees", []) + ], + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_external_id, + trigger_data=trigger_data, + ) + + if created and execution: + logger.info( + f"Calendar new_event triggered for area '{area.name}': " + f"{trigger_data['event_title']}" + ) + execute_reaction_task.delay(execution.pk) + triggered_count += 1 + + # ===== CALENDAR EVENT STARTING SOON ===== + elif action_name == "calendar_event_starting_soon": + minutes_before = int(action_config.get("minutes_before", 15)) + + # Calculate time window + now = timezone.now() + target_time_min = now + target_time_max = now + timedelta(minutes=minutes_before + 5) + + # Fetch upcoming events + events = list_upcoming_events( + access_token, + max_results=20, + time_min=target_time_min.isoformat(), + ) + + for event in events: + event_id = event.get("id") + start = event.get("start", {}) + start_time_str = start.get("dateTime") or start.get("date") + + if not event_id or not start_time_str: + continue + + # Parse start time + if "T" in start_time_str: # DateTime + start_dt = datetime.fromisoformat( + start_time_str.replace("Z", "+00:00") + ) + else: # All-day event (date only) + start_dt = datetime.fromisoformat( + start_time_str + "T00:00:00+00:00" + ) + + # Calculate minutes until event + time_until_event = (start_dt - now).total_seconds() / 60 + + # Check if within notification window + if 0 <= time_until_event <= minutes_before: + # Create unique event ID (include timestamp to avoid duplicates) + event_external_id = ( + f"calendar_event_starting_soon_{event_id}_" + f"{minutes_before}m_{area.pk}" + ) + + # Prepare trigger data + end = event.get("end", {}) + end_time = end.get("dateTime") or end.get("date", "") + + trigger_data = { + "service": "google_calendar", + "action": action_name, + "event_id": event_id, + "event_title": event.get("summary", "Untitled"), + "event_description": event.get("description", ""), + "event_location": event.get("location", ""), + "start_time": start_time_str, + "end_time": end_time, + "minutes_until_start": int(time_until_event), + "minutes_before_trigger": minutes_before, + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_external_id, + trigger_data=trigger_data, + ) + + if created and execution: + logger.info( + f"Calendar event_starting_soon triggered for area '{area.name}': " + f"{trigger_data['event_title']} " + f"(in {int(time_until_event)} minutes)" + ) + execute_reaction_task.delay(execution.pk) + triggered_count += 1 + + # Update state + state.last_checked_at = timezone.now() + state.save() + + except Exception as e: + logger.error( + f"Error checking Calendar for area '{area.name}': {e}", + exc_info=True, + ) + skipped_count += 1 + continue + + logger.info( + f"Calendar check complete: {triggered_count} triggered, " + f"{skipped_count} skipped, {no_token_count} no token" + ) + + return { + "status": "success", + "triggered": triggered_count, + "skipped": skipped_count, + "no_token": no_token_count, + "checked_areas": len(calendar_areas), + } + + except Exception as exc: + logger.error(f"Error in check_google_calendar_actions: {exc}", exc_info=True) + raise self.retry(exc=exc, countdown=300) from None + + @shared_task( name="automations.check_weather_actions", bind=True, From 26221dfdcff624cb7020a3c31589f0a754b96811 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 04:27:59 +0100 Subject: [PATCH 17/68] feat(celery): add periodic task for Google Calendar actions --- backend/area_project/celery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index 60c96aa6..f0ee4d00 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -59,6 +59,10 @@ def get_beat_schedule(): "task": "automations.check_gmail_actions", "schedule": 180.0, # Every 3 minutes (no webhook support yet) }, + "check-google-calendar-actions": { + "task": "automations.check_google_calendar_actions", + "schedule": 180.0, # Every 3 minutes (no webhook support) + }, "check-youtube-actions": { "task": "automations.check_youtube_actions", "schedule": 300.0, # Every 5 minutes (polling + webhook support) From fa3da78ef950d0509c930486d71195d27c595119 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 05:03:15 +0100 Subject: [PATCH 18/68] fix(debug): remove unnecessary text in external events trigger message --- frontend/src/pages/Debug.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/Debug.tsx b/frontend/src/pages/Debug.tsx index adb9b5df..949ac71b 100644 --- a/frontend/src/pages/Debug.tsx +++ b/frontend/src/pages/Debug.tsx @@ -314,7 +314,7 @@ const Debug: React.FC = () => { ) : (
- Triggered by external events (webhooks) + Triggered by external events
)} From 10ca16345419e3698545fc3a58f0f13c74eed956 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 05:08:04 +0100 Subject: [PATCH 19/68] feat(youtube): add schemas for YouTube actions and reactions --- backend/automations/validators.py | 135 ++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/backend/automations/validators.py b/backend/automations/validators.py index bcd1bf9f..deb5c533 100755 --- a/backend/automations/validators.py +++ b/backend/automations/validators.py @@ -369,6 +369,57 @@ def validate_email_format(email): "required": ["database_id"], "additionalProperties": False, }, + # YouTube Actions + "youtube_new_video": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "minLength": 1, + "description": "YouTube channel ID to monitor", + }, + }, + "required": ["channel_id"], + "additionalProperties": False, + }, + "youtube_channel_stats": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "minLength": 1, + "description": "YouTube channel ID to monitor", + }, + "threshold_type": { + "type": "string", + "enum": ["subscribers", "views", "videos"], + "description": "Type of metric to monitor", + }, + "threshold_value": { + "type": "number", + "minimum": 0, + "description": "Trigger when metric exceeds this value", + }, + }, + "required": ["channel_id", "threshold_type", "threshold_value"], + "additionalProperties": False, + }, + "youtube_search_videos": { + "type": "object", + "properties": { + "search_query": { + "type": "string", + "minLength": 1, + "description": "Keywords to search for", + }, + "channel_id": { + "type": "string", + "description": "Limit search to specific channel (optional)", + }, + }, + "required": ["search_query"], + "additionalProperties": False, + }, } @@ -880,6 +931,59 @@ def validate_email_format(email): "required": ["database_id", "item_name"], "additionalProperties": False, }, + # YouTube Reactions + "youtube_post_comment": { + "type": "object", + "properties": { + "video_id": { + "type": "string", + "minLength": 1, + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + }, + "comment_text": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "Comment text (supports variables: {video_title}, {channel_name})", + }, + }, + "required": ["video_id", "comment_text"], + "additionalProperties": False, + }, + "youtube_add_to_playlist": { + "type": "object", + "properties": { + "video_id": { + "type": "string", + "minLength": 1, + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + }, + "playlist_id": { + "type": "string", + "minLength": 1, + "description": "ID of the playlist to add video to", + }, + }, + "required": ["video_id", "playlist_id"], + "additionalProperties": False, + }, + "youtube_rate_video": { + "type": "object", + "properties": { + "video_id": { + "type": "string", + "minLength": 1, + "description": "YouTube video ID (automatically provided by trigger or enter manually)", + }, + "rating": { + "type": "string", + "enum": ["like", "dislike", "none"], + "description": "Like, dislike, or remove rating", + }, + }, + "required": ["video_id", "rating"], + "additionalProperties": False, + }, } @@ -1104,6 +1208,37 @@ def validate_email_format(email): "notion_update_page", "notion_create_database_item", ], + # YouTube actions - can trigger YouTube reactions and notification reactions + "youtube_new_video": [ + "youtube_post_comment", + "youtube_add_to_playlist", + "youtube_rate_video", + "send_email", + "gmail_send_email", + "slack_message", + "webhook_post", + "calendar_create_event", + "github_create_issue", + ], + "youtube_channel_stats": [ + "send_email", + "gmail_send_email", + "slack_message", + "webhook_post", + "calendar_create_event", + "github_create_issue", + ], + "youtube_search_videos": [ + "youtube_post_comment", + "youtube_add_to_playlist", + "youtube_rate_video", + "send_email", + "gmail_send_email", + "slack_message", + "webhook_post", + "calendar_create_event", + "github_create_issue", + ], } From fadfe49f92d2505c24d9396d2b29892e49d89ae5 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 05:13:37 +0100 Subject: [PATCH 20/68] feat(youtube): update reaction schemas to include default values and adjust required fields --- backend/automations/validators.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/automations/validators.py b/backend/automations/validators.py index deb5c533..ba76c2cf 100755 --- a/backend/automations/validators.py +++ b/backend/automations/validators.py @@ -938,6 +938,7 @@ def validate_email_format(email): "video_id": { "type": "string", "minLength": 1, + "default": "{video_id}", "description": "YouTube video ID (automatically provided by trigger or enter manually)", }, "comment_text": { @@ -947,7 +948,7 @@ def validate_email_format(email): "description": "Comment text (supports variables: {video_title}, {channel_name})", }, }, - "required": ["video_id", "comment_text"], + "required": ["comment_text"], "additionalProperties": False, }, "youtube_add_to_playlist": { @@ -956,6 +957,7 @@ def validate_email_format(email): "video_id": { "type": "string", "minLength": 1, + "default": "{video_id}", "description": "YouTube video ID (automatically provided by trigger or enter manually)", }, "playlist_id": { @@ -964,7 +966,7 @@ def validate_email_format(email): "description": "ID of the playlist to add video to", }, }, - "required": ["video_id", "playlist_id"], + "required": ["playlist_id"], "additionalProperties": False, }, "youtube_rate_video": { @@ -973,15 +975,17 @@ def validate_email_format(email): "video_id": { "type": "string", "minLength": 1, + "default": "{video_id}", "description": "YouTube video ID (automatically provided by trigger or enter manually)", }, "rating": { "type": "string", "enum": ["like", "dislike", "none"], + "default": "like", "description": "Like, dislike, or remove rating", }, }, - "required": ["video_id", "rating"], + "required": [], "additionalProperties": False, }, } From 29a88d1ec0f28bf5ad3560e42a6e3ea8ab22c7cd Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 05:21:17 +0100 Subject: [PATCH 21/68] feat(youtube): update check_youtube_actions to track last checked time and adjust published_after logic --- backend/automations/tasks.py | 48 +++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index e9d3f0ee..242f086d 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -878,7 +878,7 @@ def check_google_calendar_actions(self): for event in events: event_id = event.get("id") created_time = event.get("created") - + if not event_id or not created_time: continue @@ -3958,18 +3958,31 @@ def check_youtube_actions(self): skipped_count += 1 continue - # Get published_after from updated_at minus 1 hour or None + # Get published_after from last check or 24 hours ago + from datetime import timedelta + + from django.utils import timezone + + action_state = get_or_create_action_state(area) published_after = None - if area.updated_at: - from datetime import timedelta - one_hour_ago = area.updated_at - timedelta(hours=1) - published_after = one_hour_ago.isoformat() + + if action_state.last_checked_at: + # Check videos published after last check + published_after = action_state.last_checked_at.isoformat() + else: + # First check: only get videos from last 24 hours + one_day_ago = timezone.now() - timedelta(hours=24) + published_after = one_day_ago.isoformat() # Fetch latest videos videos = get_latest_videos( access_token, channel_id, max_results=5, published_after=published_after ) + # Update last checked time + action_state.last_checked_at = timezone.now() + action_state.save() + # Create execution for each new video for video in videos: event_id = f"youtube_new_video_{video['video_id']}_{area.pk}" @@ -4066,12 +4079,21 @@ def check_youtube_actions(self): skipped_count += 1 continue - # Get published_after from updated_at minus 1 hour or None + # Get published_after from last check or 24 hours ago + from datetime import timedelta + + from django.utils import timezone + + action_state = get_or_create_action_state(area) published_after = None - if area.updated_at: - from datetime import timedelta - one_hour_ago = area.updated_at - timedelta(hours=1) - published_after = one_hour_ago.isoformat() + + if action_state.last_checked_at: + # Check videos published after last check + published_after = action_state.last_checked_at.isoformat() + else: + # First check: only get videos from last 24 hours + one_day_ago = timezone.now() - timedelta(hours=24) + published_after = one_day_ago.isoformat() # Search for videos videos = search_videos( @@ -4081,6 +4103,10 @@ def check_youtube_actions(self): channel_id=channel_id, published_after=published_after, ) + + # Update last checked time + action_state.last_checked_at = timezone.now() + action_state.save() # Create execution for each matching video for video in videos: From 8c00e6096e31102bc41f03873d92cfb15c778ded Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 05:38:33 +0100 Subject: [PATCH 22/68] fix(youtube): replace get_or_create_action_state with ActionState's get_or_create method --- backend/automations/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index 242f086d..c99b93bb 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -3963,7 +3963,7 @@ def check_youtube_actions(self): from django.utils import timezone - action_state = get_or_create_action_state(area) + action_state, _ = ActionState.objects.get_or_create(area=area) published_after = None if action_state.last_checked_at: @@ -4084,7 +4084,7 @@ def check_youtube_actions(self): from django.utils import timezone - action_state = get_or_create_action_state(area) + action_state, _ = ActionState.objects.get_or_create(area=area) published_after = None if action_state.last_checked_at: From ff2ce422e3264544420d2ff280c4bd1e2e6cdac9 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 06:25:15 +0100 Subject: [PATCH 23/68] feat: Implement Google Webhook (Push Notification) Management - Added GoogleWebhookWatch model to track active push notification subscriptions for Gmail, Calendar, and YouTube. - Created views for handling incoming push notifications from Google services. - Implemented tasks for setting up and renewing Google watches. - Updated celery beat schedule to include tasks for renewing Google watches and setting up YouTube subscriptions. - Added helper functions for managing Google push notifications. - Created migrations for the new GoogleWebhookWatch model. - Updated URLs to include endpoints for Google webhooks. --- backend/area_project/celery.py | 15 +- backend/automations/google_webhook_views.py | 451 ++++++++++++++++++ backend/automations/helpers/gmail_helper.py | 43 ++ .../helpers/google_webhook_helper.py | 294 ++++++++++++ .../migrations/0012_google_webhook_watch.py | 38 ++ backend/automations/models.py | 107 +++++ backend/automations/tasks.py | 348 +++++++++++++- backend/automations/urls.py | 7 + 8 files changed, 1297 insertions(+), 6 deletions(-) create mode 100644 backend/automations/google_webhook_views.py create mode 100644 backend/automations/helpers/google_webhook_helper.py create mode 100644 backend/automations/migrations/0012_google_webhook_watch.py diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index f0ee4d00..c50e19fa 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -57,20 +57,29 @@ def get_beat_schedule(): }, "check-gmail-actions": { "task": "automations.check_gmail_actions", - "schedule": 180.0, # Every 3 minutes (no webhook support yet) + "schedule": 180.0, # Every 3 minutes (polling fallback) }, "check-google-calendar-actions": { "task": "automations.check_google_calendar_actions", - "schedule": 180.0, # Every 3 minutes (no webhook support) + "schedule": 180.0, # Every 3 minutes (polling fallback) }, "check-youtube-actions": { "task": "automations.check_youtube_actions", - "schedule": 300.0, # Every 5 minutes (polling + webhook support) + "schedule": 300.0, # Every 5 minutes (polling fallback) }, "check-weather-actions": { "task": "automations.check_weather_actions", "schedule": 300.0, # Every 5 minutes (no webhook support) }, + # Google Push Notification (Watch) Management + "renew-google-watches": { + "task": "automations.renew_google_watches", + "schedule": 3600.0, # Every hour - auto-renew expiring watches + }, + "setup-youtube-watches": { + "task": "automations.setup_youtube_watches", + "schedule": 7200.0, # Every 2 hours - ensure YouTube subscriptions active + }, "collect-execution-metrics": { "task": "automations.collect_execution_metrics", "schedule": crontab(minute=0), # Every hour diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py new file mode 100644 index 00000000..a81c5373 --- /dev/null +++ b/backend/automations/google_webhook_views.py @@ -0,0 +1,451 @@ +""" +Google Webhook (Push Notification) Receivers. + +This module handles incoming push notifications from Google services: +- Gmail: History notifications when emails change +- Calendar: Event notifications when calendar events change +- YouTube: PubSubHubbub notifications for new videos + +Google uses different notification mechanisms: +- Gmail/Calendar: Push API with channel_id verification +- YouTube: PubSubHubbub with hub.challenge verification +""" + +import base64 +import json +import logging +from xml.etree import ElementTree as ET + +from django.http import HttpResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from rest_framework import status +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import AllowAny +from rest_framework.response import Response + +from .helpers.calendar_helper import list_upcoming_events +from .helpers.gmail_helper import get_history, get_message_details +from .helpers.youtube_helper import parse_atom_feed_entry +from .models import Area, GoogleWebhookWatch +from .tasks import create_execution_safe, execute_reaction_task + +logger = logging.getLogger(__name__) + + +@csrf_exempt +@require_http_methods(["GET", "POST"]) +def gmail_webhook(request): + """ + Handle Gmail push notifications. + + GET: Domain verification (returns 200 OK) + POST: Receive Gmail history notifications + + Gmail sends notifications when the mailbox changes (new emails, labels, etc.) + We need to: + 1. Verify the channel_id matches an active watch + 2. Get the history_id from the notification + 3. Fetch actual changes using Gmail History API + 4. Trigger relevant Areas + """ + if request.method == "GET": + # Domain verification + logger.info("Gmail webhook GET request (domain verification)") + return HttpResponse("OK", status=200) + + try: + # Parse notification headers + channel_id = request.headers.get("X-Goog-Channel-ID") + resource_state = request.headers.get("X-Goog-Resource-State") + resource_id = request.headers.get("X-Goog-Resource-ID") + + if not channel_id: + logger.warning("Gmail webhook: missing X-Goog-Channel-ID header") + return HttpResponse("Missing channel ID", status=400) + + logger.info( + f"Gmail webhook received: channel={channel_id}, " + f"state={resource_state}, resource={resource_id}" + ) + + # Find the watch + try: + watch = GoogleWebhookWatch.objects.get( + channel_id=channel_id, service=GoogleWebhookWatch.Service.GMAIL + ) + except GoogleWebhookWatch.DoesNotExist: + logger.warning(f"Gmail webhook: unknown channel_id {channel_id}") + return HttpResponse("Unknown channel", status=404) + + # Record event + watch.record_event() + + # Skip sync state (initial handshake) + if resource_state == "sync": + logger.info(f"Gmail webhook: sync state for channel {channel_id}") + return HttpResponse("OK", status=200) + + # Parse notification body (contains historyId) + try: + body = json.loads(request.body.decode("utf-8")) + history_id = body.get("historyId") + except Exception as e: + logger.error(f"Failed to parse Gmail notification body: {e}") + history_id = None + + if not history_id: + logger.warning("Gmail webhook: no historyId in notification") + return HttpResponse("OK", status=200) + + # Fetch user's OAuth token + from users.oauth.manager import OAuthManager + + access_token = OAuthManager.get_valid_token(watch.user, "google") + + if not access_token: + logger.warning( + f"Gmail webhook: no valid token for user {watch.user.username}" + ) + return HttpResponse("No token", status=200) + + # Fetch history changes + try: + history = get_history(access_token, start_history_id=history_id) + + if not history: + logger.debug(f"No Gmail history changes for user {watch.user.username}") + return HttpResponse("OK", status=200) + + # Process history changes + for history_item in history: + messages_added = history_item.get("messagesAdded", []) + + for msg_info in messages_added: + message_id = msg_info.get("message", {}).get("id") + labels = msg_info.get("message", {}).get("labelIds", []) + + # Only process INBOX messages + if "INBOX" not in labels: + continue + + # Get message details + details = get_message_details(access_token, message_id) + + # Find matching Areas + gmail_areas = Area.objects.filter( + owner=watch.user, + status=Area.Status.ACTIVE, + action__name__in=[ + "gmail_new_email", + "gmail_new_from_sender", + "gmail_new_with_label", + "gmail_new_with_subject", + ], + ).select_related("action", "reaction") + + for area in gmail_areas: + # Check if message matches action criteria + if not _gmail_message_matches_action(area, details): + continue + + # Create execution + event_id = f"gmail_{message_id}" + trigger_data = { + "service": "gmail", + "action": area.action.name, + "message_id": message_id, + "subject": details["subject"], + "from": details["from"], + "to": details["to"], + "date": details["date"], + "snippet": details["snippet"], + "labels": details["labels"], + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_id, + trigger_data=trigger_data, + ) + + if created and execution: + logger.info( + f"Gmail webhook triggered area '{area.name}': " + f"Message from {details['from']}" + ) + execute_reaction_task.delay(execution.pk) + + except Exception as e: + logger.error(f"Error processing Gmail history: {e}", exc_info=True) + + return HttpResponse("OK", status=200) + + except Exception as e: + logger.error(f"Gmail webhook error: {e}", exc_info=True) + return HttpResponse("Internal error", status=500) + + +@csrf_exempt +@require_http_methods(["GET", "POST"]) +def calendar_webhook(request): + """ + Handle Google Calendar push notifications. + + GET: Domain verification + POST: Receive Calendar change notifications + + Calendar sends notifications when events change (created, updated, deleted). + """ + if request.method == "GET": + # Domain verification + logger.info("Calendar webhook GET request (domain verification)") + return HttpResponse("OK", status=200) + + try: + # Parse notification headers + channel_id = request.headers.get("X-Goog-Channel-ID") + resource_state = request.headers.get("X-Goog-Resource-State") + resource_id = request.headers.get("X-Goog-Resource-ID") + + if not channel_id: + logger.warning("Calendar webhook: missing X-Goog-Channel-ID header") + return HttpResponse("Missing channel ID", status=400) + + logger.info( + f"Calendar webhook received: channel={channel_id}, " + f"state={resource_state}, resource={resource_id}" + ) + + # Find the watch + try: + watch = GoogleWebhookWatch.objects.get( + channel_id=channel_id, service=GoogleWebhookWatch.Service.CALENDAR + ) + except GoogleWebhookWatch.DoesNotExist: + logger.warning(f"Calendar webhook: unknown channel_id {channel_id}") + return HttpResponse("Unknown channel", status=404) + + # Record event + watch.record_event() + + # Skip sync state + if resource_state == "sync": + logger.info(f"Calendar webhook: sync state for channel {channel_id}") + return HttpResponse("OK", status=200) + + # Fetch user's OAuth token + from users.oauth.manager import OAuthManager + + access_token = OAuthManager.get_valid_token(watch.user, "google") + + if not access_token: + logger.warning( + f"Calendar webhook: no valid token for user {watch.user.username}" + ) + return HttpResponse("No token", status=200) + + # Fetch recent calendar events + try: + calendar_id = watch.resource_uri or "primary" + events = list_upcoming_events(access_token, calendar_id=calendar_id, max_results=10) + + if not events: + logger.debug(f"No calendar events for user {watch.user.username}") + return HttpResponse("OK", status=200) + + # Find matching Areas + calendar_areas = Area.objects.filter( + owner=watch.user, + status=Area.Status.ACTIVE, + action__name__in=[ + "calendar_new_event", + "calendar_event_starting_soon", + ], + ).select_related("action", "reaction") + + for area in calendar_areas: + action_name = area.action.name + + if action_name == "calendar_new_event": + # Check for newly created events (created in last 5 minutes) + from datetime import timedelta + + from django.utils import timezone + + recent_threshold = timezone.now() - timedelta(minutes=5) + + for event in events: + try: + from dateutil import parser + + created_str = event.get("created", "") + if not created_str: + continue + + created_dt = parser.isoparse(created_str) + + if created_dt >= recent_threshold: + event_id = f"calendar_new_event_{event['id']}" + + trigger_data = { + "action": "calendar_new_event", + "service": "google_calendar", + "event_id": event["id"], + "event_title": event.get("summary", ""), + "event_description": event.get("description", ""), + "event_location": event.get("location", ""), + "start_time": event.get("start", {}).get("dateTime", ""), + "end_time": event.get("end", {}).get("dateTime", ""), + "attendees": [ + a.get("email", "") + for a in event.get("attendees", []) + ], + "organizer": event.get("organizer", {}).get("email", ""), + "created": created_str, + } + + execution, created = create_execution_safe( + area=area, + external_event_id=event_id, + trigger_data=trigger_data, + ) + + if created and execution: + logger.info( + f"Calendar webhook triggered area '{area.name}': " + f"New event '{event.get('summary')}'" + ) + execute_reaction_task.delay(execution.pk) + + except Exception as e: + logger.error( + f"Error processing calendar event {event.get('id')}: {e}" + ) + + except Exception as e: + logger.error(f"Error processing calendar changes: {e}", exc_info=True) + + return HttpResponse("OK", status=200) + + except Exception as e: + logger.error(f"Calendar webhook error: {e}", exc_info=True) + return HttpResponse("Internal error", status=500) + + +@csrf_exempt +@require_http_methods(["GET", "POST"]) +def youtube_webhook(request): + """ + Handle YouTube PubSubHubbub notifications. + + GET: Hub verification (must respond with hub.challenge) + POST: Receive new video notifications (Atom feed XML) + """ + if request.method == "GET": + # Hub verification + challenge = request.GET.get("hub.challenge") + if challenge: + logger.info(f"YouTube webhook verification: responding with challenge") + return HttpResponse(challenge, content_type="text/plain", status=200) + return HttpResponse("OK", status=200) + + try: + # Parse Atom feed XML + body = request.body.decode("utf-8") + video_data = parse_atom_feed_entry(body) + + if not video_data: + logger.warning("YouTube webhook: failed to parse feed") + return HttpResponse("Invalid feed", status=400) + + video_id = video_data["video_id"] + channel_id = video_data["channel_id"] + + logger.info( + f"YouTube webhook: new video {video_id} from channel {channel_id}" + ) + + # Find users watching this channel + youtube_areas = Area.objects.filter( + status=Area.Status.ACTIVE, + action__name="youtube_new_video", + action_config__channel_id=channel_id, + ).select_related("owner", "action", "reaction") + + for area in youtube_areas: + event_id = f"youtube_new_video_{video_id}_{area.pk}" + + trigger_data = { + "video_id": video_id, + "video_title": video_data["title"], + "video_description": video_data.get("description", ""), + "channel_id": channel_id, + "channel_name": video_data["channel_title"], + "published_at": video_data["published_at"], + "thumbnail_url": video_data.get("thumbnail_url", ""), + } + + execution, created = create_execution_safe( + area=area, external_event_id=event_id, trigger_data=trigger_data + ) + + if created and execution: + logger.info( + f"YouTube webhook triggered area '{area.name}': " + f"New video '{video_data['title']}'" + ) + execute_reaction_task.delay(execution.pk) + + return HttpResponse("OK", status=200) + + except Exception as e: + logger.error(f"YouTube webhook error: {e}", exc_info=True) + return HttpResponse("Internal error", status=500) + + +def _gmail_message_matches_action(area, message_details): + """ + Check if a Gmail message matches the Area's action criteria. + + Args: + area: Area instance + message_details: Dict with message details (from, subject, labels, etc.) + + Returns: + bool: True if message matches action config + """ + action_name = area.action.name + action_config = area.action_config or {} + + # gmail_new_email: matches any email + if action_name == "gmail_new_email": + # Optional filters + from_email = action_config.get("from_email") + subject_contains = action_config.get("subject_contains") + + if from_email and from_email.lower() not in message_details["from"].lower(): + return False + + if subject_contains and subject_contains.lower() not in message_details["subject"].lower(): + return False + + return True + + # gmail_new_from_sender: specific sender + elif action_name == "gmail_new_from_sender": + from_email = action_config.get("from_email", "").lower() + return from_email in message_details["from"].lower() + + # gmail_new_with_label: specific label + elif action_name == "gmail_new_with_label": + required_label = action_config.get("label", "").lower() + message_labels = [l.lower() for l in message_details.get("labels", [])] + return required_label in message_labels + + # gmail_new_with_subject: subject contains text + elif action_name == "gmail_new_with_subject": + subject_contains = action_config.get("subject_contains", "").lower() + return subject_contains in message_details["subject"].lower() + + return False diff --git a/backend/automations/helpers/gmail_helper.py b/backend/automations/helpers/gmail_helper.py index a9e0e2a2..93a225b9 100644 --- a/backend/automations/helpers/gmail_helper.py +++ b/backend/automations/helpers/gmail_helper.py @@ -72,6 +72,49 @@ def list_messages( raise +def get_history(access_token: str, start_history_id: str) -> Dict: + """ + Get Gmail history changes since a specific historyId. + + Used by Gmail push notifications to fetch changes since last notification. + + Args: + access_token: Valid Google OAuth token + start_history_id: History ID to start from + + Returns: + dict with "history" (list of changes) and "historyId" (current) + + Raises: + HttpError: If Gmail API request fails + """ + try: + service = get_gmail_service(access_token) + results = ( + service.users() + .history() + .list(userId="me", startHistoryId=start_history_id) + .execute() + ) + + history = results.get("history", []) + logger.info( + f"Gmail history fetched: {len(history)} changes since {start_history_id}" + ) + + return results + + except HttpError as e: + if e.resp.status == 404: + logger.warning(f"History ID {start_history_id} no longer valid") + return {"history": [], "historyId": start_history_id} + logger.error(f"Gmail get_history failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error in get_history: {e}") + raise + + def get_message_details(access_token: str, message_id: str) -> Dict: """ Get full message details including headers and body. diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py new file mode 100644 index 00000000..6ada410e --- /dev/null +++ b/backend/automations/helpers/google_webhook_helper.py @@ -0,0 +1,294 @@ +""" +Google Webhook (Push Notification) Helper Functions. + +This module provides utilities for managing Google push notifications (watches) +for Gmail, Calendar, and YouTube services. + +Key Concepts: +- Google uses "watches" instead of traditional webhooks +- Each watch has a unique channel_id (UUID) and resource_id (returned by Google) +- Watches expire after a period (7 days for Gmail, configurable for Calendar) +- Must renew watches before expiration + +References: +- Gmail: https://developers.google.com/gmail/api/guides/push +- Calendar: https://developers.google.com/calendar/api/guides/push +- YouTube: https://developers.google.com/youtube/v3/guides/push_notifications +""" + +import logging +import uuid +from datetime import datetime, timedelta + +from django.conf import settings +from django.utils import timezone +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +logger = logging.getLogger(__name__) + + +def create_gmail_watch(access_token, webhook_url, user_id=None): + """ + Create a Gmail push notification watch. + + Args: + access_token (str): Valid Google OAuth2 access token + webhook_url (str): Full HTTPS URL to receive notifications + user_id (str): Optional Gmail user ID (default: 'me') + + Returns: + dict: Watch info with keys: channel_id, resource_id, expiration + None: If watch creation failed + + Example: + watch = create_gmail_watch(token, "https://areaction.app/api/webhooks/gmail/") + # {'channel_id': 'uuid...', 'resource_id': 'abc123', 'expiration': datetime} + """ + try: + # Build Gmail API client + service = build("gmail", "v1", credentials=None, static_discovery=False) + service._http.credentials = type("Credentials", (), {"token": access_token})() + + # Generate unique channel ID + channel_id = str(uuid.uuid4()) + + # Create watch request + request_body = { + "labelIds": ["INBOX"], # Watch INBOX only + "topicName": f"projects/{settings.GOOGLE_CLOUD_PROJECT}/topics/gmail", + "labelFilterAction": "include", + } + + # For webhook-based notifications (not Pub/Sub) + if not hasattr(settings, "GOOGLE_CLOUD_PROJECT"): + request_body = { + "labelIds": ["INBOX"], + } + + # Gmail push notifications using Pub/Sub or direct webhook + # Note: Gmail requires Pub/Sub for production, but we'll use direct webhook for simplicity + watch_request = { + "topicName": webhook_url, # This should be a Pub/Sub topic in production + } + + # Alternative: Use direct webhook (for testing, requires domain verification) + user = user_id or "me" + response = ( + service.users() + .watch(userId=user, body={"labelIds": ["INBOX"], "topicName": webhook_url}) + .execute() + ) + + # Extract watch info + history_id = response.get("historyId") + expiration_ms = int(response.get("expiration", 0)) + + # Gmail watch expires in ~7 days + expiration = datetime.fromtimestamp(expiration_ms / 1000, tz=timezone.utc) + + logger.info( + f"Gmail watch created successfully: channel={channel_id}, " + f"expiration={expiration}" + ) + + return { + "channel_id": channel_id, + "resource_id": history_id, + "expiration": expiration, + } + + except HttpError as e: + logger.error(f"Gmail watch creation failed: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error creating Gmail watch: {e}", exc_info=True) + return None + + +def stop_gmail_watch(access_token, channel_id, resource_id): + """ + Stop (delete) a Gmail watch. + + Args: + access_token (str): Valid Google OAuth2 access token + channel_id (str): Channel ID of the watch to stop + resource_id (str): Resource ID returned when watch was created + + Returns: + bool: True if stopped successfully, False otherwise + """ + try: + service = build("gmail", "v1", credentials=None, static_discovery=False) + service._http.credentials = type("Credentials", (), {"token": access_token})() + + service.users().stop(userId="me").execute() + + logger.info(f"Gmail watch stopped successfully: channel={channel_id}") + return True + + except HttpError as e: + logger.error(f"Failed to stop Gmail watch: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error stopping Gmail watch: {e}", exc_info=True) + return False + + +def create_calendar_watch(access_token, calendar_id, webhook_url, expiration_hours=168): + """ + Create a Google Calendar push notification watch. + + Args: + access_token (str): Valid Google OAuth2 access token + calendar_id (str): Calendar ID to watch (use 'primary' for main calendar) + webhook_url (str): Full HTTPS URL to receive notifications + expiration_hours (int): Watch duration in hours (max 604800 seconds = 1 week) + + Returns: + dict: Watch info with keys: channel_id, resource_id, expiration + None: If watch creation failed + """ + try: + service = build("calendar", "v3", credentials=None, static_discovery=False) + service._http.credentials = type("Credentials", (), {"token": access_token})() + + # Generate unique channel ID + channel_id = str(uuid.uuid4()) + + # Calculate expiration (max 1 week for Calendar) + expiration_seconds = min(expiration_hours * 3600, 604800) + expiration = timezone.now() + timedelta(seconds=expiration_seconds) + + # Create watch request body + request_body = { + "id": channel_id, + "type": "web_hook", + "address": webhook_url, + "expiration": int(expiration.timestamp() * 1000), # Milliseconds + } + + # Create the watch + response = ( + service.events() + .watch(calendarId=calendar_id, body=request_body) + .execute() + ) + + # Extract watch info + resource_id = response.get("resourceId") + resource_uri = response.get("resourceUri") + + logger.info( + f"Calendar watch created: channel={channel_id}, " + f"resource={resource_id}, expiration={expiration}" + ) + + return { + "channel_id": channel_id, + "resource_id": resource_id, + "resource_uri": resource_uri, + "expiration": expiration, + } + + except HttpError as e: + logger.error(f"Calendar watch creation failed: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error creating Calendar watch: {e}", exc_info=True) + return None + + +def stop_calendar_watch(access_token, channel_id, resource_id): + """ + Stop (delete) a Google Calendar watch. + + Args: + access_token (str): Valid Google OAuth2 access token + channel_id (str): Channel ID of the watch to stop + resource_id (str): Resource ID returned when watch was created + + Returns: + bool: True if stopped successfully, False otherwise + """ + try: + service = build("calendar", "v3", credentials=None, static_discovery=False) + service._http.credentials = type("Credentials", (), {"token": access_token})() + + request_body = {"id": channel_id, "resourceId": resource_id} + + service.channels().stop(body=request_body).execute() + + logger.info(f"Calendar watch stopped: channel={channel_id}") + return True + + except HttpError as e: + logger.error(f"Failed to stop Calendar watch: {e}") + return False + except Exception as e: + logger.error(f"Unexpected error stopping Calendar watch: {e}", exc_info=True) + return False + + +def create_youtube_watch(channel_id, webhook_url): + """ + Subscribe to YouTube PubSubHubbub notifications. + + NOTE: YouTube uses a different system (PubSubHubbub) rather than the + watch API used by Gmail/Calendar. + + Args: + channel_id (str): YouTube channel ID to watch + webhook_url (str): Full HTTPS URL to receive notifications + + Returns: + dict: Subscription info with keys: channel_id, expiration (10 days) + None: If subscription failed + """ + import requests + + try: + # PubSubHubbub hub URL + hub_url = "https://pubsubhubbub.appspot.com/subscribe" + + # Topic URL (channel feed) + topic_url = f"https://www.youtube.com/xml/feeds/videos.xml?channel_id={channel_id}" + + # Subscribe request + data = { + "hub.mode": "subscribe", + "hub.topic": topic_url, + "hub.callback": webhook_url, + "hub.verify": "async", # Async verification + "hub.lease_seconds": 864000, # 10 days + } + + response = requests.post(hub_url, data=data, timeout=10) + + if response.status_code == 202: + # Subscription accepted (will be verified asynchronously) + expiration = timezone.now() + timedelta(days=10) + + logger.info( + f"YouTube PubSubHubbub subscription created for channel {channel_id}" + ) + + return { + "channel_id": channel_id, + "resource_id": topic_url, + "expiration": expiration, + } + else: + logger.error( + f"YouTube subscription failed: {response.status_code} - {response.text}" + ) + return None + + except Exception as e: + logger.error(f"Unexpected error creating YouTube watch: {e}", exc_info=True) + return None + + +def renew_youtube_watch(channel_id, webhook_url): + """Renew YouTube PubSubHubbub subscription (same as create).""" + return create_youtube_watch(channel_id, webhook_url) diff --git a/backend/automations/migrations/0012_google_webhook_watch.py b/backend/automations/migrations/0012_google_webhook_watch.py new file mode 100644 index 00000000..b0f32a1d --- /dev/null +++ b/backend/automations/migrations/0012_google_webhook_watch.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.6 on 2025-11-02 05:21 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('automations', '0011_fix_webhook_subscription_unique_constraint'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='GoogleWebhookWatch', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('service', models.CharField(choices=[('gmail', 'Gmail'), ('calendar', 'Google Calendar'), ('youtube', 'YouTube')], help_text='Google service being watched', max_length=20)), + ('channel_id', models.CharField(db_index=True, help_text='Unique UUID identifying this watch (generated by us)', max_length=255, unique=True)), + ('resource_id', models.CharField(help_text='Resource ID returned by Google after creating the watch', max_length=255)), + ('resource_uri', models.CharField(blank=True, default='', help_text='Optional: Specific resource being watched (e.g., calendar ID)', max_length=500)), + ('expiration', models.DateTimeField(help_text='When this watch expires (must renew before this time)')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('last_event_at', models.DateTimeField(blank=True, help_text='Timestamp of the last received push notification', null=True)), + ('event_count', models.IntegerField(default=0, help_text='Total number of push notifications received')), + ('user', models.ForeignKey(help_text='User who owns this watch', on_delete=django.db.models.deletion.CASCADE, related_name='google_watches', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Google Webhook Watch', + 'verbose_name_plural': 'Google Webhook Watches', + 'indexes': [models.Index(fields=['user', 'service'], name='automations_user_id_51fc8b_idx'), models.Index(fields=['expiration'], name='automations_expirat_105152_idx'), models.Index(fields=['channel_id'], name='automations_channel_8d45ab_idx')], + 'constraints': [models.UniqueConstraint(fields=('user', 'service', 'resource_uri'), name='unique_user_google_watch')], + }, + ), + ] diff --git a/backend/automations/models.py b/backend/automations/models.py index 39b136b2..3de5a3e3 100755 --- a/backend/automations/models.py +++ b/backend/automations/models.py @@ -486,3 +486,110 @@ def deactivate(self): """Mark installation as inactive (uninstalled).""" self.is_active = False self.save(update_fields=["is_active", "updated_at"]) + + +class GoogleWebhookWatch(models.Model): + """ + Track active Google push notification subscriptions (watches). + + Google services (Gmail, Calendar, YouTube) use push notifications via + their API. Each watch has: + - A unique channel_id (UUID) to identify the subscription + - A resource_id returned by Google + - An expiration time (watches must be renewed before expiry) + + Key Features: + - Automatic watch renewal before expiration + - Per-user, per-service tracking + - Support for multiple resources per user (e.g., multiple calendars) + """ + + class Service(models.TextChoices): + GMAIL = "gmail", "Gmail" + CALENDAR = "calendar", "Google Calendar" + YOUTUBE = "youtube", "YouTube" + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name="google_watches", + help_text="User who owns this watch", + ) + + service = models.CharField( + max_length=20, + choices=Service.choices, + help_text="Google service being watched", + ) + + channel_id = models.CharField( + max_length=255, + unique=True, + db_index=True, + help_text="Unique UUID identifying this watch (generated by us)", + ) + + resource_id = models.CharField( + max_length=255, + help_text="Resource ID returned by Google after creating the watch", + ) + + resource_uri = models.CharField( + max_length=500, + blank=True, + default="", + help_text="Optional: Specific resource being watched (e.g., calendar ID)", + ) + + expiration = models.DateTimeField( + help_text="When this watch expires (must renew before this time)", + ) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + # Event tracking + last_event_at = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp of the last received push notification", + ) + event_count = models.IntegerField( + default=0, + help_text="Total number of push notifications received", + ) + + class Meta: + verbose_name = "Google Webhook Watch" + verbose_name_plural = "Google Webhook Watches" + indexes = [ + models.Index(fields=["user", "service"]), + models.Index(fields=["expiration"]), + models.Index(fields=["channel_id"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["user", "service", "resource_uri"], + name="unique_user_google_watch", + ) + ] + + def __str__(self): + return f"{self.service} watch for {self.user.username} (expires {self.expiration})" + + def is_expiring_soon(self, hours=24): + """Check if watch expires within N hours.""" + from datetime import timedelta + + from django.utils import timezone + + threshold = timezone.now() + timedelta(hours=hours) + return self.expiration <= threshold + + def record_event(self): + """Record that a push notification was received.""" + from django.utils import timezone + + self.last_event_at = timezone.now() + self.event_count += 1 + self.save(update_fields=["last_event_at", "event_count", "updated_at"]) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index c99b93bb..39028455 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -4081,12 +4081,12 @@ def check_youtube_actions(self): # Get published_after from last check or 24 hours ago from datetime import timedelta - + from django.utils import timezone action_state, _ = ActionState.objects.get_or_create(area=area) published_after = None - + if action_state.last_checked_at: # Check videos published after last check published_after = action_state.last_checked_at.isoformat() @@ -4103,7 +4103,7 @@ def check_youtube_actions(self): channel_id=channel_id, published_after=published_after, ) - + # Update last checked time action_state.last_checked_at = timezone.now() action_state.save() @@ -4218,3 +4218,345 @@ def test_execution_flow(area_id: int): except Exception as e: logger.error(f"Error in test_execution_flow: {e}", exc_info=True) return {"status": "error", "message": str(e)} + + +# ============================================================================ +# GOOGLE WEBHOOK (PUSH NOTIFICATION) MANAGEMENT +# ============================================================================ + + +@shared_task( + name="automations.setup_google_watches_for_user", + bind=True, + max_retries=3, + autoretry_for=RECOVERABLE_EXCEPTIONS, +) +def setup_google_watches_for_user(self, user_id): + """ + Set up Google push notification watches for a user. + + This task is called after a user connects their Google account. + It creates watches for Gmail and Calendar to receive real-time notifications. + + Args: + user_id: User ID to set up watches for + + Returns: + dict: Summary of created watches + """ + from django.conf import settings + from django.contrib.auth import get_user_model + + from users.oauth.manager import OAuthManager + + from .helpers.google_webhook_helper import ( + create_calendar_watch, + create_gmail_watch, + ) + from .models import GoogleWebhookWatch + + User = get_user_model() + + try: + user = User.objects.get(pk=user_id) + except User.DoesNotExist: + logger.error(f"User #{user_id} not found") + return {"status": "error", "message": "User not found"} + + logger.info(f"Setting up Google watches for user {user.username}") + + # Get valid Google OAuth token + access_token = OAuthManager.get_valid_token(user, "google") + + if not access_token: + logger.warning(f"No valid Google token for user {user.username}") + return {"status": "no_token", "user": user.username} + + results = {"user": user.username, "watches_created": []} + + # Check if webhooks are enabled + gmail_webhook_enabled = getattr(settings, "GMAIL_WEBHOOK_ENABLED", False) + calendar_webhook_enabled = getattr(settings, "CALENDAR_WEBHOOK_ENABLED", False) + + # Create Gmail watch + if gmail_webhook_enabled: + gmail_webhook_url = getattr( + settings, "GMAIL_WEBHOOK_URL", f"{settings.BACKEND_URL}/api/webhooks/gmail/" + ) + + # Check if watch already exists + existing_gmail = GoogleWebhookWatch.objects.filter( + user=user, service=GoogleWebhookWatch.Service.GMAIL + ).first() + + if not existing_gmail or existing_gmail.is_expiring_soon(hours=48): + # Create new watch + watch_info = create_gmail_watch(access_token, gmail_webhook_url) + + if watch_info: + # Save watch to database + watch, created = GoogleWebhookWatch.objects.update_or_create( + user=user, + service=GoogleWebhookWatch.Service.GMAIL, + defaults={ + "channel_id": watch_info["channel_id"], + "resource_id": watch_info["resource_id"], + "expiration": watch_info["expiration"], + }, + ) + + results["watches_created"].append("gmail") + logger.info( + f"Gmail watch created for {user.username}: {watch.channel_id}" + ) + + # Create Calendar watch + if calendar_webhook_enabled: + calendar_webhook_url = getattr( + settings, + "CALENDAR_WEBHOOK_URL", + f"{settings.BACKEND_URL}/api/webhooks/calendar/", + ) + + # Check if watch already exists + existing_calendar = GoogleWebhookWatch.objects.filter( + user=user, service=GoogleWebhookWatch.Service.CALENDAR + ).first() + + if not existing_calendar or existing_calendar.is_expiring_soon(hours=48): + # Create new watch + watch_info = create_calendar_watch( + access_token, "primary", calendar_webhook_url + ) + + if watch_info: + # Save watch to database + watch, created = GoogleWebhookWatch.objects.update_or_create( + user=user, + service=GoogleWebhookWatch.Service.CALENDAR, + resource_uri="primary", + defaults={ + "channel_id": watch_info["channel_id"], + "resource_id": watch_info["resource_id"], + "expiration": watch_info["expiration"], + }, + ) + + results["watches_created"].append("calendar") + logger.info( + f"Calendar watch created for {user.username}: {watch.channel_id}" + ) + + results["status"] = "success" + results["count"] = len(results["watches_created"]) + + return results + + +@shared_task( + name="automations.renew_google_watches", + bind=True, + max_retries=3, + autoretry_for=RECOVERABLE_EXCEPTIONS, +) +def renew_google_watches(self): + """ + Renew Google push notification watches that are expiring soon. + + This task runs periodically (every hour) to check for watches + that expire within 24 hours and renews them automatically. + + Returns: + dict: Summary of renewed watches + """ + from django.utils import timezone + + from users.oauth.manager import OAuthManager + + from .helpers.google_webhook_helper import ( + create_calendar_watch, + create_gmail_watch, + ) + from .models import GoogleWebhookWatch + + logger.info("Checking for expiring Google watches...") + + # Find watches expiring in next 24 hours + expiring_threshold = timezone.now() + timezone.timedelta(hours=24) + + expiring_watches = GoogleWebhookWatch.objects.filter( + expiration__lte=expiring_threshold + ).select_related("user") + + if not expiring_watches: + logger.info("No Google watches expiring soon") + return {"status": "no_action", "count": 0} + + renewed_count = 0 + failed_count = 0 + + for watch in expiring_watches: + try: + logger.info( + f"Renewing {watch.service} watch for {watch.user.username} " + f"(expires {watch.expiration})" + ) + + # Get valid token + access_token = OAuthManager.get_valid_token(watch.user, "google") + + if not access_token: + logger.warning( + f"No valid token for {watch.user.username}, skipping renewal" + ) + failed_count += 1 + continue + + # Renew based on service type + new_watch_info = None + + if watch.service == GoogleWebhookWatch.Service.GMAIL: + from django.conf import settings + + webhook_url = getattr( + settings, + "GMAIL_WEBHOOK_URL", + f"{settings.BACKEND_URL}/api/webhooks/gmail/", + ) + new_watch_info = create_gmail_watch(access_token, webhook_url) + + elif watch.service == GoogleWebhookWatch.Service.CALENDAR: + from django.conf import settings + + webhook_url = getattr( + settings, + "CALENDAR_WEBHOOK_URL", + f"{settings.BACKEND_URL}/api/webhooks/calendar/", + ) + calendar_id = watch.resource_uri or "primary" + new_watch_info = create_calendar_watch( + access_token, calendar_id, webhook_url + ) + + if new_watch_info: + # Update watch in database + watch.channel_id = new_watch_info["channel_id"] + watch.resource_id = new_watch_info["resource_id"] + watch.expiration = new_watch_info["expiration"] + watch.save() + + renewed_count += 1 + logger.info( + f"Renewed {watch.service} watch for {watch.user.username}" + ) + else: + failed_count += 1 + logger.error( + f"Failed to renew {watch.service} watch for {watch.user.username}" + ) + + except Exception as e: + logger.error( + f"Error renewing watch for {watch.user.username}: {e}", exc_info=True + ) + failed_count += 1 + + logger.info( + f"Google watch renewal complete: {renewed_count} renewed, {failed_count} failed" + ) + + return { + "status": "success", + "renewed": renewed_count, + "failed": failed_count, + "total": len(expiring_watches), + } + + +@shared_task( + name="automations.setup_youtube_watches", + bind=True, + max_retries=3, + autoretry_for=RECOVERABLE_EXCEPTIONS, +) +def setup_youtube_watches(self): + """ + Set up YouTube PubSubHubbub subscriptions for all active youtube_new_video areas. + + YouTube uses PubSubHubbub instead of the watch API. Subscriptions expire + after 10 days and need to be renewed. + + Returns: + dict: Summary of created subscriptions + """ + from django.conf import settings + + from .helpers.google_webhook_helper import create_youtube_watch + from .models import Area, GoogleWebhookWatch + + logger.info("Setting up YouTube PubSubHubbub subscriptions...") + + # Get all active youtube_new_video areas + youtube_areas = Area.objects.filter( + status=Area.Status.ACTIVE, action__name="youtube_new_video" + ).select_related("owner", "action") + + if not youtube_areas: + logger.info("No active YouTube areas found") + return {"status": "no_areas", "count": 0} + + youtube_webhook_url = getattr( + settings, "YOUTUBE_WEBHOOK_URL", f"{settings.BACKEND_URL}/api/webhooks/youtube/" + ) + + created_count = 0 + skipped_count = 0 + + # Group by channel_id to avoid duplicate subscriptions + channels_by_id = {} + for area in youtube_areas: + channel_id = area.action_config.get("channel_id") + if channel_id: + if channel_id not in channels_by_id: + channels_by_id[channel_id] = [] + channels_by_id[channel_id].append(area) + + for channel_id, areas in channels_by_id.items(): + # Check if subscription already exists and is not expiring + existing_watch = GoogleWebhookWatch.objects.filter( + service=GoogleWebhookWatch.Service.YOUTUBE, resource_uri=channel_id + ).first() + + if existing_watch and not existing_watch.is_expiring_soon(hours=48): + logger.debug(f"YouTube subscription for channel {channel_id} already active") + skipped_count += 1 + continue + + # Create subscription + watch_info = create_youtube_watch(channel_id, youtube_webhook_url) + + if watch_info: + # Use first area's owner for the watch (any user watching this channel) + user = areas[0].owner + + # Save watch to database + watch, created = GoogleWebhookWatch.objects.update_or_create( + user=user, + service=GoogleWebhookWatch.Service.YOUTUBE, + resource_uri=channel_id, + defaults={ + "channel_id": channel_id, # Use channel_id as unique identifier + "resource_id": watch_info["resource_id"], + "expiration": watch_info["expiration"], + }, + ) + + created_count += 1 + logger.info(f"YouTube subscription created for channel {channel_id}") + + logger.info( + f"YouTube subscriptions setup complete: {created_count} created, " + f"{skipped_count} skipped" + ) + + return {"status": "success", "created": created_count, "skipped": skipped_count} diff --git a/backend/automations/urls.py b/backend/automations/urls.py index 1cb79c70..bebb5f16 100755 --- a/backend/automations/urls.py +++ b/backend/automations/urls.py @@ -29,6 +29,9 @@ from . import github_app_views, views from .webhooks import webhook_receiver +# Import Google webhook views +from .google_webhook_views import calendar_webhook, gmail_webhook, youtube_webhook + # Create router and register viewsets router = DefaultRouter() router.register(r"services", views.ServiceViewSet, basename="service") @@ -74,6 +77,10 @@ path("logos//", views.logo_proxy_view, name="logo-proxy"), # Webhook receiver endpoint path("webhooks//", webhook_receiver, name="webhook-receiver"), + # Google webhook endpoints (push notifications) + path("api/webhooks/gmail/", gmail_webhook, name="gmail-webhook"), + path("api/webhooks/calendar/", calendar_webhook, name="calendar-webhook"), + path("api/webhooks/youtube/", youtube_webhook, name="youtube-webhook"), # GitHub App endpoints path( "api/github-app/status/", From db3eb3405226652bacfe5ee5ae023084a076e60b Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 06:37:11 +0100 Subject: [PATCH 24/68] feat(tests): add script to verify Google webhook infrastructure setup --- backend/scripts/test_google_webhooks.py | 292 ++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 backend/scripts/test_google_webhooks.py diff --git a/backend/scripts/test_google_webhooks.py b/backend/scripts/test_google_webhooks.py new file mode 100644 index 00000000..cf2d4cbd --- /dev/null +++ b/backend/scripts/test_google_webhooks.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +""" +Test Google Webhooks Setup + +This script verifies the Google webhook infrastructure is correctly configured. +Run with: python scripts/test_google_webhooks.py +""" + +import os +import sys +import django + +# Setup Django environment +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "area_project.settings") +django.setup() + +from django.contrib.auth import get_user_model +from django.utils import timezone +from automations.models import GoogleWebhookWatch, Area +from automations.tasks import ( + setup_google_watches_for_user, + renew_google_watches, + setup_youtube_watches, +) + +User = get_user_model() + + +def test_model(): + """Test GoogleWebhookWatch model""" + print("\n🧪 Testing GoogleWebhookWatch Model...") + + # Check model is registered + try: + watch_count = GoogleWebhookWatch.objects.count() + print(f"✅ Model accessible - {watch_count} watches in database") + except Exception as e: + print(f"❌ Model error: {e}") + return False + + # Test creating a watch + try: + user = User.objects.first() + if not user: + print("⚠️ No users found, skipping watch creation test") + return True + + watch, created = GoogleWebhookWatch.objects.get_or_create( + user=user, + service=GoogleWebhookWatch.Service.GMAIL, + channel_id="test-channel-123", + defaults={ + "resource_id": "test-resource", + "resource_uri": "primary", + "expiration": timezone.now() + timezone.timedelta(days=7), + }, + ) + + if created: + print(f"✅ Test watch created: {watch}") + watch.delete() # Cleanup + else: + print(f"✅ Watch already exists: {watch}") + + return True + + except Exception as e: + print(f"❌ Watch creation failed: {e}") + return False + + +def test_tasks(): + """Test Celery tasks are registered""" + print("\n🧪 Testing Celery Tasks...") + + tasks = [ + ("setup_google_watches_for_user", setup_google_watches_for_user), + ("renew_google_watches", renew_google_watches), + ("setup_youtube_watches", setup_youtube_watches), + ] + + success = True + for name, task in tasks: + try: + # Check task is callable + assert callable(task), f"{name} is not callable" + print(f"✅ Task registered: {name}") + except Exception as e: + print(f"❌ Task {name} failed: {e}") + success = False + + return success + + +def test_routes(): + """Test webhook URLs are configured""" + print("\n🧪 Testing Webhook Routes...") + + from django.urls import reverse + + routes = [ + ("gmail-webhook", "/webhooks/gmail/"), + ("calendar-webhook", "/webhooks/calendar/"), + ("youtube-webhook", "/webhooks/youtube/"), + ] + + success = True + for name, expected_path in routes: + try: + url = reverse(name) + if url == expected_path: + print(f"✅ Route {name}: {url}") + else: + print(f"⚠️ Route {name}: {url} (expected {expected_path})") + except Exception as e: + print(f"❌ Route {name} not found: {e}") + success = False + + return success + + +def test_helpers(): + """Test helper functions are importable""" + print("\n🧪 Testing Helper Functions...") + + try: + from automations.helpers.google_webhook_helper import ( + create_gmail_watch, + stop_gmail_watch, + create_calendar_watch, + stop_calendar_watch, + create_youtube_watch, + renew_youtube_watch, + ) + + helpers = [ + "create_gmail_watch", + "stop_gmail_watch", + "create_calendar_watch", + "stop_calendar_watch", + "create_youtube_watch", + "renew_youtube_watch", + ] + + for helper in helpers: + print(f"✅ Helper imported: {helper}") + + return True + + except ImportError as e: + print(f"❌ Helper import failed: {e}") + return False + + +def test_views(): + """Test webhook views are importable""" + print("\n🧪 Testing Webhook Views...") + + try: + from automations.google_webhook_views import ( + gmail_webhook, + calendar_webhook, + youtube_webhook, + ) + + views = [ + "gmail_webhook", + "calendar_webhook", + "youtube_webhook", + ] + + for view in views: + print(f"✅ View imported: {view}") + + return True + + except ImportError as e: + print(f"❌ View import failed: {e}") + return False + + +def test_configuration(): + """Test Django settings""" + print("\n🧪 Testing Configuration...") + + from django.conf import settings + + configs = [ + ("GMAIL_WEBHOOK_ENABLED", False), + ("CALENDAR_WEBHOOK_ENABLED", False), + ("YOUTUBE_WEBHOOK_ENABLED", False), + ("GMAIL_WEBHOOK_URL", None), + ("CALENDAR_WEBHOOK_URL", None), + ("YOUTUBE_WEBHOOK_URL", None), + ] + + for key, default in configs: + value = getattr(settings, key, default) + status = "✅" if value else "⚠️ " + print(f"{status} {key}: {value}") + + return True + + +def show_active_watches(): + """Display active watches""" + print("\n📊 Active Google Watches:") + + watches = GoogleWebhookWatch.objects.select_related("user").all() + + if not watches: + print(" No active watches") + return + + for watch in watches: + expiring = watch.is_expiring_soon(hours=24) + status = "⏰ EXPIRING SOON" if expiring else "✅ Active" + print(f" {status} | {watch.user.username} | {watch.service} | " + f"expires {watch.expiration.strftime('%Y-%m-%d %H:%M')}") + + +def show_youtube_areas(): + """Display active YouTube areas""" + print("\n📊 Active YouTube Areas:") + + areas = Area.objects.filter( + status=Area.Status.ACTIVE, action__name="youtube_new_video" + ).select_related("owner", "action") + + if not areas: + print(" No active YouTube areas") + return + + channels = {} + for area in areas: + channel_id = area.action_config.get("channel_id", "N/A") + if channel_id not in channels: + channels[channel_id] = [] + channels[channel_id].append(area) + + for channel_id, areas_list in channels.items(): + print(f" 📺 Channel {channel_id}: {len(areas_list)} areas") + + +def main(): + print("=" * 70) + print("🧪 Google Webhooks Infrastructure Test") + print("=" * 70) + + results = [] + + # Run tests + results.append(("Model", test_model())) + results.append(("Tasks", test_tasks())) + results.append(("Routes", test_routes())) + results.append(("Helpers", test_helpers())) + results.append(("Views", test_views())) + results.append(("Configuration", test_configuration())) + + # Show data + show_active_watches() + show_youtube_areas() + + # Summary + print("\n" + "=" * 70) + print("📋 Test Summary:") + print("=" * 70) + + all_passed = True + for test_name, passed in results: + status = "✅ PASS" if passed else "❌ FAIL" + print(f" {status} - {test_name}") + if not passed: + all_passed = False + + print("=" * 70) + + if all_passed: + print("✅ All tests passed! Infrastructure ready.") + print("\n📝 Next steps:") + print(" 1. Deploy to production: ./deployment/manage.sh update") + print(" 2. Verify domain on Google Search Console (required for Gmail)") + print(" 3. Test with real events (send email, create calendar event, upload video)") + return 0 + else: + print("❌ Some tests failed. Fix errors before deploying.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From b3306bbc6099d654181a607410f1aedc1fd05323 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 07:06:41 +0100 Subject: [PATCH 25/68] fix(urls): update Google webhook paths to remove 'api/' prefix --- backend/automations/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/automations/urls.py b/backend/automations/urls.py index bebb5f16..4809f3c7 100755 --- a/backend/automations/urls.py +++ b/backend/automations/urls.py @@ -78,9 +78,9 @@ # Webhook receiver endpoint path("webhooks//", webhook_receiver, name="webhook-receiver"), # Google webhook endpoints (push notifications) - path("api/webhooks/gmail/", gmail_webhook, name="gmail-webhook"), - path("api/webhooks/calendar/", calendar_webhook, name="calendar-webhook"), - path("api/webhooks/youtube/", youtube_webhook, name="youtube-webhook"), + path("webhooks/gmail/", gmail_webhook, name="gmail-webhook"), + path("webhooks/calendar/", calendar_webhook, name="calendar-webhook"), + path("webhooks/youtube/", youtube_webhook, name="youtube-webhook"), # GitHub App endpoints path( "api/github-app/status/", From f41299562f09849925e5ddfb8b79f5e0656e31cd Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 07:23:54 +0100 Subject: [PATCH 26/68] feat(google-webhooks): add configuration for Gmail, Calendar, and YouTube webhooks --- backend/area_project/settings/base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/area_project/settings/base.py b/backend/area_project/settings/base.py index 05923327..770a40df 100755 --- a/backend/area_project/settings/base.py +++ b/backend/area_project/settings/base.py @@ -349,6 +349,22 @@ logger.debug(f"Raw value: {webhook_secrets_raw[:50]}...") WEBHOOK_SECRETS = {} +# ============================================================================= +# GOOGLE PUSH NOTIFICATIONS (WEBHOOKS) +# ============================================================================= +# Configuration for Gmail, Calendar, and YouTube webhooks +GMAIL_WEBHOOK_ENABLED = os.getenv("GMAIL_WEBHOOK_ENABLED", "false").lower() == "true" +CALENDAR_WEBHOOK_ENABLED = os.getenv("CALENDAR_WEBHOOK_ENABLED", "false").lower() == "true" +YOUTUBE_WEBHOOK_ENABLED = os.getenv("YOUTUBE_WEBHOOK_ENABLED", "false").lower() == "true" + +# Webhook URLs (must be set in environment variables) +GMAIL_WEBHOOK_URL = os.getenv("GMAIL_WEBHOOK_URL", "") +CALENDAR_WEBHOOK_URL = os.getenv("CALENDAR_WEBHOOK_URL", "") +YOUTUBE_WEBHOOK_URL = os.getenv("YOUTUBE_WEBHOOK_URL", "") + +# Watch renewal interval (in seconds) - default 6 days +GOOGLE_WATCH_RENEWAL_INTERVAL = int(os.getenv("GOOGLE_WATCH_RENEWAL_INTERVAL", "518400")) + # Security Settings (base - extended per environment) SECURE_BROWSER_XSS_FILTER = True SECURE_CONTENT_TYPE_NOSNIFF = True From 6497fc14eb8614d20c1de399e0c575c953889451 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 07:31:46 +0100 Subject: [PATCH 27/68] refactor(webhooks): replace require_http_methods with api_view and permission_classes --- backend/automations/google_webhook_views.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index a81c5373..f92967a6 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -18,7 +18,8 @@ from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_http_methods +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import AllowAny from rest_framework import status from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import AllowAny @@ -34,7 +35,8 @@ @csrf_exempt -@require_http_methods(["GET", "POST"]) +@api_view(["GET", "POST"]) +@permission_classes([AllowAny]) def gmail_webhook(request): """ Handle Gmail push notifications. @@ -187,7 +189,8 @@ def gmail_webhook(request): @csrf_exempt -@require_http_methods(["GET", "POST"]) +@api_view(["GET", "POST"]) +@permission_classes([AllowAny]) def calendar_webhook(request): """ Handle Google Calendar push notifications. @@ -334,7 +337,8 @@ def calendar_webhook(request): @csrf_exempt -@require_http_methods(["GET", "POST"]) +@api_view(["GET", "POST"]) +@permission_classes([AllowAny]) def youtube_webhook(request): """ Handle YouTube PubSubHubbub notifications. From 7c77310b2d734023b86c10e74bd211fc9ab81801 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 11:50:52 +0100 Subject: [PATCH 28/68] feat(about-service): add YouTube logo URL to AboutServiceSerializer --- backend/automations/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/automations/serializers.py b/backend/automations/serializers.py index 82020829..adab0be8 100755 --- a/backend/automations/serializers.py +++ b/backend/automations/serializers.py @@ -483,6 +483,7 @@ def get_logo(self, obj): "slack": "https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg", "twitch": "https://upload.wikimedia.org/wikipedia/commons/d/d3/Twitch_Glitch_Logo_Purple.svg", "google_calendar": "https://upload.wikimedia.org/wikipedia/commons/a/a5/Google_Calendar_icon_%282020%29.svg", + "youtube": "https://upload.wikimedia.org/wikipedia/commons/b/b8/YouTube_Logo_2017.svg", "spotify": "https://upload.wikimedia.org/wikipedia/commons/1/19/Spotify_logo_without_text.svg", "discord": "https://upload.wikimedia.org/wikipedia/commons/9/98/Discord_logo_2015.svg", "notion": "https://upload.wikimedia.org/wikipedia/commons/4/45/Notion_app_logo.png", From f4c3e6155b4701ea0af4bed03c7271c7fcfe791d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 11:54:55 +0100 Subject: [PATCH 29/68] feat(oauth-callback): automate Google webhook setup during OAuth callback --- backend/users/oauth_views.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/users/oauth_views.py b/backend/users/oauth_views.py index f38a62b4..0e8ab1e5 100755 --- a/backend/users/oauth_views.py +++ b/backend/users/oauth_views.py @@ -243,7 +243,24 @@ def get(self, request, provider: str): f"{user.email}/{provider}" ) - # Step 5: Redirect to frontend with success + # Step 5: Setup Google webhooks automatically if provider is Google + if provider == "google": + try: + from automations.tasks import setup_google_watches_for_user + + # Trigger async task to create Gmail/Calendar watches + setup_google_watches_for_user.delay(user.id) + logger.info( + f"Triggered Google webhook setup for user {user.email}" + ) + except Exception as e: + # Don't fail the OAuth flow if watch setup fails + logger.error( + f"Failed to trigger Google watch setup for {user.email}: {e}", + exc_info=True, + ) + + # Step 6: Redirect to frontend with success return self._redirect_with_success( callback_base, provider, created, expires_at ) From b27b7a8f8d106d68cdf39345bf0ceee5963f10f2 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:02:14 +0100 Subject: [PATCH 30/68] fix(about-service): update YouTube logo URL in AboutServiceSerializer --- backend/automations/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/automations/serializers.py b/backend/automations/serializers.py index adab0be8..1b6634b8 100755 --- a/backend/automations/serializers.py +++ b/backend/automations/serializers.py @@ -483,7 +483,7 @@ def get_logo(self, obj): "slack": "https://upload.wikimedia.org/wikipedia/commons/d/d5/Slack_icon_2019.svg", "twitch": "https://upload.wikimedia.org/wikipedia/commons/d/d3/Twitch_Glitch_Logo_Purple.svg", "google_calendar": "https://upload.wikimedia.org/wikipedia/commons/a/a5/Google_Calendar_icon_%282020%29.svg", - "youtube": "https://upload.wikimedia.org/wikipedia/commons/b/b8/YouTube_Logo_2017.svg", + "youtube": "https://upload.wikimedia.org/wikipedia/commons/0/09/YouTube_full-color_icon_%282017%29.svg", "spotify": "https://upload.wikimedia.org/wikipedia/commons/1/19/Spotify_logo_without_text.svg", "discord": "https://upload.wikimedia.org/wikipedia/commons/9/98/Discord_logo_2015.svg", "notion": "https://upload.wikimedia.org/wikipedia/commons/4/45/Notion_app_logo.png", From 3fbcc73967aba18722e4c1e0bc9aed1f50a8b19d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:14:53 +0100 Subject: [PATCH 31/68] fix(webhooks): update Gmail and Calendar webhook URLs to remove '/api' --- backend/automations/helpers/google_webhook_helper.py | 2 +- backend/automations/tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py index 6ada410e..be0c3397 100644 --- a/backend/automations/helpers/google_webhook_helper.py +++ b/backend/automations/helpers/google_webhook_helper.py @@ -42,7 +42,7 @@ def create_gmail_watch(access_token, webhook_url, user_id=None): None: If watch creation failed Example: - watch = create_gmail_watch(token, "https://areaction.app/api/webhooks/gmail/") + watch = create_gmail_watch(token, "https://areaction.app/webhooks/gmail/") # {'channel_id': 'uuid...', 'resource_id': 'abc123', 'expiration': datetime} """ try: diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index 39028455..ec9bf9ec 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -4281,7 +4281,7 @@ def setup_google_watches_for_user(self, user_id): # Create Gmail watch if gmail_webhook_enabled: gmail_webhook_url = getattr( - settings, "GMAIL_WEBHOOK_URL", f"{settings.BACKEND_URL}/api/webhooks/gmail/" + settings, "GMAIL_WEBHOOK_URL", f"{settings.BACKEND_URL}/webhooks/gmail/" ) # Check if watch already exists @@ -4421,7 +4421,7 @@ def renew_google_watches(self): webhook_url = getattr( settings, "GMAIL_WEBHOOK_URL", - f"{settings.BACKEND_URL}/api/webhooks/gmail/", + f"{settings.BACKEND_URL}/webhooks/gmail/", ) new_watch_info = create_gmail_watch(access_token, webhook_url) @@ -4431,7 +4431,7 @@ def renew_google_watches(self): webhook_url = getattr( settings, "CALENDAR_WEBHOOK_URL", - f"{settings.BACKEND_URL}/api/webhooks/calendar/", + f"{settings.BACKEND_URL}/webhooks/calendar/", ) calendar_id = watch.resource_uri or "primary" new_watch_info = create_calendar_watch( From 0ff5d8e5154394a7c56e9b37058aa65eca131c5d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:22:49 +0100 Subject: [PATCH 32/68] feat(settings): add BACKEND_URL for webhook callbacks configuration --- backend/area_project/settings/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/area_project/settings/base.py b/backend/area_project/settings/base.py index 770a40df..29ca9991 100755 --- a/backend/area_project/settings/base.py +++ b/backend/area_project/settings/base.py @@ -548,6 +548,11 @@ # In production: https://your-frontend-domain.com FRONTEND_URL = os.getenv("FRONTEND_URL", "http://localhost:5173") +# Backend URL for webhook callbacks +# In development: http://localhost:8000 +# In production: https://your-backend-domain.com +BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000") + # CORS Configuration # Base allowed origins (extended per environment) CORS_ALLOWED_ORIGINS = [ From de0b198c7a1e14a684ffe20758490175e4a8eb2c Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:22:53 +0100 Subject: [PATCH 33/68] fix(webhooks): update Gmail and Calendar webhook URLs to use BACKEND_URL --- backend/automations/tasks.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index ec9bf9ec..ca6190a3 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -4280,8 +4280,9 @@ def setup_google_watches_for_user(self, user_id): # Create Gmail watch if gmail_webhook_enabled: + backend_url = getattr(settings, "BACKEND_URL", "https://areaction.app") gmail_webhook_url = getattr( - settings, "GMAIL_WEBHOOK_URL", f"{settings.BACKEND_URL}/webhooks/gmail/" + settings, "GMAIL_WEBHOOK_URL", f"{backend_url}/webhooks/gmail/" ) # Check if watch already exists @@ -4312,10 +4313,11 @@ def setup_google_watches_for_user(self, user_id): # Create Calendar watch if calendar_webhook_enabled: + backend_url = getattr(settings, "BACKEND_URL", "https://areaction.app") calendar_webhook_url = getattr( settings, "CALENDAR_WEBHOOK_URL", - f"{settings.BACKEND_URL}/api/webhooks/calendar/", + f"{backend_url}/webhooks/calendar/", ) # Check if watch already exists @@ -4418,20 +4420,22 @@ def renew_google_watches(self): if watch.service == GoogleWebhookWatch.Service.GMAIL: from django.conf import settings + backend_url = getattr(settings, "BACKEND_URL", "https://areaction.app") webhook_url = getattr( settings, "GMAIL_WEBHOOK_URL", - f"{settings.BACKEND_URL}/webhooks/gmail/", + f"{backend_url}/webhooks/gmail/", ) new_watch_info = create_gmail_watch(access_token, webhook_url) elif watch.service == GoogleWebhookWatch.Service.CALENDAR: from django.conf import settings + backend_url = getattr(settings, "BACKEND_URL", "https://areaction.app") webhook_url = getattr( settings, "CALENDAR_WEBHOOK_URL", - f"{settings.BACKEND_URL}/webhooks/calendar/", + f"{backend_url}/webhooks/calendar/", ) calendar_id = watch.resource_uri or "primary" new_watch_info = create_calendar_watch( @@ -4505,8 +4509,9 @@ def setup_youtube_watches(self): logger.info("No active YouTube areas found") return {"status": "no_areas", "count": 0} + backend_url = getattr(settings, "BACKEND_URL", "https://areaction.app") youtube_webhook_url = getattr( - settings, "YOUTUBE_WEBHOOK_URL", f"{settings.BACKEND_URL}/api/webhooks/youtube/" + settings, "YOUTUBE_WEBHOOK_URL", f"{backend_url}/webhooks/youtube/" ) created_count = 0 From 69447e094d802059dd96efd766fa56e625c529ba Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:32:42 +0100 Subject: [PATCH 34/68] feat(gmail): update Gmail watch creation to use Credentials for authentication --- .../helpers/google_webhook_helper.py | 42 +++++++------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py index be0c3397..87aade10 100644 --- a/backend/automations/helpers/google_webhook_helper.py +++ b/backend/automations/helpers/google_webhook_helper.py @@ -22,6 +22,7 @@ from django.conf import settings from django.utils import timezone +from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError @@ -46,39 +47,23 @@ def create_gmail_watch(access_token, webhook_url, user_id=None): # {'channel_id': 'uuid...', 'resource_id': 'abc123', 'expiration': datetime} """ try: - # Build Gmail API client - service = build("gmail", "v1", credentials=None, static_discovery=False) - service._http.credentials = type("Credentials", (), {"token": access_token})() + # Build Gmail API client with credentials + creds = Credentials(token=access_token) + service = build("gmail", "v1", credentials=creds, static_discovery=False) # Generate unique channel ID channel_id = str(uuid.uuid4()) - # Create watch request - request_body = { - "labelIds": ["INBOX"], # Watch INBOX only - "topicName": f"projects/{settings.GOOGLE_CLOUD_PROJECT}/topics/gmail", - "labelFilterAction": "include", - } - - # For webhook-based notifications (not Pub/Sub) - if not hasattr(settings, "GOOGLE_CLOUD_PROJECT"): - request_body = { - "labelIds": ["INBOX"], - } - - # Gmail push notifications using Pub/Sub or direct webhook - # Note: Gmail requires Pub/Sub for production, but we'll use direct webhook for simplicity + # Gmail watch request body + # Note: topicName should be a Cloud Pub/Sub topic, but Gmail also supports direct push watch_request = { - "topicName": webhook_url, # This should be a Pub/Sub topic in production + "labelIds": ["INBOX"], + "topicName": webhook_url, } - # Alternative: Use direct webhook (for testing, requires domain verification) + # Create watch user = user_id or "me" - response = ( - service.users() - .watch(userId=user, body={"labelIds": ["INBOX"], "topicName": webhook_url}) - .execute() - ) + response = service.users().watch(userId=user, body=watch_request).execute() # Extract watch info history_id = response.get("historyId") @@ -89,7 +74,7 @@ def create_gmail_watch(access_token, webhook_url, user_id=None): logger.info( f"Gmail watch created successfully: channel={channel_id}, " - f"expiration={expiration}" + f"historyId={history_id}, expiration={expiration}" ) return { @@ -150,8 +135,9 @@ def create_calendar_watch(access_token, calendar_id, webhook_url, expiration_hou None: If watch creation failed """ try: - service = build("calendar", "v3", credentials=None, static_discovery=False) - service._http.credentials = type("Credentials", (), {"token": access_token})() + # Build Calendar API client with credentials + creds = Credentials(token=access_token) + service = build("calendar", "v3", credentials=creds, static_discovery=False) # Generate unique channel ID channel_id = str(uuid.uuid4()) From e64eb1ff3addf8c7e9ed14047e59a84e68d010d0 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 12:51:33 +0100 Subject: [PATCH 35/68] refactor(webhooks): remove csrf_exempt decorator from webhook views --- backend/automations/google_webhook_views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index f92967a6..b3bfd059 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -34,7 +34,6 @@ logger = logging.getLogger(__name__) -@csrf_exempt @api_view(["GET", "POST"]) @permission_classes([AllowAny]) def gmail_webhook(request): @@ -188,7 +187,6 @@ def gmail_webhook(request): return HttpResponse("Internal error", status=500) -@csrf_exempt @api_view(["GET", "POST"]) @permission_classes([AllowAny]) def calendar_webhook(request): @@ -336,7 +334,6 @@ def calendar_webhook(request): return HttpResponse("Internal error", status=500) -@csrf_exempt @api_view(["GET", "POST"]) @permission_classes([AllowAny]) def youtube_webhook(request): From 984c5183e45d0b969048479f7fea85454dd4f87f Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 13:16:17 +0100 Subject: [PATCH 36/68] fix: reorder webhook routes - Google webhooks before generic --- backend/automations/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/automations/urls.py b/backend/automations/urls.py index 4809f3c7..bb6e8a4c 100755 --- a/backend/automations/urls.py +++ b/backend/automations/urls.py @@ -75,12 +75,12 @@ path("about.json", views.about_json_view, name="about"), # Logo proxy endpoint path("logos//", views.logo_proxy_view, name="logo-proxy"), - # Webhook receiver endpoint - path("webhooks//", webhook_receiver, name="webhook-receiver"), - # Google webhook endpoints (push notifications) + # Google webhook endpoints (push notifications) - MUST be before generic webhook path("webhooks/gmail/", gmail_webhook, name="gmail-webhook"), path("webhooks/calendar/", calendar_webhook, name="calendar-webhook"), path("webhooks/youtube/", youtube_webhook, name="youtube-webhook"), + # Webhook receiver endpoint (generic, catches all other services) + path("webhooks//", webhook_receiver, name="webhook-receiver"), # GitHub App endpoints path( "api/github-app/status/", From dff316c98a58fad4368276283f3a8d4ffdb6164d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 13:25:57 +0100 Subject: [PATCH 37/68] feat(youtube): enhance parse_atom_feed_entry to handle XML and extract additional video details --- backend/automations/helpers/youtube_helper.py | 98 +++++++++++++++---- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py index cd1b8073..b54337f4 100644 --- a/backend/automations/helpers/youtube_helper.py +++ b/backend/automations/helpers/youtube_helper.py @@ -8,6 +8,7 @@ """YouTube Data API v3 helper functions for actions and reactions.""" import logging +import xml.etree.ElementTree as ET from typing import Dict, List, Optional from google.oauth2.credentials import Credentials @@ -363,26 +364,89 @@ def rate_video(access_token: str, video_id: str, rating: str) -> bool: raise -def parse_atom_feed_entry(feed_entry: Dict) -> Dict: +def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: """ - Parse YouTube PubSubHubbub Atom feed entry into structured data. + Parse YouTube PubSubHubbub Atom feed XML into structured data. Args: - feed_entry: Parsed Atom feed entry dict + xml_string: Raw XML string from YouTube PubSubHubbub notification Returns: - Dict with video details (video_id, title, channel_id, published_at) + Dict with video details (video_id, title, channel_id, published_at, etc.) + None if parsing fails """ - # Extract data from feed entry - video_data = { - "video_id": feed_entry.get("yt:videoId", ""), - "title": feed_entry.get("title", ""), - "channel_id": feed_entry.get("yt:channelId", ""), - "channel_title": feed_entry.get("author", {}).get("name", ""), - "published_at": feed_entry.get("published", ""), - "updated_at": feed_entry.get("updated", ""), - "link": feed_entry.get("link", {}).get("@href", ""), - } - - logger.debug(f"Parsed Atom feed entry: {video_data}") - return video_data + try: + + # Parse XML + root = ET.fromstring(xml_string) + + # Atom namespace + ns = { + 'atom': 'http://www.w3.org/2005/Atom', + 'yt': 'http://www.youtube.com/xml/schemas/2015' + } + + # Find entry element + entry = root.find('atom:entry', ns) + if entry is None: + logger.warning("No entry element found in Atom feed") + return None + + # Extract video ID + video_id_elem = entry.find('yt:videoId', ns) + video_id = video_id_elem.text if video_id_elem is not None else "" + + # Extract title + title_elem = entry.find('atom:title', ns) + title = title_elem.text if title_elem is not None else "" + + # Extract channel ID + channel_id_elem = entry.find('yt:channelId', ns) + channel_id = channel_id_elem.text if channel_id_elem is not None else "" + + # Extract author/channel name + author_elem = entry.find('atom:author/atom:name', ns) + channel_title = author_elem.text if author_elem is not None else "" + + # Extract published date + published_elem = entry.find('atom:published', ns) + published_at = published_elem.text if published_elem is not None else "" + + # Extract updated date + updated_elem = entry.find('atom:updated', ns) + updated_at = updated_elem.text if updated_elem is not None else "" + + # Extract link + link_elem = entry.find('atom:link[@rel="alternate"]', ns) + link = link_elem.get('href', '') if link_elem is not None else "" + + # Extract thumbnail (media:group/media:thumbnail) + thumbnail_url = "" + try: + media_ns = {'media': 'http://search.yahoo.com/mrss/'} + thumbnail_elem = entry.find('.//media:thumbnail', media_ns) + if thumbnail_elem is not None: + thumbnail_url = thumbnail_elem.get('url', '') + except: + pass + + video_data = { + "video_id": video_id, + "title": title, + "channel_id": channel_id, + "channel_title": channel_title, + "published_at": published_at, + "updated_at": updated_at, + "link": link, + "thumbnail_url": thumbnail_url, + } + + logger.debug(f"Parsed Atom feed entry: video_id={video_id}, title={title}") + return video_data + + except ET.ParseError as e: + logger.error(f"Failed to parse Atom feed XML: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error parsing Atom feed: {e}", exc_info=True) + return None From 89cf08f26ae0df6217755cea18d789dacb895baf Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 13:41:59 +0100 Subject: [PATCH 38/68] feat(calendar): add calendar_id parameter to list_upcoming_events function --- backend/automations/helpers/calendar_helper.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/automations/helpers/calendar_helper.py b/backend/automations/helpers/calendar_helper.py index da590e47..e28af988 100644 --- a/backend/automations/helpers/calendar_helper.py +++ b/backend/automations/helpers/calendar_helper.py @@ -33,7 +33,10 @@ def get_calendar_service(access_token: str): def list_upcoming_events( - access_token: str, max_results: int = 10, time_min: Optional[str] = None + access_token: str, + max_results: int = 10, + time_min: Optional[str] = None, + calendar_id: str = "primary" ) -> List[Dict]: """ List upcoming calendar events. @@ -42,6 +45,7 @@ def list_upcoming_events( access_token: Valid Google OAuth token max_results: Max events to return (default: 10) time_min: RFC3339 timestamp for earliest event (default: now) + calendar_id: Calendar ID to query (default: "primary") Returns: List of event dicts with id, summary, start, end @@ -58,7 +62,7 @@ def list_upcoming_events( events_result = ( service.events() .list( - calendarId="primary", + calendarId=calendar_id, timeMin=time_min, maxResults=max_results, singleEvents=True, From dba98b17f9b812f98e2b673e5dff73f9c5dc3b42 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:30:29 +0100 Subject: [PATCH 39/68] fix(calendar): correct formatting of parameters in list_upcoming_events function --- backend/automations/helpers/calendar_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/automations/helpers/calendar_helper.py b/backend/automations/helpers/calendar_helper.py index e28af988..12d6af20 100644 --- a/backend/automations/helpers/calendar_helper.py +++ b/backend/automations/helpers/calendar_helper.py @@ -33,8 +33,8 @@ def get_calendar_service(access_token: str): def list_upcoming_events( - access_token: str, - max_results: int = 10, + access_token: str, + max_results: int = 10, time_min: Optional[str] = None, calendar_id: str = "primary" ) -> List[Dict]: From dce1a2e1f20e5c7182673c7144839932cb2d2055 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:30:34 +0100 Subject: [PATCH 40/68] fix(youtube): improve XML parsing in parse_atom_feed_entry function --- backend/automations/helpers/youtube_helper.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py index b54337f4..6eb02101 100644 --- a/backend/automations/helpers/youtube_helper.py +++ b/backend/automations/helpers/youtube_helper.py @@ -376,50 +376,50 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: None if parsing fails """ try: - + # Parse XML root = ET.fromstring(xml_string) - + # Atom namespace ns = { 'atom': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015' } - + # Find entry element entry = root.find('atom:entry', ns) if entry is None: logger.warning("No entry element found in Atom feed") return None - + # Extract video ID video_id_elem = entry.find('yt:videoId', ns) video_id = video_id_elem.text if video_id_elem is not None else "" - + # Extract title title_elem = entry.find('atom:title', ns) title = title_elem.text if title_elem is not None else "" - + # Extract channel ID channel_id_elem = entry.find('yt:channelId', ns) channel_id = channel_id_elem.text if channel_id_elem is not None else "" - + # Extract author/channel name author_elem = entry.find('atom:author/atom:name', ns) channel_title = author_elem.text if author_elem is not None else "" - + # Extract published date published_elem = entry.find('atom:published', ns) published_at = published_elem.text if published_elem is not None else "" - + # Extract updated date updated_elem = entry.find('atom:updated', ns) updated_at = updated_elem.text if updated_elem is not None else "" - + # Extract link link_elem = entry.find('atom:link[@rel="alternate"]', ns) link = link_elem.get('href', '') if link_elem is not None else "" - + # Extract thumbnail (media:group/media:thumbnail) thumbnail_url = "" try: @@ -429,7 +429,7 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: thumbnail_url = thumbnail_elem.get('url', '') except: pass - + video_data = { "video_id": video_id, "title": title, @@ -443,7 +443,7 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: logger.debug(f"Parsed Atom feed entry: video_id={video_id}, title={title}") return video_data - + except ET.ParseError as e: logger.error(f"Failed to parse Atom feed XML: {e}") return None From fe8e27125cd9fb3841e83cf6f5c98719320b75f1 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:31:31 +0100 Subject: [PATCH 41/68] fix(celery): disable polling for Google Calendar and YouTube actions when webhooks are enabled --- backend/area_project/celery.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index c50e19fa..463e31a5 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -59,14 +59,16 @@ def get_beat_schedule(): "task": "automations.check_gmail_actions", "schedule": 180.0, # Every 3 minutes (polling fallback) }, - "check-google-calendar-actions": { - "task": "automations.check_google_calendar_actions", - "schedule": 180.0, # Every 3 minutes (polling fallback) - }, - "check-youtube-actions": { - "task": "automations.check_youtube_actions", - "schedule": 300.0, # Every 5 minutes (polling fallback) - }, + # Google Calendar: Webhooks enabled, polling disabled + # "check-google-calendar-actions": { + # "task": "automations.check_google_calendar_actions", + # "schedule": 180.0, # Every 3 minutes (polling fallback) + # }, + # YouTube: Webhooks enabled, polling disabled + # "check-youtube-actions": { + # "task": "automations.check_youtube_actions", + # "schedule": 300.0, # Every 5 minutes (polling fallback) + # }, "check-weather-actions": { "task": "automations.check_weather_actions", "schedule": 300.0, # Every 5 minutes (no webhook support) From fcd5007c3585cfd5d474e8e1f09eb1d0f1447ff3 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:32:30 +0100 Subject: [PATCH 42/68] fix(celery): disable polling for Twitch actions when webhooks are configured --- backend/area_project/celery.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index 463e31a5..6dc80c36 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -131,17 +131,6 @@ def get_beat_schedule(): else: print("✅ [CELERY BEAT] PROD: GitHub webhooks active, polling disabled") - if not webhook_secrets.get("twitch"): - schedule["check-twitch-actions"] = { - "task": "automations.check_twitch_actions", - "schedule": 60.0, # Every minute - } - print( - "⚠️ [CELERY BEAT] PROD: Twitch polling enabled (webhook not configured)" - ) - else: - print("✅ [CELERY BEAT] PROD: Twitch webhooks active, polling disabled") - if not webhook_secrets.get("slack"): schedule["check-slack-actions"] = { "task": "automations.check_slack_actions", @@ -159,6 +148,10 @@ def get_beat_schedule(): "schedule": 300.0, # Every 5 minutes } print("✅ [CELERY BEAT] Notion polling enabled (every 5 minutes)") + + # Google services: Webhooks active + print("✅ [CELERY BEAT] Google Calendar webhooks active, polling disabled") + print("✅ [CELERY BEAT] YouTube webhooks active, polling disabled") return schedule From 9d85a12ae446b25d9b7f215b8276c9a814136667 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:34 +0100 Subject: [PATCH 43/68] fix(celery): remove unnecessary whitespace in get_beat_schedule function --- backend/area_project/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/area_project/celery.py b/backend/area_project/celery.py index 6dc80c36..1c15b4b8 100755 --- a/backend/area_project/celery.py +++ b/backend/area_project/celery.py @@ -148,7 +148,7 @@ def get_beat_schedule(): "schedule": 300.0, # Every 5 minutes } print("✅ [CELERY BEAT] Notion polling enabled (every 5 minutes)") - + # Google services: Webhooks active print("✅ [CELERY BEAT] Google Calendar webhooks active, polling disabled") print("✅ [CELERY BEAT] YouTube webhooks active, polling disabled") From 18c160bd2fa35661dd4359873cdf6b402668ecd8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:38 +0100 Subject: [PATCH 44/68] refactor(settings): improve readability of webhook configuration in base.py --- backend/area_project/settings/base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/area_project/settings/base.py b/backend/area_project/settings/base.py index 29ca9991..69ffd4a0 100755 --- a/backend/area_project/settings/base.py +++ b/backend/area_project/settings/base.py @@ -354,8 +354,12 @@ # ============================================================================= # Configuration for Gmail, Calendar, and YouTube webhooks GMAIL_WEBHOOK_ENABLED = os.getenv("GMAIL_WEBHOOK_ENABLED", "false").lower() == "true" -CALENDAR_WEBHOOK_ENABLED = os.getenv("CALENDAR_WEBHOOK_ENABLED", "false").lower() == "true" -YOUTUBE_WEBHOOK_ENABLED = os.getenv("YOUTUBE_WEBHOOK_ENABLED", "false").lower() == "true" +CALENDAR_WEBHOOK_ENABLED = ( + os.getenv("CALENDAR_WEBHOOK_ENABLED", "false").lower() == "true" +) +YOUTUBE_WEBHOOK_ENABLED = ( + os.getenv("YOUTUBE_WEBHOOK_ENABLED", "false").lower() == "true" +) # Webhook URLs (must be set in environment variables) GMAIL_WEBHOOK_URL = os.getenv("GMAIL_WEBHOOK_URL", "") @@ -363,7 +367,9 @@ YOUTUBE_WEBHOOK_URL = os.getenv("YOUTUBE_WEBHOOK_URL", "") # Watch renewal interval (in seconds) - default 6 days -GOOGLE_WATCH_RENEWAL_INTERVAL = int(os.getenv("GOOGLE_WATCH_RENEWAL_INTERVAL", "518400")) +GOOGLE_WATCH_RENEWAL_INTERVAL = int( + os.getenv("GOOGLE_WATCH_RENEWAL_INTERVAL", "518400") +) # Security Settings (base - extended per environment) SECURE_BROWSER_XSS_FILTER = True From 249da19472996682787081ede280716ef424574e Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:43 +0100 Subject: [PATCH 45/68] refactor(google_webhook_views): clean up imports and improve code formatting --- backend/automations/google_webhook_views.py | 37 +++++++++++---------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index b3bfd059..b401c7e2 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -11,19 +11,13 @@ - YouTube: PubSubHubbub with hub.challenge verification """ -import base64 import json import logging -from xml.etree import ElementTree as ET -from django.http import HttpResponse -from django.views.decorators.csrf import csrf_exempt -from rest_framework.decorators import api_view, permission_classes -from rest_framework.permissions import AllowAny -from rest_framework import status from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import AllowAny -from rest_framework.response import Response + +from django.http import HttpResponse from .helpers.calendar_helper import list_upcoming_events from .helpers.gmail_helper import get_history, get_message_details @@ -249,7 +243,9 @@ def calendar_webhook(request): # Fetch recent calendar events try: calendar_id = watch.resource_uri or "primary" - events = list_upcoming_events(access_token, calendar_id=calendar_id, max_results=10) + events = list_upcoming_events( + access_token, calendar_id=calendar_id, max_results=10 + ) if not events: logger.debug(f"No calendar events for user {watch.user.username}") @@ -296,13 +292,19 @@ def calendar_webhook(request): "event_title": event.get("summary", ""), "event_description": event.get("description", ""), "event_location": event.get("location", ""), - "start_time": event.get("start", {}).get("dateTime", ""), - "end_time": event.get("end", {}).get("dateTime", ""), + "start_time": event.get("start", {}).get( + "dateTime", "" + ), + "end_time": event.get("end", {}).get( + "dateTime", "" + ), "attendees": [ a.get("email", "") for a in event.get("attendees", []) ], - "organizer": event.get("organizer", {}).get("email", ""), + "organizer": event.get("organizer", {}).get( + "email", "" + ), "created": created_str, } @@ -347,7 +349,7 @@ def youtube_webhook(request): # Hub verification challenge = request.GET.get("hub.challenge") if challenge: - logger.info(f"YouTube webhook verification: responding with challenge") + logger.info("YouTube webhook verification: responding with challenge") return HttpResponse(challenge, content_type="text/plain", status=200) return HttpResponse("OK", status=200) @@ -363,9 +365,7 @@ def youtube_webhook(request): video_id = video_data["video_id"] channel_id = video_data["channel_id"] - logger.info( - f"YouTube webhook: new video {video_id} from channel {channel_id}" - ) + logger.info(f"YouTube webhook: new video {video_id} from channel {channel_id}") # Find users watching this channel youtube_areas = Area.objects.filter( @@ -428,7 +428,10 @@ def _gmail_message_matches_action(area, message_details): if from_email and from_email.lower() not in message_details["from"].lower(): return False - if subject_contains and subject_contains.lower() not in message_details["subject"].lower(): + if ( + subject_contains + and subject_contains.lower() not in message_details["subject"].lower() + ): return False return True From 825dac29fe563a6b1a415c1d8e23212683327c1f Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:46 +0100 Subject: [PATCH 46/68] fix(GoogleWebhookWatch): format string in __str__ method for better readability --- backend/automations/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/automations/models.py b/backend/automations/models.py index 3de5a3e3..1256d1a8 100755 --- a/backend/automations/models.py +++ b/backend/automations/models.py @@ -575,7 +575,9 @@ class Meta: ] def __str__(self): - return f"{self.service} watch for {self.user.username} (expires {self.expiration})" + return ( + f"{self.service} watch for {self.user.username} (expires {self.expiration})" + ) def is_expiring_soon(self, hours=24): """Check if watch expires within N hours.""" From 68f102b3d1be796ea4e167bed858170063441536 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:50 +0100 Subject: [PATCH 47/68] fix(serializers): improve error message formatting in AreaSerializer validation --- backend/automations/serializers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/automations/serializers.py b/backend/automations/serializers.py index 1b6634b8..2ef68b1e 100755 --- a/backend/automations/serializers.py +++ b/backend/automations/serializers.py @@ -276,10 +276,12 @@ def validate(self, attrs): validate_action_reaction_compatibility(action.name, reaction.name) except serializers.ValidationError as e: # Re-raise with better field targeting for frontend display - error_message = str(e.detail[0]) if hasattr(e, 'detail') and e.detail else str(e) - raise serializers.ValidationError({ - "reaction": f"⚠️ Incompatible combination: {error_message}" - }) + error_message = ( + str(e.detail[0]) if hasattr(e, "detail") and e.detail else str(e) + ) + raise serializers.ValidationError( + {"reaction": f"⚠️ Incompatible combination: {error_message}"} + ) # Validation des configurations si elles sont fournies action_config = attrs.get("action_config", {}) From 97c98db39b4c8a8079177f2e5054d2895c352dfb Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:54 +0100 Subject: [PATCH 48/68] refactor(tasks): improve code readability by formatting conditional statements and log messages --- backend/automations/tasks.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index ca6190a3..662c9856 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -888,7 +888,10 @@ def check_google_calendar_actions(self): ) # Only trigger for events created after last check - if state.last_checked_at and created_dt <= state.last_checked_at: + if ( + state.last_checked_at + and created_dt <= state.last_checked_at + ): continue # Create unique event ID @@ -3976,7 +3979,10 @@ def check_youtube_actions(self): # Fetch latest videos videos = get_latest_videos( - access_token, channel_id, max_results=5, published_after=published_after + access_token, + channel_id, + max_results=5, + published_after=published_after, ) # Update last checked time @@ -4027,7 +4033,9 @@ def check_youtube_actions(self): stats = get_channel_statistics(access_token, channel_id) if not stats: - logger.warning(f"Could not fetch stats for channel {channel_id}") + logger.warning( + f"Could not fetch stats for channel {channel_id}" + ) skipped_count += 1 continue @@ -4450,9 +4458,7 @@ def renew_google_watches(self): watch.save() renewed_count += 1 - logger.info( - f"Renewed {watch.service} watch for {watch.user.username}" - ) + logger.info(f"Renewed {watch.service} watch for {watch.user.username}") else: failed_count += 1 logger.error( @@ -4533,7 +4539,9 @@ def setup_youtube_watches(self): ).first() if existing_watch and not existing_watch.is_expiring_soon(hours=48): - logger.debug(f"YouTube subscription for channel {channel_id} already active") + logger.debug( + f"YouTube subscription for channel {channel_id} already active" + ) skipped_count += 1 continue From 1edb907e612595934872cc803165897289c60e3b Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:33:57 +0100 Subject: [PATCH 49/68] refactor(urls): reorder import statements for better organization --- backend/automations/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/automations/urls.py b/backend/automations/urls.py index bb6e8a4c..faf275c5 100755 --- a/backend/automations/urls.py +++ b/backend/automations/urls.py @@ -27,10 +27,10 @@ from django.urls import include, path from . import github_app_views, views -from .webhooks import webhook_receiver # Import Google webhook views from .google_webhook_views import calendar_webhook, gmail_webhook, youtube_webhook +from .webhooks import webhook_receiver # Create router and register viewsets router = DefaultRouter() From ca175072bfa1b22e0ec55ad60292bfb15c317efa Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:02 +0100 Subject: [PATCH 50/68] fix(validators): improve error message formatting in validate_reaction_config function --- backend/automations/validators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/automations/validators.py b/backend/automations/validators.py index ba76c2cf..5d974053 100755 --- a/backend/automations/validators.py +++ b/backend/automations/validators.py @@ -1312,7 +1312,9 @@ def validate_reaction_config(reaction_name, config): except JsonSchemaValidationError as e: # Provide helpful context for common mistakes - error_message = f"Invalid configuration for reaction '{reaction_name}': {e.message}" + error_message = ( + f"Invalid configuration for reaction '{reaction_name}': {e.message}" + ) # Special handling for Gmail reactions that need message_id if reaction_name in ["gmail_mark_read", "gmail_add_label"]: From 74778ea2183b7fc989ca563ae7be6c5b7b1330fd Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:06 +0100 Subject: [PATCH 51/68] refactor(webhook_receiver): improve log message formatting for YouTube subscription verification --- backend/automations/webhooks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/automations/webhooks.py b/backend/automations/webhooks.py index c5f8bb52..f80a7112 100755 --- a/backend/automations/webhooks.py +++ b/backend/automations/webhooks.py @@ -538,11 +538,15 @@ def webhook_receiver(request: Request, service: str) -> Response: hub_topic = request.GET.get("hub.topic") if hub_mode == "subscribe" and hub_challenge: - logger.info(f"✅ YouTube PubSubHubbub subscription verification for topic: {hub_topic}") + logger.info( + f"✅ YouTube PubSubHubbub subscription verification for topic: {hub_topic}" + ) # Return challenge as plain text return HttpResponse(hub_challenge, content_type="text/plain", status=200) elif hub_mode == "unsubscribe" and hub_challenge: - logger.info(f"✅ YouTube PubSubHubbub unsubscribe verification for topic: {hub_topic}") + logger.info( + f"✅ YouTube PubSubHubbub unsubscribe verification for topic: {hub_topic}" + ) return HttpResponse(hub_challenge, content_type="text/plain", status=200) # Get webhook secret from settings From 4234f4c180d580156e2fb29208724311d120f3d0 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:09 +0100 Subject: [PATCH 52/68] fix(calendar_helper): add missing comma in list_upcoming_events function parameters --- backend/automations/helpers/calendar_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/automations/helpers/calendar_helper.py b/backend/automations/helpers/calendar_helper.py index 12d6af20..57618b4f 100644 --- a/backend/automations/helpers/calendar_helper.py +++ b/backend/automations/helpers/calendar_helper.py @@ -36,7 +36,7 @@ def list_upcoming_events( access_token: str, max_results: int = 10, time_min: Optional[str] = None, - calendar_id: str = "primary" + calendar_id: str = "primary", ) -> List[Dict]: """ List upcoming calendar events. From c0442c1722e80347869b32ae69775815022ee737 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:13 +0100 Subject: [PATCH 53/68] refactor(google_webhook_helper): improve import organization and simplify method calls --- backend/automations/helpers/google_webhook_helper.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py index 87aade10..0d427143 100644 --- a/backend/automations/helpers/google_webhook_helper.py +++ b/backend/automations/helpers/google_webhook_helper.py @@ -20,12 +20,12 @@ import uuid from datetime import datetime, timedelta -from django.conf import settings -from django.utils import timezone from google.oauth2.credentials import Credentials from googleapiclient.discovery import build from googleapiclient.errors import HttpError +from django.utils import timezone + logger = logging.getLogger(__name__) @@ -156,9 +156,7 @@ def create_calendar_watch(access_token, calendar_id, webhook_url, expiration_hou # Create the watch response = ( - service.events() - .watch(calendarId=calendar_id, body=request_body) - .execute() + service.events().watch(calendarId=calendar_id, body=request_body).execute() ) # Extract watch info @@ -238,7 +236,9 @@ def create_youtube_watch(channel_id, webhook_url): hub_url = "https://pubsubhubbub.appspot.com/subscribe" # Topic URL (channel feed) - topic_url = f"https://www.youtube.com/xml/feeds/videos.xml?channel_id={channel_id}" + topic_url = ( + f"https://www.youtube.com/xml/feeds/videos.xml?channel_id={channel_id}" + ) # Subscribe request data = { From 3884e13d3d7721cd5c892ea8a72e6655ebf0790f Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:21 +0100 Subject: [PATCH 54/68] refactor(youtube_helper): simplify function calls and improve code readability --- backend/automations/helpers/youtube_helper.py | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py index 6eb02101..75c47a34 100644 --- a/backend/automations/helpers/youtube_helper.py +++ b/backend/automations/helpers/youtube_helper.py @@ -118,11 +118,7 @@ def get_channel_statistics(access_token: str, channel_id: str) -> Dict: try: service = get_youtube_service(access_token) - results = ( - service.channels() - .list(part="statistics", id=channel_id) - .execute() - ) + results = service.channels().list(part="statistics", id=channel_id).execute() if not results.get("items"): logger.warning(f"Channel not found: {channel_id}") @@ -213,9 +209,7 @@ def search_videos( raise -def post_comment( - access_token: str, video_id: str, comment_text: str -) -> Dict: +def post_comment(access_token: str, video_id: str, comment_text: str) -> Dict: """ Post a comment on a YouTube video. @@ -237,11 +231,7 @@ def post_comment( comment_resource = { "snippet": { "videoId": video_id, - "topLevelComment": { - "snippet": { - "textOriginal": comment_text - } - } + "topLevelComment": {"snippet": {"textOriginal": comment_text}}, } } @@ -255,7 +245,9 @@ def post_comment( comment_data = { "comment_id": result["id"], "text": result["snippet"]["topLevelComment"]["snippet"]["textDisplay"], - "published_at": result["snippet"]["topLevelComment"]["snippet"]["publishedAt"], + "published_at": result["snippet"]["topLevelComment"]["snippet"][ + "publishedAt" + ], "video_id": video_id, } @@ -270,9 +262,7 @@ def post_comment( raise -def add_video_to_playlist( - access_token: str, video_id: str, playlist_id: str -) -> Dict: +def add_video_to_playlist(access_token: str, video_id: str, playlist_id: str) -> Dict: """ Add a video to a YouTube playlist. @@ -294,18 +284,13 @@ def add_video_to_playlist( playlist_item = { "snippet": { "playlistId": playlist_id, - "resourceId": { - "kind": "youtube#video", - "videoId": video_id - } + "resourceId": {"kind": "youtube#video", "videoId": video_id}, } } # Insert playlist item result = ( - service.playlistItems() - .insert(part="snippet", body=playlist_item) - .execute() + service.playlistItems().insert(part="snippet", body=playlist_item).execute() ) item_data = { @@ -345,7 +330,9 @@ def rate_video(access_token: str, video_id: str, rating: str) -> bool: ValueError: If rating is not 'like', 'dislike', or 'none' """ if rating not in ["like", "dislike", "none"]: - raise ValueError(f"Invalid rating: {rating}. Must be 'like', 'dislike', or 'none'") + raise ValueError( + f"Invalid rating: {rating}. Must be 'like', 'dislike', or 'none'" + ) try: service = get_youtube_service(access_token) @@ -376,57 +363,56 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: None if parsing fails """ try: - # Parse XML root = ET.fromstring(xml_string) # Atom namespace ns = { - 'atom': 'http://www.w3.org/2005/Atom', - 'yt': 'http://www.youtube.com/xml/schemas/2015' + "atom": "http://www.w3.org/2005/Atom", + "yt": "http://www.youtube.com/xml/schemas/2015", } # Find entry element - entry = root.find('atom:entry', ns) + entry = root.find("atom:entry", ns) if entry is None: logger.warning("No entry element found in Atom feed") return None # Extract video ID - video_id_elem = entry.find('yt:videoId', ns) + video_id_elem = entry.find("yt:videoId", ns) video_id = video_id_elem.text if video_id_elem is not None else "" # Extract title - title_elem = entry.find('atom:title', ns) + title_elem = entry.find("atom:title", ns) title = title_elem.text if title_elem is not None else "" # Extract channel ID - channel_id_elem = entry.find('yt:channelId', ns) + channel_id_elem = entry.find("yt:channelId", ns) channel_id = channel_id_elem.text if channel_id_elem is not None else "" # Extract author/channel name - author_elem = entry.find('atom:author/atom:name', ns) + author_elem = entry.find("atom:author/atom:name", ns) channel_title = author_elem.text if author_elem is not None else "" # Extract published date - published_elem = entry.find('atom:published', ns) + published_elem = entry.find("atom:published", ns) published_at = published_elem.text if published_elem is not None else "" # Extract updated date - updated_elem = entry.find('atom:updated', ns) + updated_elem = entry.find("atom:updated", ns) updated_at = updated_elem.text if updated_elem is not None else "" # Extract link link_elem = entry.find('atom:link[@rel="alternate"]', ns) - link = link_elem.get('href', '') if link_elem is not None else "" + link = link_elem.get("href", "") if link_elem is not None else "" # Extract thumbnail (media:group/media:thumbnail) thumbnail_url = "" try: - media_ns = {'media': 'http://search.yahoo.com/mrss/'} - thumbnail_elem = entry.find('.//media:thumbnail', media_ns) + media_ns = {"media": "http://search.yahoo.com/mrss/"} + thumbnail_elem = entry.find(".//media:thumbnail", media_ns) if thumbnail_elem is not None: - thumbnail_url = thumbnail_elem.get('url', '') + thumbnail_url = thumbnail_elem.get("url", "") except: pass From 1c7853a41a4aae0e631b13d7690fe097541fc8d3 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:24 +0100 Subject: [PATCH 55/68] refactor(test_google_webhooks): improve import organization and enhance code readability --- backend/scripts/test_google_webhooks.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/scripts/test_google_webhooks.py b/backend/scripts/test_google_webhooks.py index cf2d4cbd..e8868aeb 100644 --- a/backend/scripts/test_google_webhooks.py +++ b/backend/scripts/test_google_webhooks.py @@ -8,6 +8,7 @@ import os import sys + import django # Setup Django environment @@ -17,10 +18,11 @@ from django.contrib.auth import get_user_model from django.utils import timezone -from automations.models import GoogleWebhookWatch, Area + +from automations.models import Area, GoogleWebhookWatch from automations.tasks import ( - setup_google_watches_for_user, renew_google_watches, + setup_google_watches_for_user, setup_youtube_watches, ) @@ -126,12 +128,12 @@ def test_helpers(): try: from automations.helpers.google_webhook_helper import ( - create_gmail_watch, - stop_gmail_watch, create_calendar_watch, - stop_calendar_watch, + create_gmail_watch, create_youtube_watch, renew_youtube_watch, + stop_calendar_watch, + stop_gmail_watch, ) helpers = [ @@ -159,8 +161,8 @@ def test_views(): try: from automations.google_webhook_views import ( - gmail_webhook, calendar_webhook, + gmail_webhook, youtube_webhook, ) @@ -216,8 +218,10 @@ def show_active_watches(): for watch in watches: expiring = watch.is_expiring_soon(hours=24) status = "⏰ EXPIRING SOON" if expiring else "✅ Active" - print(f" {status} | {watch.user.username} | {watch.service} | " - f"expires {watch.expiration.strftime('%Y-%m-%d %H:%M')}") + print( + f" {status} | {watch.user.username} | {watch.service} | " + f"expires {watch.expiration.strftime('%Y-%m-%d %H:%M')}" + ) def show_youtube_areas(): @@ -281,7 +285,9 @@ def main(): print("\n📝 Next steps:") print(" 1. Deploy to production: ./deployment/manage.sh update") print(" 2. Verify domain on Google Search Console (required for Gmail)") - print(" 3. Test with real events (send email, create calendar event, upload video)") + print( + " 3. Test with real events (send email, create calendar event, upload video)" + ) return 0 else: print("❌ Some tests failed. Fix errors before deploying.") From 828174f4bac55d835950b8d6d8c841f54bc71d4d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:34:28 +0100 Subject: [PATCH 56/68] refactor(oauth_views): simplify log message formatting in OAuthCallbackView --- backend/users/oauth_views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/users/oauth_views.py b/backend/users/oauth_views.py index 0e8ab1e5..cac56ded 100755 --- a/backend/users/oauth_views.py +++ b/backend/users/oauth_views.py @@ -250,9 +250,7 @@ def get(self, request, provider: str): # Trigger async task to create Gmail/Calendar watches setup_google_watches_for_user.delay(user.id) - logger.info( - f"Triggered Google webhook setup for user {user.email}" - ) + logger.info(f"Triggered Google webhook setup for user {user.email}") except Exception as e: # Don't fail the OAuth flow if watch setup fails logger.error( From c56cda8429fbb24195d93229b29d04e0ad6aeccc Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:39:54 +0100 Subject: [PATCH 57/68] refactor(gmail_webhook): simplify message matching logic in _gmail_message_matches_action function --- backend/automations/google_webhook_views.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index b401c7e2..a76d3288 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -428,13 +428,10 @@ def _gmail_message_matches_action(area, message_details): if from_email and from_email.lower() not in message_details["from"].lower(): return False - if ( + return not ( subject_contains and subject_contains.lower() not in message_details["subject"].lower() - ): - return False - - return True + ) # gmail_new_from_sender: specific sender elif action_name == "gmail_new_from_sender": @@ -444,7 +441,7 @@ def _gmail_message_matches_action(area, message_details): # gmail_new_with_label: specific label elif action_name == "gmail_new_with_label": required_label = action_config.get("label", "").lower() - message_labels = [l.lower() for l in message_details.get("labels", [])] + message_labels = [label.lower() for label in message_details.get("labels", [])] return required_label in message_labels # gmail_new_with_subject: subject contains text From 19ff923f8f3336afeefa624e8d257b888b156a8a Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:39:59 +0100 Subject: [PATCH 58/68] refactor(check_google_calendar_actions): remove unnecessary target_time_max calculation --- backend/automations/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/automations/tasks.py b/backend/automations/tasks.py index 662c9856..63eafd6b 100755 --- a/backend/automations/tasks.py +++ b/backend/automations/tasks.py @@ -943,7 +943,6 @@ def check_google_calendar_actions(self): # Calculate time window now = timezone.now() target_time_min = now - target_time_max = now + timedelta(minutes=minutes_before + 5) # Fetch upcoming events events = list_upcoming_events( From cae075d1e2ce1603e86d27899b905bb04aecb984 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:40:02 +0100 Subject: [PATCH 59/68] refactor(youtube_helper): enhance XML parsing error handling and improve logging --- backend/automations/helpers/youtube_helper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/automations/helpers/youtube_helper.py b/backend/automations/helpers/youtube_helper.py index 75c47a34..c8e3cd89 100644 --- a/backend/automations/helpers/youtube_helper.py +++ b/backend/automations/helpers/youtube_helper.py @@ -363,8 +363,8 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: None if parsing fails """ try: - # Parse XML - root = ET.fromstring(xml_string) + # Parse XML (YouTube Atom feeds are trusted sources from youtube.com) + root = ET.fromstring(xml_string) # noqa: S314 # nosec B314 # Atom namespace ns = { @@ -413,8 +413,9 @@ def parse_atom_feed_entry(xml_string: str) -> Optional[Dict]: thumbnail_elem = entry.find(".//media:thumbnail", media_ns) if thumbnail_elem is not None: thumbnail_url = thumbnail_elem.get("url", "") - except: - pass + except Exception as e: + # Thumbnail is optional, ignore parsing errors + logger.debug(f"Could not parse thumbnail: {e}") video_data = { "video_id": video_id, From e21e67a9ab80658107fa425585c82c8a4f999953 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:40:06 +0100 Subject: [PATCH 60/68] refactor(test_google_webhooks): reorder imports for better organization --- backend/scripts/test_google_webhooks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/scripts/test_google_webhooks.py b/backend/scripts/test_google_webhooks.py index e8868aeb..91219018 100644 --- a/backend/scripts/test_google_webhooks.py +++ b/backend/scripts/test_google_webhooks.py @@ -16,11 +16,12 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "area_project.settings") django.setup() -from django.contrib.auth import get_user_model -from django.utils import timezone +# Imports must be after django.setup() for Django scripts +from django.contrib.auth import get_user_model # noqa: E402 +from django.utils import timezone # noqa: E402 -from automations.models import Area, GoogleWebhookWatch -from automations.tasks import ( +from automations.models import Area, GoogleWebhookWatch # noqa: E402 +from automations.tasks import ( # noqa: E402 renew_google_watches, setup_google_watches_for_user, setup_youtube_watches, From 8affd154702940bd164e0c8b0f781459e061970a Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:40:48 +0100 Subject: [PATCH 61/68] refactor(Debug): simplify external event trigger message display --- frontend/src/pages/Debug.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/src/pages/Debug.tsx b/frontend/src/pages/Debug.tsx index 949ac71b..d5ff9555 100644 --- a/frontend/src/pages/Debug.tsx +++ b/frontend/src/pages/Debug.tsx @@ -313,9 +313,7 @@ const Debug: React.FC = () => { {triggering ? 'Triggering...' : '⚡ Trigger Now'} ) : ( -
- Triggered by external events -
+
Triggered by external events
)} ))} From 914309a894b9948c3297d035165d82902e6baf4e Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:42:37 +0100 Subject: [PATCH 62/68] refactor(ServiceDetail): update OAuth comments for clarity and completeness --- frontend/src/pages/ServiceDetail.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/ServiceDetail.tsx b/frontend/src/pages/ServiceDetail.tsx index c6df636f..1a1b6ca7 100755 --- a/frontend/src/pages/ServiceDetail.tsx +++ b/frontend/src/pages/ServiceDetail.tsx @@ -149,12 +149,13 @@ const ServiceDetail: React.FC = () => { const logo = resolveLogo(service.logo, service.name); // Check if this service requires OAuth and if it's connected - // Google Calendar and Gmail use the same OAuth as Google + // Google Calendar, Gmail and YouTube use the same OAuth as Google const oauthProviders = [ 'github', 'google', 'gmail', 'google_calendar', + 'youtube', 'twitch', 'slack', 'spotify', @@ -162,7 +163,7 @@ const ServiceDetail: React.FC = () => { ]; const requiresOAuth = service && oauthProviders.includes(service.name.toLowerCase()); - // For gmail and google_calendar, check if 'google' OAuth is connected + // For gmail, google_calendar and youtube, check if 'google' OAuth is connected const getOAuthServiceName = (serviceName: string) => { const lower = serviceName.toLowerCase(); if (lower === 'gmail' || lower === 'google_calendar' || lower === 'youtube') { From d8e9b6496f57186521927c873ea7bbc48a0b36f8 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:51:57 +0100 Subject: [PATCH 63/68] feat(GoogleWebhookBanner): add informative banner for Google webhook services --- .../src/components/GoogleWebhookBanner.tsx | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 frontend/src/components/GoogleWebhookBanner.tsx diff --git a/frontend/src/components/GoogleWebhookBanner.tsx b/frontend/src/components/GoogleWebhookBanner.tsx new file mode 100644 index 00000000..0f2a03a9 --- /dev/null +++ b/frontend/src/components/GoogleWebhookBanner.tsx @@ -0,0 +1,110 @@ +/* + ** EPITECH PROJECT, 2025 + ** Area + ** File description: + ** GoogleWebhookBanner - Informative banner for Google webhook services + */ + +import React from 'react'; + +interface GoogleWebhookBannerProps { + serviceName: string; + isOAuthConnected: boolean; +} + +const GoogleWebhookBanner: React.FC = ({ + serviceName, + isOAuthConnected, +}) => { + // Get display name + const displayName = + serviceName.toLowerCase() === 'youtube' + ? 'YouTube' + : serviceName.toLowerCase() === 'google_calendar' + ? 'Google Calendar' + : serviceName; + + return ( +
+
+ {/* Icon */} +
+ + + +
+ + {/* Content */} +
+
+

Real-time Webhooks

+ + Active + +
+ + {isOAuthConnected ? ( + // Connected state - webhooks active +
+

+ ⚡ {displayName} events are delivered instantly through real-time webhooks +

+ +
+ + + + Webhooks Enabled +
+ +
+
+ + + +
+

Instant notifications

+

+ Your automations react immediately when events occur - no delays from polling +

+
+
+
+
+ ) : ( + // Not connected state +
+

+ ⚠️ Connect your Google account to enable real-time webhook notifications +

+
+ )} +
+
+
+ ); +}; + +export default GoogleWebhookBanner; From cce615fe489ab4b9fefaf3500ee9a3767a5dda7a Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 14:52:03 +0100 Subject: [PATCH 64/68] feat(ServiceDetail): add Google Webhook Banner for Calendar and YouTube services --- frontend/src/pages/ServiceDetail.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/pages/ServiceDetail.tsx b/frontend/src/pages/ServiceDetail.tsx index 1a1b6ca7..1e3afd3c 100755 --- a/frontend/src/pages/ServiceDetail.tsx +++ b/frontend/src/pages/ServiceDetail.tsx @@ -6,6 +6,7 @@ import { useAuthCheck } from '../hooks/useAuthCheck'; import Notification from '../components/Notification'; import GitHubAppSection from '../components/GitHubAppSection'; import NotionWebhookSection from '../components/NotionWebhookSection'; +import GoogleWebhookBanner from '../components/GoogleWebhookBanner'; import { API_BASE, getStoredUser } from '../utils/helper'; import type { User } from '../types'; @@ -338,6 +339,12 @@ const ServiceDetail: React.FC = () => { )} + {/* Google Webhook Banner - Show for Calendar and YouTube services */} + {(service.name.toLowerCase() === 'google_calendar' || + service.name.toLowerCase() === 'youtube') && ( + + )} +

From f12009fbbc53105975d1f4b56bbbc23071900f0d Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 15:16:33 +0100 Subject: [PATCH 65/68] fix(gmail_webhook): handle missing history in Gmail notifications --- backend/automations/google_webhook_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index a76d3288..ceaddd00 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -108,12 +108,12 @@ def gmail_webhook(request): try: history = get_history(access_token, start_history_id=history_id) - if not history: + if not history.get('history'): logger.debug(f"No Gmail history changes for user {watch.user.username}") return HttpResponse("OK", status=200) # Process history changes - for history_item in history: + for history_item in history.get('history', []): messages_added = history_item.get("messagesAdded", []) for msg_info in messages_added: From 0003d1b259f51b55782b0d7d770abbb131919d10 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 15:17:46 +0100 Subject: [PATCH 66/68] fix(google_webhook_helper): update Gmail and Calendar service credential handling --- backend/automations/helpers/google_webhook_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py index 0d427143..36aa0dc2 100644 --- a/backend/automations/helpers/google_webhook_helper.py +++ b/backend/automations/helpers/google_webhook_helper.py @@ -104,8 +104,8 @@ def stop_gmail_watch(access_token, channel_id, resource_id): bool: True if stopped successfully, False otherwise """ try: - service = build("gmail", "v1", credentials=None, static_discovery=False) - service._http.credentials = type("Credentials", (), {"token": access_token})() + creds = Credentials(token=access_token) + service = build("gmail", "v1", credentials=creds, static_discovery=False) service.users().stop(userId="me").execute() @@ -196,8 +196,8 @@ def stop_calendar_watch(access_token, channel_id, resource_id): bool: True if stopped successfully, False otherwise """ try: - service = build("calendar", "v3", credentials=None, static_discovery=False) - service._http.credentials = type("Credentials", (), {"token": access_token})() + creds = Credentials(token=access_token) + service = build("calendar", "v3", credentials=creds, static_discovery=False) request_body = {"id": channel_id, "resourceId": resource_id} From cc8f9d0861f785dc165588cd1b5fa4a85e9f2668 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 15:18:44 +0100 Subject: [PATCH 67/68] fix(google_webhook_helper): clarify parameters for stopping Gmail watches --- backend/automations/helpers/google_webhook_helper.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/automations/helpers/google_webhook_helper.py b/backend/automations/helpers/google_webhook_helper.py index 36aa0dc2..92bd6a73 100644 --- a/backend/automations/helpers/google_webhook_helper.py +++ b/backend/automations/helpers/google_webhook_helper.py @@ -95,10 +95,15 @@ def stop_gmail_watch(access_token, channel_id, resource_id): """ Stop (delete) a Gmail watch. + NOTE: Gmail API does not support stopping individual watches by channel_id. + The users().stop() endpoint stops ALL watches for the user. + The channel_id and resource_id parameters are kept for API consistency + and logging purposes only. + Args: access_token (str): Valid Google OAuth2 access token - channel_id (str): Channel ID of the watch to stop - resource_id (str): Resource ID returned when watch was created + channel_id (str): Channel ID (used for logging only) + resource_id (str): Resource ID (used for logging only) Returns: bool: True if stopped successfully, False otherwise From d7e63dd20663ed6773066920a9df7694c7be6796 Mon Sep 17 00:00:00 2001 From: Paul-Antoine SALMON Date: Sun, 2 Nov 2025 15:20:14 +0100 Subject: [PATCH 68/68] fix(gmail_webhook): standardize quotes for history retrieval checks --- backend/automations/google_webhook_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/automations/google_webhook_views.py b/backend/automations/google_webhook_views.py index ceaddd00..3b04cc12 100644 --- a/backend/automations/google_webhook_views.py +++ b/backend/automations/google_webhook_views.py @@ -108,12 +108,12 @@ def gmail_webhook(request): try: history = get_history(access_token, start_history_id=history_id) - if not history.get('history'): + if not history.get("history"): logger.debug(f"No Gmail history changes for user {watch.user.username}") return HttpResponse("OK", status=200) # Process history changes - for history_item in history.get('history', []): + for history_item in history.get("history", []): messages_added = history_item.get("messagesAdded", []) for msg_info in messages_added: