[ENG-12449][ENG-12705] Project PR: Post-ENTER-P2

admin/notifications/forms.py

@@ -3,6 +3,9 @@
 from website import settings
 import json
 
+# SendGrid Mail Send accepts at most 1000 personalizations (one per recipient in bulk mode) per request.
+SENDGRID_MAX_PERSONALIZATIONS = 1000
+
 
 class NotificationTypeForm(forms.ModelForm):
     class Meta:
@@ -11,6 +14,13 @@ class Meta:
 
 
 class NotificationCampaignCreateForm(forms.ModelForm):
+    # Execution values the create page switches to when "SendGrid Bulk" is selected.
+    BULK_EXECUTION_DEFAULTS = {
+        'batch_size': settings.DEFAULT_CAMPAIGN_BULK_BATCH_SIZE,
+        'max_queued_batches': settings.DEFAULT_CAMPAIGN_BULK_MAX_QUEUED_BATCHES,
+        'dispatch_interval': settings.DEFAULT_CAMPAIGN_BULK_DISPATCH_INTERVAL,
+    }
+
     context = forms.CharField(
         required=False,
         widget=forms.Textarea(attrs={'rows': 8}),
@@ -49,7 +59,9 @@ class NotificationCampaignCreateForm(forms.ModelForm):
         required=False,
         initial=False,
         help_text=(
-            'SendGrid caps personalizations at about 1000 per request - keep Batch Size ≤ 1000 for bulk requests'
+            f'Send each batch as a single SendGrid request (max {SENDGRID_MAX_PERSONALIZATIONS} recipients) '
+            'and track delivery via the SendGrid webhook. Selecting it switches Batch Size, Max Queued Batches '
+            'and Dispatch Interval to bulk defaults that avoid SendGrid rate limiting.'
         ),
     )
 
@@ -65,13 +77,29 @@ class NotificationCampaignCreateForm(forms.ModelForm):
         help_text='The time in seconds between dispatches of campaign batches.',
     )
 
+    log_email_recipients = forms.CharField(
+        required=False,
+        initial='',
+        help_text='List of email addresses to receive campaign log emails, separated by commas.',
+    )
+
     class Meta:
         model = NotificationCampaign
         fields = (
             'name',
             'notification_type',
         )
 
+    def clean(self):
+        cleaned_data = super().clean()
+        batch_size = cleaned_data.get('batch_size')
+        if cleaned_data.get('sendgrid_bulk') and batch_size and batch_size > SENDGRID_MAX_PERSONALIZATIONS:
+            self.add_error(
+                'batch_size',
+                f'Batch Size cannot exceed {SENDGRID_MAX_PERSONALIZATIONS} for SendGrid bulk requests.',
+            )
+        return cleaned_data
+
     def clean_context(self):
         value = self.cleaned_data['context'] or '{}'
         try:

admin/notifications/views.py

@@ -1,7 +1,7 @@
 import re
 import json
 from collections import defaultdict
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone as dt_timezone
 from django.urls import reverse_lazy
 from django.utils import timezone
 from django.db.models import Q, F, Subquery
@@ -19,7 +19,7 @@
 from mako.parsetree import ControlLine
 from string import Formatter
 from osf.email import _render_email_html
-from osf.email.notification_campaign import FILTER_PRESETS, counter_subquery, build_campaign_filter_query
+from osf.email.notification_campaign import FILTER_PRESETS, counter_subquery, build_campaign_filter_query, get_campaign_recipient_stats
 from website import settings
 from urllib.parse import urlencode
 
@@ -392,6 +392,21 @@ def get_object(self, queryset=None):
     def get_context_data(self, *args, **kwargs):
         notification_campaign = self.get_object()
         metadata = notification_campaign.metadata or {}
+        stats = get_campaign_recipient_stats(notification_campaign.id)
+        queued = stats['queued_count']
+        awaiting = stats['awaiting_count']
+        # Recipient rows are the source of truth while they exist; the stored counters
+        # are a snapshot for campaigns whose recipients were purged.
+        if stats['recipient_count']:
+            recipient_count = stats['recipient_count']
+            sent_count = stats['sent_count']
+            failed_count = stats['failed_count']
+            skipped_count = stats['skipped_count']
+        else:
+            recipient_count = notification_campaign.recipient_count
+            sent_count = notification_campaign.sent_count
+            failed_count = notification_campaign.failed_count
+            skipped_count = notification_campaign.skipped_count
 
         context = {
             'notification_campaign': notification_campaign,
@@ -401,9 +416,12 @@ def get_context_data(self, *args, **kwargs):
                 ('Created By', notification_campaign.created_by),
                 ('Developer reminder ', 'Sent' if notification_campaign.developer_reminder_sent else ''),
                 ('Status', notification_campaign.get_status_display()),
-                ('Recipients', notification_campaign.recipient_count),
-                ('Sent', notification_campaign.sent_count),
-                ('Failed', notification_campaign.failed_count),
+                ('Recipients', recipient_count),
+                ('Sent', sent_count),
+                ('Queued', queued),
+                ('Awaiting Delivery', awaiting),
+                ('Failed', failed_count),
+                ('Skipped', skipped_count),
                 ('Retries', notification_campaign.retries),
                 ('Created', notification_campaign.created_at),
                 ('Started', notification_campaign.started_at),
@@ -430,61 +448,114 @@ def get_context_data(self, *args, **kwargs):
                 for k, v in metadata.items()
                 if k not in {'filters', 'context', 'execution', 'template'}
             },
-            'allow_restart_stuck': True if timezone.now() - notification_campaign.updated_at > timedelta(minutes=15) else False,
             'delete_allowed': notification_campaign.status != NotificationCampaignStatus.RUNNING,
             'start_allowed': notification_campaign.status == NotificationCampaignStatus.CREATED and metadata.get('recipients_creation_finished', False),
         }
 
         if notification_campaign.status != NotificationCampaignStatus.CREATED:
-            processed = notification_campaign.sent_count + notification_campaign.failed_count
-            pending = max(notification_campaign.recipient_count - processed, 0)
+            processed = sent_count + failed_count + skipped_count
+            pending = max(
+                recipient_count - processed - queued - awaiting,
+                0,
+            )
+            awaiting_delivery = (
+                notification_campaign.status == NotificationCampaignStatus.RUNNING
+                and pending == 0
+                and queued == 0
+                and awaiting > 0
+            )
 
             sent_percent = (
-                notification_campaign.sent_count / notification_campaign.recipient_count * 100
-                if notification_campaign.recipient_count else 0
+                sent_count / recipient_count * 100
+                if recipient_count else 0
             )
             failed_percent = (
-                notification_campaign.failed_count / notification_campaign.recipient_count * 100
-                if notification_campaign.recipient_count else 0
+                failed_count / recipient_count * 100
+                if recipient_count else 0
+            )
+            skipped_percent = (
+                skipped_count / recipient_count * 100
+                if recipient_count else 0
+            )
+            queued_percent = (
+                queued / recipient_count * 100
+                if recipient_count else 0
+            )
+            awaiting_percent = (
+                awaiting / recipient_count * 100
+                if recipient_count else 0
             )
 
-            pending_percent = max(100 - sent_percent - failed_percent, 0)
+            pending_percent = max(
+                100 - sent_percent - failed_percent - skipped_percent - queued_percent - awaiting_percent,
+                0,
+            )
             elapsed = None
             speed = None
             eta = None
             estimated_finish = None
             last_activity_ago = None
+            last_sent_at = None
+            restart_stuck_after = None
+            restart_stuck_suggested = False
             failure_rate = None
+            skip_rate = None
 
             if notification_campaign.started_at:
                 end_time = notification_campaign.completed_at or timezone.now()
                 elapsed = end_time - notification_campaign.started_at
                 elapsed_seconds = elapsed.total_seconds()
                 if processed > 0 and elapsed_seconds > 0:
                     speed = processed / elapsed_seconds
-                    if pending:
-                        eta = timedelta(seconds=int(pending / speed))
+                    remaining = pending + queued
+                    if remaining:
+                        eta = timedelta(seconds=int(remaining / speed))
                         estimated_finish = timezone.now() + eta
 
-            if notification_campaign.updated_at:
-                last_activity_ago = timezone.now() - notification_campaign.updated_at
+                if (stats['last_sent_at'] or 0) >= int(notification_campaign.started_at.timestamp()):
+                    last_sent_at = datetime.fromtimestamp(stats['last_sent_at'], tz=dt_timezone.utc)
+                last_activity_ago = timezone.now() - (last_sent_at or notification_campaign.started_at)
+                # Suggest "Restart stuck" when nothing was sent within two dispatch intervals
+                restart_stuck_after = 2 * metadata.get('execution', {}).get('dispatch_interval', settings.CAMPAIGN_DISPATCH_INTERVAL)
+                restart_stuck_suggested = (
+                    notification_campaign.status == NotificationCampaignStatus.RUNNING
+                    and not awaiting_delivery
+                    and last_activity_ago > timedelta(seconds=restart_stuck_after)
+                )
+
             if processed:
-                failure_rate = notification_campaign.failed_count / processed * 100
+                failure_rate = failed_count / processed * 100
+                skip_rate = skipped_count / processed * 100
             else:
                 failure_rate = 0
+                skip_rate = 0
 
             context.update({
+                'recipient_count': recipient_count,
+                'sent_count': sent_count,
+                'failed_count': failed_count,
+                'skipped_count': skipped_count,
                 'processed': processed,
                 'pending': pending,
+                'queued': queued,
+                'awaiting': awaiting,
+                'awaiting_delivery': awaiting_delivery,
                 'sent_percent': sent_percent,
                 'failed_percent': failed_percent,
+                'skipped_percent': skipped_percent,
+                'queued_percent': queued_percent,
+                'awaiting_percent': awaiting_percent,
                 'pending_percent': pending_percent,
                 'elapsed': elapsed,
                 'speed': speed,
                 'eta': eta,
                 'estimated_finish': estimated_finish,
                 'last_activity_ago': last_activity_ago,
+                'last_sent_at': last_sent_at,
+                'restart_stuck_after': restart_stuck_after,
+                'restart_stuck_suggested': restart_stuck_suggested,
                 'failure_rate': failure_rate,
+                'skip_rate': skip_rate,
             })
 
         return context
@@ -587,6 +658,7 @@ def form_valid(self, form):
                 'time_window': form.cleaned_data['time_window'],
                 'max_queued_batches': form.cleaned_data['max_queued_batches'],
                 'dispatch_interval': form.cleaned_data['dispatch_interval'],
+                'log_email_recipients': form.cleaned_data['log_email_recipients'],
             },
             'sendgrid_bulk': form.cleaned_data.get('sendgrid_bulk', False),
             'recipients_creation_finished': False,
@@ -634,6 +706,7 @@ def get_context_data(self, **kwargs):
         context['filters'] = []
         context['predefined_filters'] = FILTER_PRESETS.keys()
         context['default_context'] = json.dumps({'domain': settings.DOMAIN, 'osf_contact_email': settings.OSF_CONTACT_EMAIL}, indent=4)
+        context['bulk_defaults'] = NotificationCampaignCreateForm.BULK_EXECUTION_DEFAULTS
         return context
 
 
@@ -690,10 +763,8 @@ def get_queryset(self):
         if not campaign_id:
             return NotificationCampaignRecipient.objects.none()
         query = {'campaign_id': campaign_id}
-        if status == NotificationCampaignRecipientStatus.SENT:
-            query['status'] = NotificationCampaignRecipientStatus.SENT
-        elif status == NotificationCampaignRecipientStatus.FAILED:
-            query['status__in'] = [NotificationCampaignRecipientStatus.FAILED, NotificationCampaignRecipientStatus.SKIPPED]
+        if status in NotificationCampaignRecipientStatus.values:
+            query['status'] = status
 
         qs = NotificationCampaignRecipient.objects.filter(**query)