Compare commits

...

3 Commits

Author SHA1 Message Date
Matthew Meszaros
2999f21a26 feat: bound dashboard overall aggregates to the requested campaign send window
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-14 03:21:14 +00:00
Matthew Meszaros
b41c015079 feat: share complaint bounce and tampering aggregate scans in warmup health counts
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-14 03:21:13 +00:00
Matthew Meszaros
d3c1715f64 feat: bound task daily ledgers and mailbox send clocks to existing time indexes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-14 03:21:02 +00:00
7 changed files with 612 additions and 51 deletions

View File

@ -0,0 +1,134 @@
package repository
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/warmbly/warmbly/internal/infrastructure/db"
"github.com/warmbly/warmbly/internal/models"
)
func legacyDashboardOverallSQL(scope *models.CampaignScope, mailboxes []uuid.UUID) string {
return `SELECT
COUNT(CASE WHEN ccp.sent_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND NOT ccp.opened_machine AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
COUNT(CASE WHEN ccp.opened_at IS NOT NULL AND ccp.opened_machine AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
COUNT(CASE WHEN ccp.clicked_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
COUNT(CASE WHEN ccp.clicked_at IS NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 AND EXISTS (
SELECT 1 FROM email_link_clicks lc WHERE lc.campaign_id=ccp.campaign_id AND lc.contact_id=ccp.contact_id AND lc.sequence_id=ccp.sequence_id AND lc.machine
) AND NOT EXISTS (
SELECT 1 FROM email_link_clicks lc WHERE lc.campaign_id=ccp.campaign_id AND lc.contact_id=ccp.contact_id AND lc.sequence_id=ccp.sequence_id AND NOT lc.machine
) THEN 1 END),
COUNT(CASE WHEN ccp.replied_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
COUNT(CASE WHEN ccp.bounced_at IS NOT NULL AND ccp.sent_at >= $2 AND ccp.sent_at <= $3 THEN 1 END),
(SELECT COUNT(*) FROM campaigns ac WHERE ac.organization_id=$1 AND ac.status='active'` + inCampaignScope("ac.id", scope, 4) + `),
(SELECT COUNT(*) FROM email_accounts WHERE organization_id=$1 AND status='active'` + inMailboxes("id", mailboxes, 4+len(scopeArgs(scope))) + `)
FROM campaign_contact_progress ccp JOIN campaigns c ON c.id=ccp.campaign_id
JOIN sequences s ON s.id=ccp.sequence_id AND s.kind='email'
WHERE c.organization_id=$1` + inCampaignScope("c.id", scope, 4)
}
func TestLiveDashboardOverallBoundsHistoryAndPreservesScopedCounts(t *testing.T) {
f, _ := lineageFixture(t)
ctx := t.Context()
sf := newSharedOrgFixture(t, f.pool)
foreign := newSharedOrgFixture(t, f.pool)
repo := NewAnalyticsRepository(&db.DB{Pool: f.pool})
from := time.Date(2026, time.March, 1, 0, 0, 0, 0, time.UTC)
to := from.Add(24 * time.Hour)
exec := func(query string, args ...any) {
t.Helper()
if _, err := f.pool.Exec(ctx, query, args...); err != nil {
t.Fatal(err)
}
}
sequence := uuid.New()
for _, fixture := range []*sharedOrgFixture{sf, foreign} {
step := sequence
if fixture == foreign {
step = uuid.New()
}
exec(`UPDATE campaigns SET status='active' WHERE organization_id=$1`, fixture.org)
exec(`INSERT INTO sequences(id,campaign_id,organization_id,name,subject,body_plain,body_html)
VALUES($1,$2,$3,'Report','Report','','')`, step, fixture.campaign, fixture.org)
for i, sent := range []any{from.Add(-time.Microsecond), from, from.Add(time.Hour), to, to.Add(time.Microsecond), nil} {
contact := addLead(t, fixture, uuid.NewString()+"@test.local", "valid", true)
exec(`INSERT INTO campaign_contact_progress(campaign_id,contact_id,sequence_id,sent_at,opened_at,opened_machine,clicked_at,replied_at,bounced_at)
VALUES($1,$2,$3,$4,$5,$6,$7,$5,$5)`, fixture.campaign, contact, step, sent, to.Add(48*time.Hour), i%2 == 1,
func() any {
if i%2 == 0 {
return to.Add(time.Hour)
}
return nil
}())
exec(`INSERT INTO email_link_clicks(task_id,campaign_id,contact_id,sequence_id,destination,clicked_at,machine)
VALUES($1,$2,$3,$4,'https://test.local',$5,true)`, uuid.New(), fixture.campaign, contact, step, to.Add(time.Hour))
}
}
account := uuid.New()
exec(`INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider,status)
VALUES($1,$2,$3,$4,'Report','','','smtp_imap','active')`, account, sf.owner, sf.org, account.String()+"@test.local")
t.Cleanup(func() {
if _, err := f.pool.Exec(context.Background(), `DELETE FROM email_accounts WHERE id=$1`, account); err != nil {
t.Error(err)
}
})
action := uuid.New()
exec(`INSERT INTO sequences(id,campaign_id,organization_id,name,subject,body_plain,body_html,kind)
VALUES($1,$2,$3,'Action','','','','action')`, action, sf.campaign, sf.org)
exec(`INSERT INTO campaign_contact_progress(campaign_id,contact_id,sequence_id,sent_at,opened_at)
VALUES($1,$2,$3,$4,$4)`, sf.campaign, sf.contact, action, from)
for _, window := range [][2]time.Time{{from, to}, {to.Add(24 * time.Hour), to.Add(48 * time.Hour)}, {to, from}, {from, from}} {
for _, scope := range []*models.CampaignScope{nil, {CampaignIDs: []uuid.UUID{sf.campaign}}, {}, {CampaignIDs: []uuid.UUID{foreign.campaign}}} {
for _, mailboxes := range [][]uuid.UUID{nil, {}, {account}, {f.sender}} {
params := append(append([]any{sf.org, window[0], window[1]}, scopeArgs(scope)...), mailboxArgs(mailboxes)...)
var want models.DashboardOverallStats
err := f.pool.QueryRow(ctx, legacyDashboardOverallSQL(scope, mailboxes), params...).Scan(&want.TotalEmailsSent,
&want.TotalOpens, &want.MachineOpens, &want.TotalClicks, &want.MachineClicks, &want.TotalReplies,
&want.TotalBounces, &want.ActiveCampaigns, &want.ActiveAccounts)
if err != nil {
t.Fatal(err)
}
if want.TotalEmailsSent > 0 {
want.OpenRate = float64(want.TotalOpens) / float64(want.TotalEmailsSent) * 100
want.ClickRate = float64(want.TotalClicks) / float64(want.TotalEmailsSent) * 100
want.ReplyRate = float64(want.TotalReplies) / float64(want.TotalEmailsSent) * 100
want.BounceRate = float64(want.TotalBounces) / float64(want.TotalEmailsSent) * 100
}
got, xerr := repo.GetDashboardOverallStats(ctx, sf.org, window[0], window[1], scope, mailboxes)
if xerr != nil || got == nil || !reflect.DeepEqual(*got, want) {
t.Fatalf("window=%v scope=%+v mailboxes=%v: got=%+v want=%+v err=%v", window, scope, mailboxes, got, want, xerr)
}
}
}
}
exec(`WITH added AS (
INSERT INTO contacts(id,user_id,organization_id,email,first_name,last_name,company,phone,custom_fields,updated_at,created_at)
SELECT gen_random_uuid(),$1,$2,g::text || '@history.test','','','','','{}',NOW(),NOW() FROM generate_series(1,20000) g RETURNING id)
INSERT INTO campaign_contact_progress(campaign_id,contact_id,sequence_id,sent_at)
SELECT $3,id,$4,$5::timestamptz-interval '90 days' FROM added`, sf.owner, sf.org, sf.campaign, sequence, from)
exec(`ANALYZE campaign_contact_progress`)
conn, err := pgx.ConnectConfig(ctx, f.pool.Config().ConnConfig.Copy())
if err != nil {
t.Fatal(err)
}
defer conn.Close(ctx)
if _, err := conn.Exec(ctx, `SET plan_cache_mode=force_generic_plan`); err != nil {
t.Fatal(err)
}
before := legacyDashboardOverallSQL(nil, nil)
after := before + ` AND ccp.sent_at >= $2 AND ccp.sent_at <= $3`
args := fmt.Sprintf("'%s','%s'::timestamptz,'%s'::timestamptz", sf.org, from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano))
beforeRows, beforeMS, _ := taskPressureExplain(t, conn, before, args, "campaign_contact_progress")
afterRows, afterMS, plan := taskPressureExplain(t, conn, after, args, "campaign_contact_progress")
t.Logf("synthetic 20000 historical progress rows: generic plan before %.3fms / %.0f progress rows, after %.3fms / %.0f progress rows", beforeMS, beforeRows, afterMS, afterRows)
if afterRows >= beforeRows || !strings.Contains(plan, "sent_at >=") {
t.Fatalf("expected bounded progress scan: before=%v after=%v plan=%s", beforeRows, afterRows, plan)
}
}

View File

@ -9,12 +9,6 @@ import (
"github.com/jackc/pgx/v5"
)
// sendClaimClock feeds a claimed-but-not-yet-handed-off campaign send into the
// min-gap clock ($1 is the mailbox id array), so the next pass of any campaign
// sees the mailbox as just used.
const sendClaimClock = `SELECT ea.id, ea.send_claimed_at FROM email_accounts ea
WHERE ea.id = ANY($1) AND ea.send_claimed_at IS NOT NULL`
// MailboxSendClaimer takes a mailbox for one campaign send.
type MailboxSendClaimer interface {
ClaimMailboxSend(ctx context.Context, mailboxID uuid.UUID, window time.Duration) (bool, error)

View File

@ -796,6 +796,7 @@ func (r *analyticsRepository) GetDashboardOverallStats(ctx context.Context, orgI
JOIN campaigns c ON c.id = ccp.campaign_id
JOIN sequences s ON s.id = ccp.sequence_id AND s.kind = 'email'
WHERE c.organization_id = $1` + inCampaignScope("c.id", scope, 4) + `
AND ccp.sent_at >= $2 AND ccp.sent_at <= $3
`
params := append(append([]any{orgID, from, to}, scopeArgs(scope)...), mailboxArgs(mailboxes)...)

View File

@ -490,7 +490,8 @@ const taskDispatchedEmail = `(t.task_type <> 'campaign' OR t.message_id <> '' OR
SELECT 1 FROM campaign_contact_progress ccp WHERE ccp.dispatch_task_id = t.id))`
const campaignSendToday = `t.status = 'completed' AND t.task_type = 'campaign'
AND (t.completed_at AT TIME ZONE 'UTC')::date = (NOW() AT TIME ZONE 'UTC')::date
AND t.completed_at >= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date::timestamp AT TIME ZONE 'UTC'
AND t.completed_at < ((CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date + 1)::timestamp AT TIME ZONE 'UTC'
AND ` + taskDispatchedEmail
// CountCampaignEmailsSentToday counts the campaign emails a mailbox dispatched
@ -513,26 +514,26 @@ func (r *taskRepository) CountCampaignEmailsSentToday(ctx context.Context, accou
return count, err
}
const lastEmailTimesSQL = `
SELECT m.id, GREATEST(t.completed_at, ea.send_claimed_at)
FROM unnest($1::uuid[]) AS m(id)
LEFT JOIN LATERAL (
SELECT t.completed_at FROM tasks t
WHERE t.email_account_id = m.id AND t.status = 'completed'
AND t.completed_at IS NOT NULL AND ` + taskDispatchedEmail + `
ORDER BY t.completed_at DESC LIMIT 1
) t ON true
LEFT JOIN email_accounts ea ON ea.id = m.id
WHERE t.completed_at IS NOT NULL OR ea.send_claimed_at IS NOT NULL
`
// GetLastEmailTimes is the min-gap clock for a whole pool in one query.
func (r *taskRepository) GetLastEmailTimes(ctx context.Context, accountIDs []uuid.UUID) (map[uuid.UUID]time.Time, error) {
out := make(map[uuid.UUID]time.Time, len(accountIDs))
if len(accountIDs) == 0 {
return out, nil
}
query := `
SELECT id, MAX(at) FROM (
SELECT t.email_account_id AS id, t.completed_at AS at
FROM tasks t
WHERE t.email_account_id = ANY($1)
AND t.status = 'completed'
AND t.completed_at IS NOT NULL
AND ` + taskDispatchedEmail + `
UNION ALL
` + sendClaimClock + `
) last
GROUP BY id
`
rows, err := r.db.Query(ctx, query, accountIDs)
rows, err := r.db.Query(ctx, lastEmailTimesSQL, accountIDs)
if err != nil {
return nil, err
}
@ -620,7 +621,9 @@ func (r *taskRepository) CampaignSendsPerDayByAccounts(ctx context.Context, acco
// Queued placement probes hold today's UTC capacity until they send or cancel.
const placementSendToday = `t.task_type = 'placement'
AND ((t.status = 'completed' AND (t.completed_at AT TIME ZONE 'UTC')::date = (NOW() AT TIME ZONE 'UTC')::date)
AND ((t.status = 'completed'
AND t.completed_at >= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date::timestamp AT TIME ZONE 'UTC'
AND t.completed_at < ((CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date + 1)::timestamp AT TIME ZONE 'UTC')
OR (t.status IN ('pending', 'active') AND t.scheduled_at < (((NOW() AT TIME ZONE 'UTC')::date + 1)::timestamp AT TIME ZONE 'UTC')))`
const placementSentTodaySQL = `SELECT COUNT(*) FROM tasks t WHERE ` + placementSendToday
@ -714,7 +717,8 @@ func (r *taskRepository) CountEmailsSentToday(ctx context.Context, accountID uui
FROM tasks t
WHERE t.email_account_id = $1
AND t.status = 'completed'
AND DATE(t.completed_at) = CURRENT_DATE
AND t.completed_at >= CURRENT_DATE::timestamptz
AND t.completed_at < (CURRENT_DATE + 1)::timestamptz
AND ` + taskDispatchedEmail + `
`
@ -730,7 +734,8 @@ func (r *taskRepository) CountWarmupEmailsSentToday(ctx context.Context, account
WHERE email_account_id = $1
AND status = 'completed'
AND task_type = 'warmup'
AND DATE(completed_at) = CURRENT_DATE
AND completed_at >= CURRENT_DATE::timestamptz
AND completed_at < (CURRENT_DATE + 1)::timestamptz
`
var count int
@ -744,11 +749,13 @@ func (r *taskRepository) CountWarmupEmailsSentToday(ctx context.Context, account
func (r *taskRepository) GetLastEmailTime(ctx context.Context, accountID uuid.UUID) (*time.Time, error) {
query := `
SELECT GREATEST(
(SELECT MAX(t.completed_at)
(SELECT t.completed_at
FROM tasks t
WHERE t.email_account_id = $1
AND t.status = 'completed'
AND ` + taskDispatchedEmail + `),
AND t.completed_at IS NOT NULL
AND ` + taskDispatchedEmail + `
ORDER BY t.completed_at DESC LIMIT 1),
(SELECT send_claimed_at FROM email_accounts WHERE id = $1))
`
@ -762,6 +769,18 @@ func (r *taskRepository) GetLastEmailTime(ctx context.Context, accountID uuid.UU
return lastTime, err
}
const lastSendTimesSQL = `
SELECT m.id, t.completed_at
FROM unnest($1::uuid[]) AS m(id)
CROSS JOIN LATERAL (
SELECT t.completed_at FROM tasks t
WHERE t.email_account_id = m.id
AND t.status = 'completed' AND t.task_type = $2::task_type
AND t.completed_at IS NOT NULL AND ` + taskDispatchedEmail + `
ORDER BY t.completed_at DESC LIMIT 1
) t
`
// GetLastSendTimes returns the most recent completed send per account for a
// task type, in one query.
//
@ -777,18 +796,7 @@ func (r *taskRepository) GetLastSendTimes(ctx context.Context, accountIDs []uuid
return out, nil
}
query := `
SELECT t.email_account_id, MAX(t.completed_at)
FROM tasks t
WHERE t.email_account_id = ANY($1)
AND t.status = 'completed'
AND t.task_type = $2::task_type
AND t.completed_at IS NOT NULL
AND ` + taskDispatchedEmail + `
GROUP BY t.email_account_id
`
rows, err := r.db.Query(ctx, query, accountIDs, taskType)
rows, err := r.db.Query(ctx, lastSendTimesSQL, accountIDs, taskType)
if err != nil {
return nil, err
}

View File

@ -863,9 +863,18 @@ func (r *warmupRepository) ListParticipantHealth(ctx context.Context) ([]models.
return out, rows.Err()
}
// HealthMetricCounts runs the aggregates as one statement; each keeps its own
// predicate so the (type, created_at) indexes still serve it. Placement is read
// through the sender's verified receipts, so its rate is over deliveries.
const healthFeedbackCountsSQL = `SELECT
COUNT(*) FILTER (WHERE de.event_type = 'complaint') AS complaints,
COUNT(*) FILTER (WHERE de.event_type = 'bounce') AS bounces
FROM deliverability_events de JOIN tasks t ON t.id = de.task_id
WHERE t.email_account_id = $1 AND de.created_at >= $3 AND de.event_type IN ('complaint', 'bounce')`
const healthTamperingCountsSQL = `SELECT
COUNT(*) FILTER (WHERE kind = 'deletion') AS deletions,
COUNT(*) FILTER (WHERE kind = 'spam_flag') AS spam_flags
FROM warmup_tampering_events WHERE email_account_id = $1 AND created_at >= $2`
// HealthMetricCounts shares scans only between signals with identical predicates.
func (r *warmupRepository) HealthMetricCounts(ctx context.Context, accountID uuid.UUID, since7d, since30d time.Time) (models.WarmupHealthCounts, error) {
query := `
SELECT
@ -875,20 +884,14 @@ func (r *warmupRepository) HealthMetricCounts(ctx context.Context, accountID uui
WHERE reported_account_id = $1 AND created_at >= $2 AND report_type = 'spam_placement'),
(SELECT COUNT(*) FROM warmup_spam_reports
WHERE reported_account_id = $1 AND created_at >= $2 AND report_type IN ('user_complaint', 'spam', 'spam_folder')),
(SELECT COUNT(*) FILTER (WHERE de.event_type = 'complaint') FROM deliverability_events de
JOIN tasks t ON t.id = de.task_id
WHERE t.email_account_id = $1 AND de.created_at >= $3 AND de.event_type IN ('complaint', 'bounce')),
(SELECT COUNT(*) FILTER (WHERE de.event_type = 'bounce') FROM deliverability_events de
JOIN tasks t ON t.id = de.task_id
WHERE t.email_account_id = $1 AND de.created_at >= $3 AND de.event_type IN ('complaint', 'bounce')),
feedback.complaints, feedback.bounces,
(SELECT COUNT(*) FROM tasks
WHERE email_account_id = $1 AND status = 'completed' AND completed_at >= $3),
(SELECT COUNT(*) FILTER (WHERE kind = 'deletion') FROM warmup_tampering_events
WHERE email_account_id = $1 AND created_at >= $2),
(SELECT COUNT(*) FILTER (WHERE kind = 'spam_flag') FROM warmup_tampering_events
WHERE email_account_id = $1 AND created_at >= $2),
tampering.deletions, tampering.spam_flags,
placed.major, placed.major_spam, placed.other, placed.other_spam
FROM (` + placementEvidenceSQL("$1", "$2") + `) placed
CROSS JOIN (` + healthFeedbackCountsSQL + `) feedback
CROSS JOIN (` + healthTamperingCountsSQL + `) tampering
`
var c models.WarmupHealthCounts
p := &c.Placement

View File

@ -0,0 +1,294 @@
package repository
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const legacyCampaignSendToday = `t.status = 'completed' AND t.task_type = 'campaign'
AND (t.completed_at AT TIME ZONE 'UTC')::date = (NOW() AT TIME ZONE 'UTC')::date
AND ` + taskDispatchedEmail
const legacyPlacementSendToday = `t.task_type = 'placement'
AND ((t.status = 'completed' AND (t.completed_at AT TIME ZONE 'UTC')::date = (NOW() AT TIME ZONE 'UTC')::date)
OR (t.status IN ('pending', 'active') AND t.scheduled_at < (((NOW() AT TIME ZONE 'UTC')::date + 1)::timestamp AT TIME ZONE 'UTC')))`
const legacyLastEmailTimesSQL = `SELECT id, MAX(at) FROM (
SELECT t.email_account_id AS id, t.completed_at AS at FROM tasks t
WHERE t.email_account_id = ANY($1) AND t.status = 'completed'
AND t.completed_at IS NOT NULL AND ` + taskDispatchedEmail + `
UNION ALL SELECT ea.id, ea.send_claimed_at FROM email_accounts ea
WHERE ea.id = ANY($1) AND ea.send_claimed_at IS NOT NULL
) last GROUP BY id`
const legacyLastSendTimesSQL = `SELECT t.email_account_id, MAX(t.completed_at) FROM tasks t
WHERE t.email_account_id = ANY($1) AND t.status = 'completed' AND t.task_type = $2::task_type
AND t.completed_at IS NOT NULL AND ` + taskDispatchedEmail + ` GROUP BY t.email_account_id`
func taskClockMap(t *testing.T, pool *pgxpool.Pool, query string, args ...any) map[uuid.UUID]time.Time {
t.Helper()
rows, err := pool.Query(t.Context(), query, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
out := make(map[uuid.UUID]time.Time)
for rows.Next() {
var id uuid.UUID
var at time.Time
if err := rows.Scan(&id, &at); err != nil {
t.Fatal(err)
}
out[id] = at.UTC()
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return out
}
func TestLiveTaskLedgerRangesAndClocksMatchLegacy(t *testing.T) {
f, _ := lineageFixture(t)
ctx := t.Context()
sf := newSharedOrgFixture(t, f.pool)
sequence := uuid.New()
if _, err := f.pool.Exec(ctx, `INSERT INTO sequences(id,campaign_id,organization_id,name,subject,body_plain,body_html)
VALUES($1,$2,$3,'Ledger','Ledger','','')`, sequence, sf.campaign, sf.org); err != nil {
t.Fatal(err)
}
for _, zone := range []string{"UTC", "America/New_York", "Europe/Budapest", "Pacific/Auckland"} {
t.Run(zone, func(t *testing.T) {
cfg := f.pool.Config().Copy()
cfg.ConnConfig.RuntimeParams["timezone"] = zone
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
r := NewTaskRepository(pool)
if _, err := pool.Exec(ctx, `DELETE FROM tasks WHERE email_account_id = ANY($1)`, []uuid.UUID{f.sender, f.recipient}); err != nil {
t.Fatal(err)
}
var utcStart, utcEnd, localStart, localEnd time.Time
if err := pool.QueryRow(ctx, `SELECT
(NOW() AT TIME ZONE 'UTC')::date::timestamp AT TIME ZONE 'UTC',
((NOW() AT TIME ZONE 'UTC')::date + 1)::timestamp AT TIME ZONE 'UTC',
CURRENT_DATE::timestamptz, (CURRENT_DATE + 1)::timestamptz`).Scan(&utcStart, &utcEnd, &localStart, &localEnd); err != nil {
t.Fatal(err)
}
for _, kind := range []string{"campaign", "warmup", "email", "placement"} {
for _, status := range []string{"completed", "pending", "active", "failed"} {
for _, at := range []any{utcStart.Add(-time.Microsecond), utcStart, utcEnd.Add(-time.Microsecond), utcEnd,
localStart.Add(-time.Microsecond), localStart, localEnd.Add(-time.Microsecond), localEnd, nil} {
for _, message := range []string{"", "sent@test.local"} {
if _, err := pool.Exec(ctx, `INSERT INTO tasks(email_account_id,task_type,status,message_id,completed_at,scheduled_at)
VALUES($1,$2::task_type,$3::task_status,$4,$5,$5)`, f.sender, kind, status, message, at); err != nil {
t.Fatal(err)
}
}
}
}
}
reserved := uuid.New()
if _, err := pool.Exec(ctx, `INSERT INTO tasks(id,email_account_id,task_type,status,message_id,completed_at)
VALUES($1,$2,'campaign','completed','',$3)`, reserved, f.sender, utcEnd.Add(2*time.Hour)); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO campaign_contact_progress(campaign_id,contact_id,sequence_id,dispatch_task_id)
VALUES($1,$2,$3,$4) ON CONFLICT(campaign_id,contact_id,sequence_id) DO UPDATE SET dispatch_task_id=$4`, sf.campaign, sf.contact, sequence, reserved); err != nil {
t.Fatal(err)
}
ids := []uuid.UUID{f.sender, f.recipient, f.sender, uuid.New()}
var wantCold, wantAll, wantWarm int
if err := pool.QueryRow(ctx, `SELECT
(SELECT COUNT(*) FROM tasks t WHERE email_account_id=$1 AND `+legacyCampaignSendToday+`) +
(SELECT COUNT(*) FROM tasks t WHERE email_account_id=$1 AND `+legacyPlacementSendToday+`),
(SELECT COUNT(*) FROM tasks t WHERE email_account_id=$1 AND status='completed' AND DATE(completed_at)=CURRENT_DATE AND `+taskDispatchedEmail+`),
(SELECT COUNT(*) FROM tasks WHERE email_account_id=$1 AND status='completed' AND task_type='warmup' AND DATE(completed_at)=CURRENT_DATE)`, f.sender).Scan(&wantCold, &wantAll, &wantWarm); err != nil {
t.Fatal(err)
}
if got, err := r.CountCampaignEmailsSentToday(ctx, f.sender); err != nil || got != wantCold {
t.Fatalf("cold count=%d want=%d err=%v", got, wantCold, err)
}
if got, err := r.CountCampaignEmailsSentTodayByAccounts(ctx, ids); err != nil || got[f.sender] != wantCold || len(got) != 1 {
t.Fatalf("cold batch=%v want=%d err=%v", got, wantCold, err)
}
if got, err := r.CountEmailsSentToday(ctx, f.sender); err != nil || got != wantAll {
t.Fatalf("all count=%d want=%d err=%v", got, wantAll, err)
}
if got, err := r.CountWarmupEmailsSentToday(ctx, f.sender); err != nil || got != wantWarm {
t.Fatalf("warmup count=%d want=%d err=%v", got, wantWarm, err)
}
for _, claim := range []any{nil, utcStart.Add(-48 * time.Hour), utcEnd.Add(48 * time.Hour)} {
if _, err := pool.Exec(ctx, `UPDATE email_accounts SET send_claimed_at=$2 WHERE id=ANY($1)`, []uuid.UUID{f.sender, f.recipient}, claim); err != nil {
t.Fatal(err)
}
want := taskClockMap(t, pool, legacyLastEmailTimesSQL, ids)
got, err := r.GetLastEmailTimes(ctx, ids)
if err != nil || !reflect.DeepEqual(taskClockUTC(got), want) {
t.Fatalf("last clock=%v want=%v err=%v", got, want, err)
}
for _, id := range ids {
last, err := r.GetLastEmailTime(ctx, id)
at, ok := want[id]
if err != nil || (last != nil) != ok || ok && !last.Equal(at) {
t.Fatalf("single clock=%v want=%v present=%t err=%v", last, at, ok, err)
}
}
}
for _, kind := range []string{"campaign", "warmup", "email", "placement"} {
want := taskClockMap(t, pool, legacyLastSendTimesSQL, ids, kind)
got, err := r.GetLastSendTimes(ctx, ids, kind)
if err != nil || !reflect.DeepEqual(taskClockUTC(got), want) {
t.Fatalf("last %s=%v want=%v err=%v", kind, got, want, err)
}
}
if got, err := r.GetLastEmailTimes(ctx, nil); err != nil || len(got) != 0 {
t.Fatalf("empty clock=%v err=%v", got, err)
}
if got, err := r.GetLastSendTimes(ctx, nil, "campaign"); err != nil || len(got) != 0 {
t.Fatalf("empty send clock=%v err=%v", got, err)
}
})
}
}
func taskClockUTC(in map[uuid.UUID]time.Time) map[uuid.UUID]time.Time {
for id, at := range in {
in[id] = at.UTC()
}
return in
}
func TestLiveTaskLedgerDayRangePreservesDST(t *testing.T) {
f, _ := lineageFixture(t)
for _, day := range []string{"2026-03-08", "2026-11-01"} {
var mismatches int
err := f.pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM generate_series(
($1::date::timestamp AT TIME ZONE 'America/New_York') - interval '2 hours',
(($1::date+1)::timestamp AT TIME ZONE 'America/New_York') + interval '2 hours', interval '1 second') s(at)
WHERE ((at AT TIME ZONE 'America/New_York')::date = $1::date) IS DISTINCT FROM
(at >= ($1::date::timestamp AT TIME ZONE 'America/New_York') AND
at < (($1::date+1)::timestamp AT TIME ZONE 'America/New_York'))`, day).Scan(&mismatches)
if err != nil || mismatches != 0 {
t.Fatalf("DST day %s mismatches=%d err=%v", day, mismatches, err)
}
}
}
type taskPressureNode struct {
RelationName string `json:"Relation Name"`
Rows float64 `json:"Actual Rows"`
Loops float64 `json:"Actual Loops"`
Filtered float64 `json:"Rows Removed by Filter"`
Plans []taskPressureNode `json:"Plans"`
}
func taskPressureExplain(t *testing.T, conn *pgx.Conn, query, args, relation string) (float64, float64, string) {
t.Helper()
if _, err := conn.Exec(t.Context(), `PREPARE task_pressure AS `+query); err != nil {
t.Fatal(err)
}
defer func() {
if _, err := conn.Exec(t.Context(), `DEALLOCATE task_pressure`); err != nil {
t.Error(err)
}
}()
var raw []byte
for range 4 {
if err := conn.QueryRow(t.Context(), `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE task_pressure(`+args+`)`).Scan(&raw); err != nil {
t.Fatal(err)
}
}
var plans []struct {
Plan taskPressureNode `json:"Plan"`
ExecutionTime float64 `json:"Execution Time"`
}
if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 {
t.Fatalf("plan decode: %v", err)
}
var examined float64
var visit func(taskPressureNode)
visit = func(p taskPressureNode) {
if p.RelationName == relation {
examined += (p.Rows + p.Filtered) * p.Loops
}
for _, child := range p.Plans {
visit(child)
}
}
visit(plans[0].Plan)
return examined, plans[0].ExecutionTime, string(raw)
}
func TestLiveTaskQueryPressureUsesBoundedHistory(t *testing.T) {
f, _ := lineageFixture(t)
ctx := t.Context()
ids := []uuid.UUID{f.sender, f.recipient}
for range 22 {
id := uuid.New()
if _, err := f.pool.Exec(ctx, `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider)
VALUES($1,$2,$3,$4,'Plan','','','smtp_imap')`, id, f.user, f.org, id.String()+"@test.local"); err != nil {
t.Fatal(err)
}
ids = append(ids, id)
}
if _, err := f.pool.Exec(ctx, `INSERT INTO tasks(email_account_id,task_type,status,message_id,completed_at)
SELECT a, 'campaign', 'completed', 'history@test.local', NOW()-interval '2 days' - make_interval(secs => g)
FROM unnest($1::uuid[]) a CROSS JOIN generate_series(1,2500) g`, ids); err != nil {
t.Fatal(err)
}
if _, err := f.pool.Exec(ctx, `INSERT INTO tasks(email_account_id,task_type,status,message_id,completed_at)
SELECT a, 'campaign', 'completed', 'today@test.local', NOW() FROM unnest($1::uuid[]) a`, ids); err != nil {
t.Fatal(err)
}
if _, err := f.pool.Exec(ctx, `ANALYZE tasks`); err != nil {
t.Fatal(err)
}
conn, err := pgx.ConnectConfig(ctx, f.pool.Config().ConnConfig.Copy())
if err != nil {
t.Fatal(err)
}
defer conn.Close(ctx)
if _, err := conn.Exec(ctx, `SET plan_cache_mode=force_generic_plan`); err != nil {
t.Fatal(err)
}
array := "ARRAY['" + strings.Join(uuidStrings(ids), "','") + "']::uuid[]"
for _, tc := range []struct{ name, before, after, args string }{
{"daily_campaign", `SELECT COUNT(*) FROM tasks t WHERE email_account_id=$1 AND ` + legacyCampaignSendToday,
`SELECT COUNT(*) FROM tasks t WHERE email_account_id=$1 AND ` + campaignSendToday, fmt.Sprintf("'%s'", f.sender)},
{"pool_clock", legacyLastEmailTimesSQL, lastEmailTimesSQL, array},
{"rotation_clock", legacyLastSendTimesSQL, lastSendTimesSQL, array + ",'campaign'"},
} {
t.Run(tc.name, func(t *testing.T) {
beforeRows, beforeMS, _ := taskPressureExplain(t, conn, tc.before, tc.args, "tasks")
afterRows, afterMS, plan := taskPressureExplain(t, conn, tc.after, tc.args, "tasks")
t.Logf("synthetic fixture: 24 mailboxes x 2501 sends; generic plan before %.3fms / %.0f task rows, after %.3fms / %.0f task rows", beforeMS, beforeRows, afterMS, afterRows)
if afterRows >= beforeRows || !strings.Contains(plan, `"Index Name": "idx_tasks_`) {
t.Fatalf("expected fewer task rows and an indexed lookup: %s", plan)
}
if tc.name != "daily_campaign" && !strings.Contains(plan, `"Node Type": "Limit"`) {
t.Fatalf("send clocks must bound each mailbox lookup: %s", plan)
}
if tc.name == "daily_campaign" && !strings.Contains(plan, "completed_at >=") {
t.Fatalf("ledger must bound indexed completed_at: %s", plan)
}
})
}
}
func uuidStrings(ids []uuid.UUID) []string {
out := make([]string, len(ids))
for i, id := range ids {
out[i] = id.String()
}
return out
}

View File

@ -0,0 +1,127 @@
package repository
import (
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/warmbly/warmbly/internal/models"
)
const legacyHealthFeedbackCountsSQL = `SELECT
(SELECT COUNT(*) FILTER (WHERE de.event_type='complaint') FROM deliverability_events de JOIN tasks t ON t.id=de.task_id
WHERE t.email_account_id=$1 AND de.created_at >= $3 AND de.event_type IN ('complaint','bounce')) AS complaints,
(SELECT COUNT(*) FILTER (WHERE de.event_type='bounce') FROM deliverability_events de JOIN tasks t ON t.id=de.task_id
WHERE t.email_account_id=$1 AND de.created_at >= $3 AND de.event_type IN ('complaint','bounce')) AS bounces`
const legacyHealthTamperingCountsSQL = `SELECT
(SELECT COUNT(*) FILTER (WHERE kind='deletion') FROM warmup_tampering_events WHERE email_account_id=$1 AND created_at >= $2) AS deletions,
(SELECT COUNT(*) FILTER (WHERE kind='spam_flag') FROM warmup_tampering_events WHERE email_account_id=$1 AND created_at >= $2) AS spam_flags`
func legacyHealthCounts(t *testing.T, pool *pgxpool.Pool, id uuid.UUID, since7d, since30d time.Time) models.WarmupHealthCounts {
t.Helper()
query := `SELECT
(SELECT COALESCE(SUM(emails_sent),0) FROM warmup_statistics WHERE email_account_id=$1 AND date >= DATE($2)),
(SELECT COUNT(*) FROM warmup_spam_reports WHERE reported_account_id=$1 AND created_at >= $2 AND report_type='spam_placement'),
(SELECT COUNT(*) FROM warmup_spam_reports WHERE reported_account_id=$1 AND created_at >= $2 AND report_type IN ('user_complaint','spam','spam_folder')),
f.complaints, f.bounces,
(SELECT COUNT(*) FROM tasks WHERE email_account_id=$1 AND status='completed' AND completed_at >= $3),
t.deletions, t.spam_flags, p.major, p.major_spam, p.other, p.other_spam
FROM (` + legacyHealthFeedbackCountsSQL + `) f CROSS JOIN (` + legacyHealthTamperingCountsSQL + `) t
CROSS JOIN (` + placementEvidenceSQL("$1", "$2") + `) p`
var c models.WarmupHealthCounts
p := &c.Placement
if err := pool.QueryRow(t.Context(), query, id, since7d, since30d).Scan(&c.SentLast7d, &c.SpamPlacementsLast7d,
&c.UserComplaintsLast7d, &c.ComplaintsLast30d, &c.BouncesLast30d, &c.DeliveredLast30d,
&c.DeletionsLast7d, &c.SpamFlagsLast7d, &p.MajorDelivered, &p.MajorSpam, &p.OtherDelivered, &p.OtherSpam); err != nil {
t.Fatal(err)
}
return c
}
func TestLiveWarmupHealthCountsShareScansWithoutChangingSignals(t *testing.T) {
f, _ := lineageFixture(t)
ctx := t.Context()
now := time.Now().UTC().Truncate(time.Microsecond)
since7d, since30d := now.Add(-7*24*time.Hour), now.Add(-30*24*time.Hour)
r := NewWarmupRepository(f.pool)
for _, id := range []uuid.UUID{f.sender, f.recipient, uuid.New()} {
got, err := r.HealthMetricCounts(ctx, id, since7d, since30d)
want := legacyHealthCounts(t, f.pool, id, since7d, since30d)
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("empty health=%+v want=%+v err=%v", got, want, err)
}
}
for _, id := range []uuid.UUID{f.sender, f.recipient} {
var task uuid.UUID
if err := f.pool.QueryRow(ctx, `INSERT INTO tasks(email_account_id,task_type,status,message_id,completed_at)
VALUES($1,'warmup','completed','',$2) RETURNING id`, id, now).Scan(&task); err != nil {
t.Fatal(err)
}
for _, at := range []time.Time{since30d.Add(-time.Microsecond), since30d, since7d.Add(-time.Microsecond), since7d, now} {
for _, event := range []string{"complaint", "bounce", "unrelated"} {
if _, err := f.pool.Exec(ctx, `INSERT INTO deliverability_events(organization_id,task_id,event_type,recipient_email,idempotency_key,created_at)
VALUES($1,$2,$3,'signal@test.local',$4,$5)`, f.org, task, event, uuid.NewString(), at); err != nil {
t.Fatal(err)
}
}
for _, kind := range []string{"deletion", "spam_flag", "other"} {
if _, err := f.pool.Exec(ctx, `INSERT INTO warmup_tampering_events(email_account_id,message_id,kind,created_at,verified_at)
VALUES($1,$2,$3,$4,NULL)`, id, uuid.NewString(), kind, at); err != nil {
t.Fatal(err)
}
}
}
}
for _, id := range []uuid.UUID{f.sender, f.recipient, uuid.New()} {
got, err := r.HealthMetricCounts(ctx, id, since7d, since30d)
want := legacyHealthCounts(t, f.pool, id, since7d, since30d)
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("health=%+v want=%+v err=%v", got, want, err)
}
if id == f.sender && (got.ComplaintsLast30d != 4 || got.BouncesLast30d != 4 || got.DeletionsLast7d != 2 || got.SpamFlagsLast7d != 2) {
t.Fatalf("boundary counts changed: %+v", got)
}
}
if _, err := f.pool.Exec(ctx, `INSERT INTO warmup_tampering_events(email_account_id,message_id,kind,created_at)
SELECT $1,g::text || '@pressure.test',CASE WHEN g%2=0 THEN 'deletion' ELSE 'spam_flag' END,$2
FROM generate_series(1,10000) g`, f.sender, now); err != nil {
t.Fatal(err)
}
if _, err := f.pool.Exec(ctx, `INSERT INTO deliverability_events(organization_id,task_id,event_type,recipient_email,idempotency_key,created_at)
SELECT $1,$2,CASE WHEN g%2=0 THEN 'complaint' ELSE 'bounce' END,'signal@test.local',g::text || '@pressure.test',$3
FROM generate_series(1,10000) g`, f.org, f.task, now); err != nil {
t.Fatal(err)
}
for _, table := range []string{"deliverability_events", "warmup_tampering_events"} {
if _, err := f.pool.Exec(ctx, "ANALYZE "+table); err != nil {
t.Fatal(err)
}
}
conn, err := pgx.ConnectConfig(ctx, f.pool.Config().ConnConfig.Copy())
if err != nil {
t.Fatal(err)
}
defer conn.Close(ctx)
if _, err := conn.Exec(ctx, `SET plan_cache_mode=force_generic_plan`); err != nil {
t.Fatal(err)
}
for _, tc := range []struct{ name, before, after, args string }{
{"deliverability_events", strings.ReplaceAll(legacyHealthFeedbackCountsSQL, "$3", "$2"), strings.ReplaceAll(healthFeedbackCountsSQL, "$3", "$2"), fmt.Sprintf("'%s','%s'::timestamptz", f.sender, since30d.Format(time.RFC3339Nano))},
{"warmup_tampering_events", legacyHealthTamperingCountsSQL, healthTamperingCountsSQL, fmt.Sprintf("'%s','%s'::timestamptz", f.sender, since7d.Format(time.RFC3339Nano))},
} {
t.Run(tc.name, func(t *testing.T) {
beforeRows, beforeMS, _ := taskPressureExplain(t, conn, tc.before, tc.args, tc.name)
afterRows, afterMS, _ := taskPressureExplain(t, conn, tc.after, tc.args, tc.name)
t.Logf("synthetic 10000 added signals: generic plan before %.3fms / %.0f signal rows, after %.3fms / %.0f signal rows", beforeMS, beforeRows, afterMS, afterRows)
if afterRows == 0 || beforeRows != 2*afterRows {
t.Fatalf("expected one signal scan instead of two: before=%v after=%v", beforeRows, afterRows)
}
})
}
}