AI-native A/B testing: auto-generate variants, auto-promote winners

THE AUTOMATION ENGINE IS NOW SELF-IMPROVING.

## Core: AI generates challenger variants
Click '✨ AI: Test a new approach' → GPT-4o-mini analyzes variant A
and creates variant B using a fundamentally DIFFERENT psychological
approach. Not a rephrase — a different strategy:
- Social proof ('47 others have already paid')
- Urgency (deadline framing)
- Impact storytelling ('£50 = 3 weeks of food')
- Personal connection (heavy name usage)
- Brevity (strip to minimum)
- Gratitude-first (lead with thanks)
- Loss framing ('pledge at risk of being unfulfilled')
- Community ('join 23 others who completed this week')

AI explains WHY: 'This variant uses social proof instead of a
gentle reminder — peer pressure converts better for step 2.'

## Core: Automatic winner promotion
Click '🏆 Pick winners' → system evaluates ALL running A/B tests:
1. Checks minimum sample size (20 sends per variant)
2. Runs z-test for statistical significance (90% confidence)
3. Promotes winner to variant A (resets counters)
4. Deletes loser
5. AUTOMATICALLY generates a NEW AI challenger

The cycle never stops. Messages continuously evolve.

## Core: AI rewrite toolbar
8 one-click AI rewrites for any template:
✂️ Make shorter · 💛 Make warmer · ⏰ Add urgency
👥 Add social proof · 💚 Add impact story · 🎯 Strip to essentials
🇵🇰 Translate to Urdu · 🇸🇦 Translate to Arabic

All rewrites preserve {{variable}} placeholders.
All use GPT-4o-mini (~/usr/bin/bash.15/1M tokens).

## UI: A/B Stats Card (below phone mockup)
- Side-by-side conversion rates with trophy icon on winner
- Progress bar to verdict (% of minimum sample collected)
- Lift calculation: 'Variant B converts 63% better'
- Real-time during test: 'A: 33% → B: 54% ★'

## UI: Winner Results Banner
After 'Pick winners' runs:
- Green banner: '🏆 Winners promoted — Gentle reminder · WhatsApp
  → Variant B wins (54% vs 33%) ✨ New AI challenger created'
- Gray banner if not enough data: 'Need 20+ sends per variant'

## API: /api/automations/ai (POST)
Actions:
- generate_variant: AI creates challenger B with strategy reasoning
- rewrite: AI rewrites template with specific instruction
- check_winners: evaluate all tests, promote, regenerate

## Architecture
The system is a GENETIC ALGORITHM for messaging:
1. Start with default templates (generation 0)
2. AI creates a challenger (mutation)
3. Traffic splits 50/50 (fitness test)
4. Winner survives, loser dies (selection)
5. AI creates new challenger (next generation)
6. Repeat forever → messages get better over time

pledge-now-pay-later/src/app/api/automations/ai/route.ts

@@ -0,0 +1,320 @@
+import { NextRequest, NextResponse } from "next/server"
+import prisma from "@/lib/prisma"
+import { getUser } from "@/lib/session"
+
+const OPENAI_KEY = process.env.OPENAI_API_KEY
+const MODEL = "gpt-4o-mini"
+
+async function chat(messages: Array<{ role: string; content: string }>, maxTokens = 600): Promise<string> {
+  if (!OPENAI_KEY) return ""
+  try {
+    const res = await fetch("https://api.openai.com/v1/chat/completions", {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: `Bearer ${OPENAI_KEY}` },
+      body: JSON.stringify({ model: MODEL, messages, max_tokens: maxTokens, temperature: 0.8 }),
+    })
+    const data = await res.json()
+    return data.choices?.[0]?.message?.content || ""
+  } catch { return "" }
+}
+
+/**
+ * POST /api/automations/ai
+ *
+ * Actions:
+ * - generate_variant: AI creates a challenger variant B
+ * - rewrite: AI rewrites a template with a specific instruction
+ * - check_winners: Evaluate all A/B tests and auto-promote winners
+ */
+export async function POST(request: NextRequest) {
+  const user = await getUser()
+  if (!user) return NextResponse.json({ error: "Not authenticated" }, { status: 401 })
+  if (!prisma) return NextResponse.json({ error: "No DB" }, { status: 503 })
+
+  const body = await request.json()
+  const { action } = body
+  const orgId = user.orgId
+
+  // ─── GENERATE VARIANT ───────────────────────────
+  if (action === "generate_variant") {
+    const { step, channel } = body
+
+    const existing = await prisma.messageTemplate.findFirst({
+      where: { organizationId: orgId, step, channel, variant: "A" },
+    })
+    if (!existing) return NextResponse.json({ error: "No variant A found" }, { status: 404 })
+
+    // Get org context
+    const org = await prisma.organization.findUnique({
+      where: { id: orgId },
+      select: { name: true, orgType: true },
+    })
+
+    const stepLabels: Record<number, string> = {
+      0: "pledge receipt (instant confirmation with bank details)",
+      1: "gentle reminder (day 2, donor hasn't paid yet)",
+      2: "impact nudge (day 7, building urgency with purpose)",
+      3: "final reminder (day 14, last message before marking overdue)",
+    }
+
+    const channelRules: Record<string, string> = {
+      whatsapp: "WhatsApp message. Use WhatsApp formatting: *bold*, _italic_, `code` for references. Emojis are good. Keep conversational. Can use ━━━ dividers. Reply keywords: PAID, HELP, CANCEL.",
+      email: "Email body (plain text, will be formatted). Can be slightly longer. Include {{pledge_url}} and {{cancel_url}} links.",
+      sms: "SMS message. MUST be under 160 characters. No formatting. Be extremely concise. Every character counts.",
+    }
+
+    const result = await chat([
+      {
+        role: "system",
+        content: `You are an A/B testing expert for UK charity fundraising messages. You generate CHALLENGER variants that take a fundamentally different psychological approach.
+
+Your job: given an existing message (variant A), create a variant B that tests a DIFFERENT strategy. Don't just rephrase — change the APPROACH.
+
+Strategies to try (pick ONE that's different from variant A):
+- SOCIAL PROOF: "47 others have already paid their pledge"
+- URGENCY: deadline framing, countdown language
+- IMPACT STORYTELLING: specific impact ("£50 = 3 weeks of food for a family")
+- PERSONAL CONNECTION: use donor's name heavily, feel like a 1-to-1 conversation
+- BREVITY: strip to absolute minimum — sometimes less is more
+- GRATITUDE-FIRST: lead with thanks, make the ask secondary
+- LOSS FRAMING: "Your pledge is at risk of being marked unfulfilled"
+- COMMUNITY: "Join 23 others who've completed their pledge this week"
+
+Rules:
+- Use the SAME template variables: {{name}}, {{amount}}, {{event}}, {{reference}}, {{bank_name}}, {{sort_code}}, {{account_no}}, {{org_name}}, {{days}}, {{cancel_url}}, {{pledge_url}}
+- ${channelRules[channel] || "Keep appropriate for the channel."}
+- UK English. Warm but professional. Charity context: ${org?.orgType || "charity"}.
+- Org name: ${org?.name || "the charity"}
+
+Return ONLY valid JSON:
+{
+  "body": "the new message template with {{variables}}",
+  "subject": "email subject line with {{variables}} (null if not email)",
+  "name": "short name for this variant (e.g. 'Social proof nudge')",
+  "strategy": "which strategy you used (e.g. 'social_proof')",
+  "reasoning": "1-2 sentences explaining WHY this approach might outperform variant A"
+}`
+      },
+      {
+        role: "user",
+        content: `Step: ${stepLabels[step] || `step ${step}`}
+Channel: ${channel}
+Current variant A:
+---
+${existing.body}
+---
+
+Generate a challenger variant B using a DIFFERENT psychological approach.`
+      }
+    ], 800)
+
+    if (!result) {
+      return NextResponse.json({ error: "AI unavailable — no API key configured" }, { status: 503 })
+    }
+
+    try {
+      const parsed = JSON.parse(result)
+
+      // Save as variant B
+      await prisma.messageTemplate.upsert({
+        where: {
+          organizationId_step_channel_variant: {
+            organizationId: orgId, step, channel, variant: "B",
+          },
+        },
+        update: {
+          body: parsed.body,
+          subject: parsed.subject || null,
+          name: parsed.name || "AI Challenger",
+          isActive: true,
+          splitPercent: 50,
+        },
+        create: {
+          organizationId: orgId, step, channel, variant: "B",
+          body: parsed.body,
+          subject: parsed.subject || null,
+          name: parsed.name || "AI Challenger",
+          isActive: true,
+          splitPercent: 50,
+        },
+      })
+
+      // Set variant A to 50%
+      await prisma.messageTemplate.updateMany({
+        where: { organizationId: orgId, step, channel, variant: "A" },
+        data: { splitPercent: 50 },
+      })
+
+      return NextResponse.json({
+        ok: true,
+        variant: {
+          body: parsed.body,
+          subject: parsed.subject,
+          name: parsed.name,
+          strategy: parsed.strategy,
+          reasoning: parsed.reasoning,
+        },
+      })
+    } catch {
+      return NextResponse.json({ error: "AI returned invalid response" }, { status: 500 })
+    }
+  }
+
+  // ─── REWRITE ────────────────────────────────────
+  if (action === "rewrite") {
+    const { instruction, currentBody } = body
+    const channel = body.channel
+
+    const channelHints: Record<string, string> = {
+      whatsapp: "WhatsApp formatting (*bold*, _italic_, `code`). Emojis OK. Conversational.",
+      email: "Email body (plain text). Can include links like {{pledge_url}}.",
+      sms: "SMS — MUST be under 160 characters. No formatting.",
+    }
+
+    const result = await chat([
+      {
+        role: "system",
+        content: `You rewrite UK charity fundraising messages. Follow the instruction exactly. Keep ALL {{variable}} placeholders. ${channelHints[channel] || ""}
+Return ONLY the rewritten message text — nothing else. No JSON, no explanation.`
+      },
+      {
+        role: "user",
+        content: `Instruction: ${instruction}
+
+Current message:
+---
+${currentBody}
+---
+
+Rewrite it following the instruction.`
+      }
+    ], 600)
+
+    if (!result) {
+      return NextResponse.json({ error: "AI unavailable" }, { status: 503 })
+    }
+
+    return NextResponse.json({ ok: true, body: result.trim() })
+  }
+
+  // ─── CHECK WINNERS ──────────────────────────────
+  if (action === "check_winners") {
+    const MIN_SAMPLE = body.minSample || 20 // minimum sends per variant before judging
+
+    // Find all A/B pairs for this org
+    const templates = await prisma.messageTemplate.findMany({
+      where: { organizationId: orgId },
+      orderBy: [{ step: "asc" }, { channel: "asc" }, { variant: "asc" }],
+    })
+
+    // Group by step+channel
+    const groups = new Map<string, typeof templates>()
+    for (const t of templates) {
+      const key = `${t.step}:${t.channel}`
+      if (!groups.has(key)) groups.set(key, [])
+      groups.get(key)!.push(t)
+    }
+
+    const results: Array<{
+      step: number; channel: string
+      winner: string; loser: string
+      winnerRate: number; loserRate: number
+      action: "promoted" | "too_early" | "no_ab"
+      newChallenger?: boolean
+    }> = []
+
+    for (const group of Array.from(groups.values())) {
+      if (group.length < 2) {
+        results.push({ step: group[0].step, channel: group[0].channel, winner: "A", loser: "-", winnerRate: 0, loserRate: 0, action: "no_ab" })
+        continue
+      }
+
+      const a = group.find(t => t.variant === "A")
+      const b = group.find(t => t.variant === "B")
+      if (!a || !b) continue
+
+      // Not enough data yet
+      if (a.sentCount < MIN_SAMPLE || b.sentCount < MIN_SAMPLE) {
+        results.push({
+          step: a.step, channel: a.channel,
+          winner: "-", loser: "-",
+          winnerRate: a.sentCount > 0 ? Math.round((a.convertedCount / a.sentCount) * 100) : 0,
+          loserRate: b.sentCount > 0 ? Math.round((b.convertedCount / b.sentCount) * 100) : 0,
+          action: "too_early",
+        })
+        continue
+      }
+
+      const rateA = a.convertedCount / a.sentCount
+      const rateB = b.convertedCount / b.sentCount
+
+      // Statistical significance: simplified z-test
+      const pooledRate = (a.convertedCount + b.convertedCount) / (a.sentCount + b.sentCount)
+      const se = Math.sqrt(pooledRate * (1 - pooledRate) * (1 / a.sentCount + 1 / b.sentCount))
+      const z = se > 0 ? Math.abs(rateA - rateB) / se : 0
+
+      // z > 1.65 ≈ 90% confidence
+      if (z < 1.65) {
+        results.push({
+          step: a.step, channel: a.channel,
+          winner: "-", loser: "-",
+          winnerRate: Math.round(rateA * 100),
+          loserRate: Math.round(rateB * 100),
+          action: "too_early",
+        })
+        continue
+      }
+
+      // We have a winner!
+      const winnerIs = rateB > rateA ? "B" : "A"
+      const winner = winnerIs === "B" ? b : a
+      const loser = winnerIs === "B" ? a : b
+
+      // Promote winner: winner becomes A, loser gets deleted
+      await prisma.messageTemplate.update({
+        where: { id: a.id },
+        data: {
+          body: winner.body,
+          subject: winner.subject,
+          name: winner.name,
+          splitPercent: 100,
+          sentCount: 0,
+          convertedCount: 0,
+        },
+      })
+
+      // Delete B
+      await prisma.messageTemplate.delete({ where: { id: b.id } })
+
+      // Generate new challenger
+      let newChallenger = false
+      if (OPENAI_KEY) {
+        try {
+          // Recursively call generate_variant
+          const genRes = await fetch(new URL("/api/automations/ai", request.url), {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              cookie: request.headers.get("cookie") || "",
+            },
+            body: JSON.stringify({ action: "generate_variant", step: winner.step, channel: winner.channel }),
+          })
+          if (genRes.ok) newChallenger = true
+        } catch { /* */ }
+      }
+
+      results.push({
+        step: winner.step, channel: winner.channel,
+        winner: winnerIs, loser: winnerIs === "B" ? "A" : "B",
+        winnerRate: Math.round((winner.convertedCount / winner.sentCount) * 100),
+        loserRate: Math.round((loser.convertedCount / loser.sentCount) * 100),
+        action: "promoted",
+        newChallenger,
+      })
+    }
+
+    return NextResponse.json({ ok: true, results })
+  }
+
+  return NextResponse.json({ error: "Unknown action" }, { status: 400 })
+}