diff --git a/.github/workflows/deploy-ec2.yml b/.github/workflows/deploy-ec2.yml index 0c4dd1a..4be151e 100644 --- a/.github/workflows/deploy-ec2.yml +++ b/.github/workflows/deploy-ec2.yml @@ -148,14 +148,19 @@ jobs: echo "=== ENSURE pgvector extension (vector type) ===" docker compose -f docker-compose.ec2.yml --env-file .env exec -T db sh -lc "psql -U \"\${POSTGRES_USER:-postgres}\" -d \"\${POSTGRES_DB:-dropshipping}\" -c \"CREATE EXTENSION IF NOT EXISTS vector;\"" - # Start/roll app after pgvector extension is available - echo "=== START APP (docker compose up -d) ===" - if ! docker compose -f docker-compose.ec2.yml --env-file .env up -d --remove-orphans app; then + # Start/roll app after pgvector extension is available. + # Force recreate so deploy is visible even when image tag remains :latest. + echo "=== START APP (docker compose up -d --force-recreate) ===" + if ! docker compose -f docker-compose.ec2.yml --env-file .env up -d --force-recreate --remove-orphans app; then echo "ERROR: Deployment failed" docker compose -f docker-compose.ec2.yml down || true exit 1 fi + APP_CID="$(docker compose -f docker-compose.ec2.yml --env-file .env ps -q app)" + echo "App container details:" + docker inspect --format='id={{.Id}} startedAt={{.State.StartedAt}} image={{.Image}}' "$APP_CID" || true + echo "✅ Deployment successful! Current services:" docker compose -f docker-compose.ec2.yml --env-file .env ps diff --git a/.gitignore b/.gitignore index 93bb99f..a1a6ba9 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env +# test artifacts +/tests/gmail-auto-reply/artifacts/ + # vercel .vercel diff --git a/package.json b/package.json index c9fe950..ce70b6c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "next start", "lint": "eslint", "test": "vitest run", + "test:gmail-auto-reply": "set -e; docker compose -f tests/gmail-auto-reply/docker-compose.test.yml up -d; export $(cat tests/gmail-auto-reply/env.test | grep -v '^#' | xargs); docker compose -f tests/gmail-auto-reply/docker-compose.test.yml exec -T db sh -lc \"psql -U postgres -d dropshipping -c 'CREATE EXTENSION IF NOT EXISTS vector;'\"; npx prisma db push --accept-data-loss; npx tsx tests/gmail-auto-reply/seed.ts; vitest run tests/gmail-auto-reply/run.test.ts; docker compose -f tests/gmail-auto-reply/docker-compose.test.yml down -v", "re-embed-knowledge-base": "tsx scripts/re-embed-knowledge-base.ts", "db:reset": "prisma migrate reset", "db:studio": "prisma studio", diff --git a/src/lib/ai/gmailOrderDisclosure.test.ts b/src/lib/ai/gmailOrderDisclosure.test.ts index 8a8d9dc..703a396 100644 --- a/src/lib/ai/gmailOrderDisclosure.test.ts +++ b/src/lib/ai/gmailOrderDisclosure.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { looksLikeSensitiveOrderTopic } from "./gmailOrderDisclosure" +import { looksLikeOffTopic, looksLikeSensitiveOrderTopic } from "./gmailOrderDisclosure" describe("looksLikeSensitiveOrderTopic", () => { it("detects shipping and refund language", () => { @@ -13,3 +13,15 @@ describe("looksLikeSensitiveOrderTopic", () => { expect(looksLikeSensitiveOrderTopic("Hello")).toBe(false) }) }) + +describe("looksLikeOffTopic", () => { + it("detects obvious non-store topics", () => { + expect(looksLikeOffTopic("Hey, what's the weather in Islamabad?")).toBe(true) + expect(looksLikeOffTopic("Can you tell me today's news headlines?")).toBe(true) + }) + + it("does not trigger on store-ish questions", () => { + expect(looksLikeOffTopic("Where is my package?")).toBe(false) + expect(looksLikeOffTopic("I need a refund for order #1002")).toBe(false) + }) +}) diff --git a/src/lib/ai/gmailOrderDisclosure.ts b/src/lib/ai/gmailOrderDisclosure.ts index 81fd48f..7b6a873 100644 --- a/src/lib/ai/gmailOrderDisclosure.ts +++ b/src/lib/ai/gmailOrderDisclosure.ts @@ -27,3 +27,28 @@ export function looksLikeSensitiveOrderTopic(text: string): boolean { ] return checks.some((re) => re.test(t)) } + +/** + * Obvious non-store topics that should route to `off_topic` even when classification fails. + * Keep conservative; this should not trigger on normal store support emails. + */ +export function looksLikeOffTopic(text: string): boolean { + const t = text.toLowerCase() + if (t.length < 3) return false + + const checks: RegExp[] = [ + /\bweather\b/, + /\btemperature\b/, + /\brain\b|\bsnow\b|\bforecast\b/, + /\bnews\b|\bheadline(s)?\b/, + /\bpolitic(s|al)?\b|\belection\b/, + /\bhomework\b|\bmath\b|\bessay\b|\bassignment\b/, + /\bjoke\b|\bfunny\b|\bmeme\b/, + /\bstock(s)?\b|\bcrypto\b|\bbitcoin\b/, + /\bsports?\b|\bscore\b|\bmatch\b/, + /\brestaurant\b|\bhotel\b|\bflight\b/, + /\bislamabad\b|\blahore\b|\bkarachi\b|\bparis\b|\blondon\b|\bnew york\b/, + ] + + return checks.some((re) => re.test(t)) +} diff --git a/src/lib/gmail/autoReplyPreview.ts b/src/lib/gmail/autoReplyPreview.ts new file mode 100644 index 0000000..ae4f89e --- /dev/null +++ b/src/lib/gmail/autoReplyPreview.ts @@ -0,0 +1,386 @@ +import { prisma } from "@/lib/prisma" +import { logger } from "@/lib/logger" +import { escapeHtmlForEmail, escapeTemplateContextForHtml, renderTemplate } from "@/lib/templates/simpleRenderer" +import { classifyCustomerEmail, type ClassifyCustomerEmailResult } from "@/lib/ai/gmailGroq" +import { screenInboundEmailForAbuse } from "@/lib/ai/gmailSafety" +import { + buildOrderSummaryForEmail, + extractOrderNumberHints, + stripQuotedEmailTrailForHints, +} from "@/lib/ai/gmailContext" +import { looksLikeOffTopic, looksLikeSensitiveOrderTopic } from "@/lib/ai/gmailOrderDisclosure" +import { parseEnvFloat } from "@/lib/gmail/gmailAutoReplyGuards" +import { findTemplateWithFallback, type TemplateRecord } from "@/lib/juno/templatesRepo" +import type { GmailClassification } from "@/lib/ai/gmailTaxonomy" + +/** Local-part of email, title-cased for greetings (templates use {{customer_name}}). */ +function greetingNameFromEmail(email: string): string { + const local = email.split("@")[0]?.trim() || "" + if (!local) return "there" + return local.charAt(0).toUpperCase() + local.slice(1).toLowerCase() +} + +export function buildMinimalFallbackHtml(emailBody: string, storeName: string): string { + const safeSnippet = escapeHtmlForEmail(emailBody.replace(/<[^>]+>/g, "").slice(0, 400)) + return ` +
+
+

Thanks for reaching out

+

+ We have received your message and our team will review it shortly. +

+

Summary of what you sent:

+
+

+${safeSnippet || "No additional message content was detected."} +

+
+

+ You can reply to this email if you have more details to share. +

+

+ Best regards,
+ ${storeName} Support +

+
+
+ ` +} + +/** When no template exists but we need an order # before sharing order rows. */ +export function buildNeedOrderIdFallbackHtml(storeName: string, storeEmail: string): string { + const contact = storeEmail.trim() + ? `Reply with your order number, or email ${storeEmail}.` + : `Reply with your order number from your confirmation email.` + return ` +
+
+

Hello,

+

+ We didn't see an order number in your message. To look up the right order for ${storeName}, + please reply with your order number (for example #1002 or 1002) from your order confirmation. +

+

${contact}

+

+ Best regards,
+ ${storeName} Support +

+
+
+ ` +} + +/** When hints did not match cache — no unrelated order rows. */ +export function buildHintUnmatchedFallbackHtml(storeName: string, storeEmail: string): string { + const contact = storeEmail.trim() + ? `Reply with the correct order number or email us at ${storeEmail}.` + : `Reply with the order number from your confirmation email.` + return ` +
+
+

Hello,

+

+ We could not match the order number in your message to an order on file for ${storeName} under this email address. + Please double-check the number or confirm the email you used when placing the order. +

+

${contact}

+

+ Best regards,
+ ${storeName} Support +

+
+
+ ` +} + +/** When no off_topic template exists — polite boundary, no order data. */ +export function buildOffTopicFallbackHtml(storeName: string, storeEmail: string): string { + const contact = storeEmail.trim() + ? `If you have a question about an order from ${storeName}, reply with details or email us at ${storeEmail}.` + : `If you have a question about an order from ${storeName}, reply with your question and order number if you have one.` + return ` +
+
+

Hello,

+

+ This inbox only handles orders, products, shipping, returns, and billing for ${storeName}. + We can't answer unrelated requests (news, weather, other websites, or general topics). +

+

${contact}

+

+ Best regards,
+ ${storeName} Support +

+
+
+ ` +} + +function renderFromTemplate( + template: TemplateRecord, + context: Record, +): string { + const safeContext = escapeTemplateContextForHtml(context) + return template.isHtml + ? renderTemplate(template.body, safeContext) + : ` +
+ 
${renderTemplate(
+                                template.body,
+                                safeContext,
+                            )}
+
+ ` +} + +export type AutoReplyPreviewInput = { + storeId: string + fromEmail: string + subject: string + body: string +} + +export type AutoReplyPreviewOutput = { + classification: GmailClassification + classifySource: ClassifyCustomerEmailResult["source"] + orderHints: string[] + offTopicByHeuristic: boolean + suppressOrdersOnFallbackClassification: boolean + needsOrderNumberFirst: boolean + hintUnmatched: boolean + lowConfidenceSuppressOrders: boolean + messageTooSparse: boolean + skipOrderCacheForQuestion: boolean + shouldLoadOrderCache: boolean + sensitiveTopic: boolean + orderSummaryRaw: string + orderSummaryForContext: string + ticketSummary: string + chosenTemplateId: string | null + usedFallback: "none" | "off_topic" | "need_order_id" | "hint_unmatched" | "minimal" + html: string + flagged: boolean + flaggedReason?: string +} + +export async function generateAutoReplyPreview( + input: AutoReplyPreviewInput, +): Promise { + const senderEmail = input.fromEmail.trim().toLowerCase() + const emailSubject = input.subject || "No Subject" + const emailBody = input.body || "" + + const plainForAi = emailBody.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() + + const safety = await screenInboundEmailForAbuse(emailSubject, plainForAi) + if (safety?.flagged) { + const store = await prisma.store.findUnique({ + where: { id: input.storeId }, + select: { businessName: true, email: true }, + }) + const storeName = store?.businessName || "Your store" + return { + classification: { intent: "general", mood: "neutral", confidence: 0.0, short_summary: "" }, + classifySource: "fallback", + orderHints: [], + offTopicByHeuristic: false, + suppressOrdersOnFallbackClassification: false, + needsOrderNumberFirst: false, + hintUnmatched: false, + lowConfidenceSuppressOrders: false, + messageTooSparse: false, + skipOrderCacheForQuestion: false, + shouldLoadOrderCache: false, + sensitiveTopic: false, + orderSummaryRaw: "", + orderSummaryForContext: "", + ticketSummary: "", + chosenTemplateId: null, + usedFallback: "minimal", + html: buildMinimalFallbackHtml(emailBody, storeName), + flagged: true, + flaggedReason: safety.reason, + } + } + + const classifyResult = await classifyCustomerEmail(emailSubject, plainForAi) + const classification = classifyResult.classification + + const hintSourceFull = `${emailSubject}\n${plainForAi}` + const hintSourceForHints = stripQuotedEmailTrailForHints(hintSourceFull) + + const extractedHints = extractOrderNumberHints(hintSourceForHints) + const offTopicByHeuristic = looksLikeOffTopic(hintSourceFull) && extractedHints.length === 0 + const isOffTopic = classification.intent === "off_topic" || offTopicByHeuristic + const orderHints = isOffTopic ? [] : extractedHints + + const intentsNeedingOrderIdWhenAmbiguous = new Set([ + "shipping_delay", + "refund_request", + "account_billing", + ]) + const sensitiveTopic = !isOffTopic && looksLikeSensitiveOrderTopic(hintSourceFull) + const needsOrderNumberFirst = + !isOffTopic && + orderHints.length === 0 && + (intentsNeedingOrderIdWhenAmbiguous.has(classification.intent) || sensitiveTopic) + + const confidenceMin = parseEnvFloat("GMAIL_CLASSIFY_CONFIDENCE_MIN", 0.5) + const lowConfidenceSuppressOrders = + classifyResult.source === "groq" && + classification.confidence < confidenceMin && + orderHints.length === 0 && + !isOffTopic && + (classification.intent === "general" || classification.intent === "question") + + const messageTooSparse = plainForAi.trim().length < 12 && orderHints.length === 0 + + const skipOrderCacheForQuestion = + classification.intent === "question" && orderHints.length === 0 && !isOffTopic + + // Default-safe: if Groq classification failed, do not attach CachedOrder unless the customer provided an order number hint. + const suppressOrdersOnFallbackClassification = + classifyResult.source === "fallback" && extractedHints.length === 0 && !isOffTopic + + const shouldLoadOrderCache = + !isOffTopic && + !needsOrderNumberFirst && + !lowConfidenceSuppressOrders && + !messageTooSparse && + !skipOrderCacheForQuestion && + !suppressOrdersOnFallbackClassification + + let orderSummaryRaw = "" + let hintUnmatched = false + if (shouldLoadOrderCache) { + const built = await buildOrderSummaryForEmail(input.storeId, senderEmail, hintSourceForHints) + orderSummaryRaw = built.summary + hintUnmatched = built.hintUnmatched + } + + logger.info("Gmail auto-reply preview", { + storeId: input.storeId, + senderEmail, + intent: classification.intent, + mood: classification.mood, + confidence: classification.confidence, + classifySource: classifyResult.source, + orderHints, + offTopicByHeuristic, + needsOrderNumberFirst, + hintUnmatched, + lowConfidenceSuppressOrders, + messageTooSparse, + suppressOrdersOnFallbackClassification, + }) + + const store = await prisma.store.findUnique({ + where: { id: input.storeId }, + select: { businessName: true, email: true }, + }) + + const storeName = store?.businessName || "Your store" + const ticketSummary = messageTooSparse + ? "We could not read enough text in your message to look up an order automatically. Please reply with a short description and your order number if you have one." + : classification.short_summary?.trim() || plainForAi.slice(0, 500) || emailBody.slice(0, 500) + + const offTopicOrderLine = `We don't include order or purchase history for unrelated topics. For help with ${storeName}, reply with a store-related question and your order number if you have one.` + + const counterOrderIdLine = `No order details are shown yet — we did not see an order number in your message. Please reply with your order number (e.g. #1002 or 1002) from your ${storeName} confirmation email so we can look up the correct order.` + + const hintUnmatchedOrderLine = `We could not match the order number(s) in your message to an order on file for ${storeName} under this email. Please double-check the number or confirm the email address used at checkout.` + + const lowConfidenceOrderLine = `We were not fully confident how to categorize your message. If this is about an order, reply with your order number from your ${storeName} confirmation email so we can pull up the right record.` + + const sparseBodyOrderLine = `We could not read enough detail in your message to look anything up automatically. Please reply with a sentence or two and your order number if your question is about a ${storeName} order.` + + const questionNoOrderLine = `We are replying without attaching order history. If your question relates to a specific purchase, reply with your order number (e.g. #1002).` + + const noOrdersFallback = sensitiveTopic + ? `No matching recent orders on file for ${storeName} under this email. If you used a different email at checkout, reply from that address or send your order number.` + : "No matching recent orders on file for this email." + + let orderSummaryForContext: string + if (isOffTopic) { + orderSummaryForContext = offTopicOrderLine + } else if (needsOrderNumberFirst) { + orderSummaryForContext = counterOrderIdLine + } else if (hintUnmatched) { + orderSummaryForContext = hintUnmatchedOrderLine + } else if (lowConfidenceSuppressOrders) { + orderSummaryForContext = lowConfidenceOrderLine + } else if (messageTooSparse) { + orderSummaryForContext = sparseBodyOrderLine + } else if (skipOrderCacheForQuestion) { + orderSummaryForContext = questionNoOrderLine + } else { + orderSummaryForContext = orderSummaryRaw || noOrdersFallback + } + + const context: Record = { + customer_name: greetingNameFromEmail(senderEmail), + customer_email: senderEmail, + subject: emailSubject, + store_name: storeName, + store_email: store?.email || "", + ticket_summary: ticketSummary, + order_summary: orderSummaryForContext, + } + + let html: string | null = null + let chosenTemplateId: string | null = null + + try { + const template = await findTemplateWithFallback( + input.storeId, + classification.intent, + classification.mood, + ) + if (template) { + chosenTemplateId = template.id + html = renderFromTemplate(template, context) + } + } catch (e) { + logger.error("generateAutoReplyPreview: template load/render failed", e) + } + + let usedFallback: AutoReplyPreviewOutput["usedFallback"] = "none" + if (!html) { + if (isOffTopic) { + usedFallback = "off_topic" + html = buildOffTopicFallbackHtml(storeName, store?.email || "") + } else if (needsOrderNumberFirst) { + usedFallback = "need_order_id" + html = buildNeedOrderIdFallbackHtml(storeName, store?.email || "") + } else if (hintUnmatched) { + usedFallback = "hint_unmatched" + html = buildHintUnmatchedFallbackHtml(storeName, store?.email || "") + } else { + usedFallback = "minimal" + html = buildMinimalFallbackHtml(emailBody, storeName) + } + } + + return { + classification, + classifySource: classifyResult.source, + orderHints, + offTopicByHeuristic, + suppressOrdersOnFallbackClassification, + needsOrderNumberFirst, + hintUnmatched, + lowConfidenceSuppressOrders, + messageTooSparse, + skipOrderCacheForQuestion, + shouldLoadOrderCache, + sensitiveTopic, + orderSummaryRaw, + orderSummaryForContext, + ticketSummary, + chosenTemplateId, + usedFallback, + html, + flagged: false, + } +} + diff --git a/src/lib/gmail/emailProcessor.ts b/src/lib/gmail/emailProcessor.ts index b68ec61..5de47c3 100644 --- a/src/lib/gmail/emailProcessor.ts +++ b/src/lib/gmail/emailProcessor.ts @@ -10,7 +10,8 @@ import { extractOrderNumberHints, stripQuotedEmailTrailForHints, } from "@/lib/ai/gmailContext" -import { looksLikeSensitiveOrderTopic } from "@/lib/ai/gmailOrderDisclosure" +import { looksLikeOffTopic, looksLikeSensitiveOrderTopic } from "@/lib/ai/gmailOrderDisclosure" +import { generateAutoReplyPreview } from "@/lib/gmail/autoReplyPreview" import { checkGmailAutoReplyRateLimits, parseEnvFloat, @@ -312,156 +313,35 @@ export async function processIncomingEmail( } try { - const classifyResult = await classifyCustomerEmail(emailSubject, plainForAi) - const classification = classifyResult.classification - const hintSourceFull = `${emailSubject}\n${plainForAi}` - const hintSourceForHints = stripQuotedEmailTrailForHints(hintSourceFull) - - const isOffTopic = classification.intent === "off_topic" - const orderHints = isOffTopic ? [] : extractOrderNumberHints(hintSourceForHints) - - const intentsNeedingOrderIdWhenAmbiguous = new Set([ - "shipping_delay", - "refund_request", - "account_billing", - ]) - const sensitiveTopic = !isOffTopic && looksLikeSensitiveOrderTopic(hintSourceFull) - const needsOrderNumberFirst = - !isOffTopic && - orderHints.length === 0 && - (intentsNeedingOrderIdWhenAmbiguous.has(classification.intent) || sensitiveTopic) - - const confidenceMin = parseEnvFloat("GMAIL_CLASSIFY_CONFIDENCE_MIN", 0.5) - const lowConfidenceSuppressOrders = - classifyResult.source === "groq" && - classification.confidence < confidenceMin && - orderHints.length === 0 && - !isOffTopic && - (classification.intent === "general" || classification.intent === "question") - - const messageTooSparse = plainForAi.trim().length < 12 && orderHints.length === 0 - - const skipOrderCacheForQuestion = - classification.intent === "question" && orderHints.length === 0 && !isOffTopic - - const shouldLoadOrderCache = - !isOffTopic && - !needsOrderNumberFirst && - !lowConfidenceSuppressOrders && - !messageTooSparse && - !skipOrderCacheForQuestion - - let orderSummary = "" - let hintUnmatched = false - if (shouldLoadOrderCache) { - const built = await buildOrderSummaryForEmail(storeId, senderEmail, hintSourceForHints) - orderSummary = built.summary - hintUnmatched = built.hintUnmatched - } + const preview = await generateAutoReplyPreview({ + storeId, + fromEmail: senderEmail, + subject: emailSubject, + body: emailBody, + }) logger.info("Gmail classification", { userId, storeId, - intent: classification.intent, - mood: classification.mood, - confidence: classification.confidence, - classifySource: classifyResult.source, + intent: preview.classification.intent, + mood: preview.classification.mood, + confidence: preview.classification.confidence, + classifySource: preview.classifySource, }) logger.info("Gmail order context for reply", { storeId, - orderSummaryChars: orderSummary.trim().length, - orderHints, - offTopic: isOffTopic, - needsOrderNumberFirst, - hintUnmatched, - lowConfidenceSuppressOrders, - messageTooSparse, + orderSummaryChars: preview.orderSummaryRaw.trim().length, + orderHints: preview.orderHints, + offTopic: preview.classification.intent === "off_topic" || preview.offTopicByHeuristic, + offTopicByHeuristic: preview.offTopicByHeuristic, + needsOrderNumberFirst: preview.needsOrderNumberFirst, + hintUnmatched: preview.hintUnmatched, + lowConfidenceSuppressOrders: preview.lowConfidenceSuppressOrders, + messageTooSparse: preview.messageTooSparse, + suppressOrdersOnFallbackClassification: preview.suppressOrdersOnFallbackClassification, }) - const store = await prisma.store.findUnique({ - where: { id: storeId }, - select: { - businessName: true, - email: true, - }, - }) - - const storeName = store?.businessName || "Your store" - const ticketSummary = messageTooSparse - ? "We could not read enough text in your message to look up an order automatically. Please reply with a short description and your order number if you have one." - : classification.short_summary?.trim() || - plainForAi.slice(0, 500) || - emailBody.slice(0, 500) - - const offTopicOrderLine = `We don't include order or purchase history for unrelated topics. For help with ${storeName}, reply with a store-related question and your order number if you have one.` - - const counterOrderIdLine = `No order details are shown yet — we did not see an order number in your message. Please reply with your order number (e.g. #1002 or 1002) from your ${storeName} confirmation email so we can look up the correct order.` - - const hintUnmatchedOrderLine = `We could not match the order number(s) in your message to an order on file for ${storeName} under this email. Please double-check the number or confirm the email address used at checkout.` - - const lowConfidenceOrderLine = `We were not fully confident how to categorize your message. If this is about an order, reply with your order number from your ${storeName} confirmation email so we can pull up the right record.` - - const sparseBodyOrderLine = `We could not read enough detail in your message to look anything up automatically. Please reply with a sentence or two and your order number if your question is about a ${storeName} order.` - - const questionNoOrderLine = `We are replying without attaching order history. If your question relates to a specific purchase, reply with your order number (e.g. #1002).` - - const noOrdersFallback = sensitiveTopic - ? `No matching recent orders on file for ${storeName} under this email. If you used a different email at checkout, reply from that address or send your order number.` - : "No matching recent orders on file for this email." - - let orderSummaryForContext: string - if (isOffTopic) { - orderSummaryForContext = offTopicOrderLine - } else if (needsOrderNumberFirst) { - orderSummaryForContext = counterOrderIdLine - } else if (hintUnmatched) { - orderSummaryForContext = hintUnmatchedOrderLine - } else if (lowConfidenceSuppressOrders) { - orderSummaryForContext = lowConfidenceOrderLine - } else if (messageTooSparse) { - orderSummaryForContext = sparseBodyOrderLine - } else if (skipOrderCacheForQuestion) { - orderSummaryForContext = questionNoOrderLine - } else { - orderSummaryForContext = orderSummary || noOrdersFallback - } - - const context: Record = { - customer_name: greetingNameFromEmail(senderEmail), - customer_email: senderEmail, - subject: emailSubject, - store_name: storeName, - store_email: store?.email || "", - ticket_summary: ticketSummary, - order_summary: orderSummaryForContext, - } - - let finalReply: string | null = null - - try { - const template = await findTemplateWithFallback( - storeId, - classification.intent, - classification.mood, - ) - if (template) { - finalReply = renderFromTemplate(template, context) - } - } catch (replyBuildError) { - logger.error("Failed to load/render template for auto-reply", replyBuildError) - } - - if (!finalReply) { - if (isOffTopic) { - finalReply = buildOffTopicFallbackHtml(storeName, store?.email || "") - } else if (needsOrderNumberFirst) { - finalReply = buildNeedOrderIdFallbackHtml(storeName, store?.email || "") - } else if (hintUnmatched) { - finalReply = buildHintUnmatchedFallbackHtml(storeName, store?.email || "") - } else { - finalReply = buildMinimalFallbackHtml(emailBody, storeName) - } - } + const finalReply = preview.html const replySubject = emailSubject.startsWith("Re:") ? emailSubject : `Re: ${emailSubject}` diff --git a/tests/gmail-auto-reply/docker-compose.test.yml b/tests/gmail-auto-reply/docker-compose.test.yml new file mode 100644 index 0000000..7f5cfa5 --- /dev/null +++ b/tests/gmail-auto-reply/docker-compose.test.yml @@ -0,0 +1,22 @@ +services: + db: + image: pgvector/pgvector:pg17 + restart: unless-stopped + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: dropshipping + ports: + - "5544:5432" + volumes: + - postgres_test_data:/var/lib/postgresql/data + - ../../prisma/init-pgvector.sql:/docker-entrypoint-initdb.d/init-pgvector.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d dropshipping"] + interval: 5s + timeout: 5s + retries: 20 + +volumes: + postgres_test_data: + diff --git a/tests/gmail-auto-reply/env.test b/tests/gmail-auto-reply/env.test new file mode 100644 index 0000000..61c2983 --- /dev/null +++ b/tests/gmail-auto-reply/env.test @@ -0,0 +1,2 @@ +DATABASE_URL=postgresql://postgres:password@localhost:5544/dropshipping?schema=public + diff --git a/tests/gmail-auto-reply/run.test.ts b/tests/gmail-auto-reply/run.test.ts new file mode 100644 index 0000000..f8bca23 --- /dev/null +++ b/tests/gmail-auto-reply/run.test.ts @@ -0,0 +1,114 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { describe, it, expect } from "vitest" +import { generateAutoReplyPreview } from "@/lib/gmail/autoReplyPreview" +import { scenarios, type Scenario } from "./scenarios" + +function projectRoot(): string { + return process.cwd() +} + +function artifactsDir(): string { + return path.join(projectRoot(), "tests", "gmail-auto-reply", "artifacts") +} + +function normalizeForContains(html: string): string { + return html.replace(/\s+/g, " ").toLowerCase() +} + +function assertScenarioInvariants(s: Scenario, out: Awaited>) { + const html = normalizeForContains(out.html) + + if (s.expect.expectNeedsOrderNumberFirst != null) { + expect(out.needsOrderNumberFirst, `${s.id}: needsOrderNumberFirst`).toBe( + s.expect.expectNeedsOrderNumberFirst, + ) + } + + if (s.expect.expectHintUnmatched != null) { + expect(out.hintUnmatched, `${s.id}: hintUnmatched`).toBe(s.expect.expectHintUnmatched) + } + + if (s.expect.expectOffTopic) { + const isOffTopic = out.classification.intent === "off_topic" || out.offTopicByHeuristic + expect(isOffTopic, `${s.id}: off_topic`).toBe(true) + } + + // Order lines presence/absence (raw summary is the safest signal; template HTML varies). + if (s.expect.expectOrderLines) { + expect(out.orderSummaryRaw.trim().length, `${s.id}: orderSummaryRaw non-empty`).toBeGreaterThan(0) + } else { + expect(out.orderSummaryRaw.trim().length, `${s.id}: orderSummaryRaw empty`).toBe(0) + } + + // A few “sanity contains” checks for the human-readable copy paths. + if (s.expect.expectNeedsOrderNumberFirst) { + expect(html, `${s.id}: counter-question copy present`).toContain("order number") + } + if (s.expect.expectHintUnmatched) { + expect(html, `${s.id}: hint-unmatched copy present`).toContain("could not match") + } + if (s.expect.expectQuestionNoOrders) { + expect(html, `${s.id}: question no-order-history copy present`).toContain("without attaching order history") + } + if (s.expect.expectOffTopic) { + expect(html, `${s.id}: off-topic boundary present`).toContain("unrelated") + } +} + +describe("gmail auto-reply scenarios (preview mode)", () => { + it("runs scenarios and writes artifacts", async () => { + const storeId = process.env.TEST_STORE_ID || "test-store" + + const results: Array<{ + id: string + title: string + skipped?: string + output?: unknown + }> = [] + + await fs.mkdir(artifactsDir(), { recursive: true }) + + for (const s of scenarios) { + if (s.skip) { + results.push({ id: s.id, title: s.title, skipped: s.skip }) + continue + } + + const out = await generateAutoReplyPreview({ + storeId, + fromEmail: s.from, + subject: s.subject, + body: s.body, + }) + + assertScenarioInvariants(s, out) + + results.push({ + id: s.id, + title: s.title, + output: { + classification: out.classification, + classifySource: out.classifySource, + orderHints: out.orderHints, + offTopicByHeuristic: out.offTopicByHeuristic, + needsOrderNumberFirst: out.needsOrderNumberFirst, + hintUnmatched: out.hintUnmatched, + lowConfidenceSuppressOrders: out.lowConfidenceSuppressOrders, + messageTooSparse: out.messageTooSparse, + skipOrderCacheForQuestion: out.skipOrderCacheForQuestion, + suppressOrdersOnFallbackClassification: out.suppressOrdersOnFallbackClassification, + orderSummaryRaw: out.orderSummaryRaw, + orderSummaryForContext: out.orderSummaryForContext, + usedFallback: out.usedFallback, + chosenTemplateId: out.chosenTemplateId, + html: out.html, + }, + }) + } + + const outPath = path.join(artifactsDir(), "results.json") + await fs.writeFile(outPath, JSON.stringify({ generatedAt: new Date().toISOString(), results }, null, 2)) + }, 120_000) +}) + diff --git a/tests/gmail-auto-reply/scenarios.ts b/tests/gmail-auto-reply/scenarios.ts new file mode 100644 index 0000000..739d069 --- /dev/null +++ b/tests/gmail-auto-reply/scenarios.ts @@ -0,0 +1,196 @@ +export type ScenarioExpectation = { + /** Expect CachedOrder lines to be included in the reply context (orderSummaryRaw non-empty). */ + expectOrderLines: boolean + /** Expect the pipeline to require order number first (counter-question path). */ + expectNeedsOrderNumberFirst?: boolean + /** Expect the hint-unmatched behavior when order # doesn't match cache. */ + expectHintUnmatched?: boolean + /** Expect off-topic boundary behavior (no orders). */ + expectOffTopic?: boolean + /** Expect question w/out order history line (no orders). */ + expectQuestionNoOrders?: boolean +} + +export type Scenario = { + id: string + title: string + from: string + subject: string + body: string + notes?: string + skip?: string + expect: ScenarioExpectation +} + +export const DEFAULT_STORE_NAME = "YOUR_STORE" +export const DEFAULT_CUSTOMER_EMAIL = "customer@gmail.com" + +export const scenarios: Scenario[] = [ + { + id: "1", + title: "Counter-question (shipping-style, no order number)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Where is my package?", + body: "Hi, I still haven't received anything. Can you check the status for me? Thanks.", + expect: { expectOrderLines: false, expectNeedsOrderNumberFirst: true }, + }, + { + id: "2", + title: "Counter-question (refund language, no order #)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Refund", + body: "I need a refund on my last purchase. Please process it.", + expect: { expectOrderLines: false, expectNeedsOrderNumberFirst: true }, + }, + { + id: "3", + title: "Counter-question (billing language, no order #)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Wrong charge", + body: "I see a duplicate charge on my card for your store. Please fix it.", + expect: { expectOrderLines: false, expectNeedsOrderNumberFirst: true }, + }, + { + id: "4", + title: "Order context shown (single clear order hint)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Order status", + body: "Hi, can you confirm status for order #1002? Thanks.", + expect: { expectOrderLines: true, expectHintUnmatched: false }, + }, + { + id: "5", + title: "Order context shown (hint in subject only)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Re: #1002 shipping", + body: "When will it ship?", + expect: { expectOrderLines: true, expectHintUnmatched: false }, + }, + { + id: "6", + title: "Hint mismatch (order # not on file for this email)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Order 9999", + body: "I need help with order #9999.", + expect: { expectOrderLines: false, expectHintUnmatched: true }, + }, + { + id: "7", + title: "Multiple order numbers, not all match cache", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Orders", + body: "Please check orders #1002 and #8888.", + expect: { expectOrderLines: false, expectHintUnmatched: true }, + }, + { + id: "8", + title: "Off-topic (no store-related request)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Weather tomorrow", + body: "What is the weather in Paris tomorrow?", + expect: { expectOrderLines: false, expectOffTopic: true }, + }, + { + id: "9", + title: "Product/policy question without order number", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Sizing question", + body: "Does the medium hoodie run true to size?", + expect: { expectOrderLines: false, expectQuestionNoOrders: true }, + }, + { + id: "10", + title: "General thank-you (may include recent orders by design)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Thanks", + body: "Thank you so much for the fast delivery and great product!", + expect: { expectOrderLines: true }, + }, + { + id: "11", + title: "Very short body (sparse message)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Hi", + body: "Help", + expect: { expectOrderLines: false }, + }, + { + id: "12", + title: "Low confidence (general/question only)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Hello", + body: "Just checking in.", + skip: "Hard to force with live Groq; invariants depend on model confidence.", + expect: { expectOrderLines: false }, + }, + { + id: "13", + title: "Quoted thread with stale order # in reply chain", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Still waiting", + body: "Still waiting on my new order.\n\nOn Mon, Jan 1, 2026 at 9:00 AM wrote:\n> Order #9999 from old ticket", + expect: { expectOrderLines: false }, + }, + { + id: "14", + title: "Groq unavailable / bad JSON (classification fallback)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Where is my package?", + body: "Any update?", + skip: "Requires intentionally breaking GROQ_API_KEY; skipped in live-Groq mode.", + expect: { expectOrderLines: false, expectNeedsOrderNumberFirst: true }, + }, + { + id: "15", + title: "No CachedOrder for sender (guest / wrong email)", + from: "other.person@gmail.com", + subject: "Order help", + body: "I placed an order last week, can you find it?", + expect: { expectOrderLines: false }, + }, + { + id: "16", + title: "Safety / abuse screen (no auto-reply)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Test", + body: "You are an idiot and I will ruin your life.", // may or may not trigger safety depending on rules + skip: "Depends on gmailSafety rules; keep manual unless you confirm a deterministic trigger phrase.", + expect: { expectOrderLines: false }, + }, + { + id: "17", + title: "Duplicate inbound message (idempotency)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Order status", + body: "Hi, can you confirm status for order #1002? Thanks.", + skip: "Idempotency is enforced via Gmail message id + lease in production; preview mode does not model GmailInboundAutoReply.", + expect: { expectOrderLines: true }, + }, + { + id: "18", + title: "Per-sender rate limit", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Ping", + body: "Status?", + skip: "Rate limits are enforced in processIncomingEmail; preview mode does not perform rate checks.", + expect: { expectOrderLines: false }, + }, + { + id: "19", + title: "Non-Gmail sender skipped", + from: "customer@company.com", + subject: "Order help", + body: "Where is my order?", + skip: "Non-gmail sender skip is enforced in processIncomingEmail; preview mode focuses on reply generation only.", + expect: { expectOrderLines: false }, + }, + { + id: "20", + title: "HTML escaping (security smoke test)", + from: DEFAULT_CUSTOMER_EMAIL, + subject: "Question", + body: "Question: and order #1002", + expect: { expectOrderLines: true }, + }, +] + diff --git a/tests/gmail-auto-reply/seed.ts b/tests/gmail-auto-reply/seed.ts new file mode 100644 index 0000000..f66c2b1 --- /dev/null +++ b/tests/gmail-auto-reply/seed.ts @@ -0,0 +1,174 @@ +import "dotenv/config" +import { prisma } from "@/lib/prisma" +import { insertTemplate } from "@/lib/juno/templatesRepo" + +function requireEnv(name: string): string { + const v = process.env[name] + if (!v) throw new Error(`Missing required env var: ${name}`) + return v +} + +async function main() { + // This relies on Prisma using DATABASE_URL from env.test (loaded by the runner script). + // Create a minimal user + active store, plus a few templates and CachedOrder rows. + const userEmail = process.env.TEST_VENDOR_EMAIL || "vendor@example.com" + + const user = await prisma.user.upsert({ + where: { email: userEmail }, + update: { role: "VENDOR" }, + create: { + email: userEmail, + role: "VENDOR", + }, + select: { id: true }, + }) + + const store = await prisma.store.upsert({ + where: { id: process.env.TEST_STORE_ID || "test-store" }, + update: { + userId: user.id, + isActive: true, + businessName: process.env.TEST_STORE_NAME || "YOUR_STORE", + email: process.env.TEST_STORE_EMAIL || "support@yourstore.test", + timezone: "UTC", + country: "US", + shopifyStoreName: "test", + shopifyDomain: "test.myshopify.com", + shopifyApiKey: "test", + shopifyApiSecret: "test", + shopifyAccessToken: "test", + }, + create: { + id: process.env.TEST_STORE_ID || "test-store", + userId: user.id, + isActive: true, + businessName: process.env.TEST_STORE_NAME || "YOUR_STORE", + email: process.env.TEST_STORE_EMAIL || "support@yourstore.test", + timezone: "UTC", + country: "US", + shopifyStoreName: "test", + shopifyDomain: "test.myshopify.com", + shopifyApiKey: "test", + shopifyApiSecret: "test", + shopifyAccessToken: "test", + }, + select: { id: true }, + }) + + // Ensure the app logic sees an active store for the user. + await prisma.user.update({ + where: { id: user.id }, + data: { activeStoreId: store.id }, + }) + + // Minimal templates so we get stable rendering without requiring a full catalog. + // These templates reference {{ticket_summary}} and {{order_summary}}. + const templateBody = ` +
+

Hello {{customer_name}},

+

{{ticket_summary}}

+

{{order_summary}}

+

— {{store_name}} Support

+
+`.trim() + + const intents = [ + ["general", "neutral"], + ["general", "happy"], + ["shipping_delay", "neutral"], + ["refund_request", "neutral"], + ["account_billing", "neutral"], + ["question", "neutral"], + ["off_topic", "neutral"], + ] as const + + for (const [intent, mood] of intents) { + await insertTemplate({ + storeId: store.id, + name: `Test template — ${intent}/${mood}`, + intent, + mood, + description: "Test harness template", + body: templateBody, + variables: { ticket_summary: true, order_summary: true }, + isHtml: true, + }).catch(() => { + // If duplicate-ish templates exist, ignore; findTemplateWithFallback selects latest. + }) + } + + const customerEmail = process.env.TEST_CUSTOMER_EMAIL || "customer@gmail.com" + + const baseLineItems = [ + { title: "Hoodie", quantity: 1, product_id: 111 }, + { title: "Socks", quantity: 2, product_id: 222 }, + ] + + const now = new Date() + + const orders = [ + { orderNumber: "1002", shopifyOrderId: "gid://shopify/Order/1002", totalPrice: "49.99" }, + { orderNumber: "1003", shopifyOrderId: "gid://shopify/Order/1003", totalPrice: "19.99" }, + ] + + for (const o of orders) { + await prisma.cachedOrder.upsert({ + where: { + shopifyOrderId_storeId: { + shopifyOrderId: o.shopifyOrderId, + storeId: store.id, + }, + }, + update: { + orderNumber: o.orderNumber, + email: customerEmail, + totalPrice: o.totalPrice, + currency: "USD", + financialStatus: "paid", + fulfillmentStatus: "unfulfilled", + lineItems: baseLineItems as unknown as object, + customer: { email: customerEmail } as unknown as object, + shopifyCreatedAt: now, + lastSyncedAt: now, + }, + create: { + shopifyOrderId: o.shopifyOrderId, + storeId: store.id, + orderNumber: o.orderNumber, + email: customerEmail, + totalPrice: o.totalPrice, + currency: "USD", + financialStatus: "paid", + fulfillmentStatus: "unfulfilled", + lineItems: baseLineItems as unknown as object, + customer: { email: customerEmail } as unknown as object, + shopifyCreatedAt: now, + lastSyncedAt: now, + }, + }) + } + + console.log( + JSON.stringify( + { + ok: true, + storeId: store.id, + userEmail, + customerEmail, + seededOrders: orders.map((o) => o.orderNumber), + }, + null, + 2, + ), + ) +} + +main() + .catch((e) => { + console.error(e) + process.exit(1) + }) + .finally(async () => { + await prisma.$disconnect() + }) +