Show suspended since and new emails for suspend continue approve and rejection

src/app/[locale]/dashboard/page.tsx

@@ -2,7 +2,10 @@ import { getSessionUser, canMutate } from "@/lib/session";
 import { getTranslations, getFormatter } from "next-intl/server";
 import { redirect } from "next/navigation";
 import { listTenants } from "@/lib/k8s";
-import { listActiveTenantRequestsByOrgId } from "@/lib/db";
+import {
+  listActiveTenantRequestsByOrgId,
+  syncProvisioningStatuses,
+} from "@/lib/db";
 import {
   listVisibleTenants,
   canSeeInflightRequests,
@@ -160,6 +163,23 @@ export default async function DashboardPage() {
 
   // Pending/in-flight requests are only shown to roles that can act on
   // them. `user`-role customers see no request cards.
+  //
+  // syncProvisioningStatuses runs on every dashboard load: it walks
+  // active and provisioning rows and reconciles them against the
+  // current cluster state. Without this, the operator-initiated
+  // 60-day TTL deletion (Bug 37b) leaves the portal showing "Your
+  // assistant is ready!" cards for tenants that no longer exist —
+  // the operator deletes the CR, but the DB row stays at active=true
+  // until something updates it. Running the sync at every dashboard
+  // load keeps the portal eventually consistent with the cluster
+  // without needing a separate cron/job.
+  //
+  // Cost: one K8s GET per row in (active, provisioning) status. At
+  // pilot scale this is small; if it grows we'd cache or move to a
+  // periodic background job.
+  if (canSeeInflightRequests(user)) {
+    await syncProvisioningStatuses();
+  }
   const orgRequests = canSeeInflightRequests(user)
     ? await listActiveTenantRequestsByOrgId(user.orgId)
     : [];

src/app/[locale]/tenants/[name]/page.tsx

@@ -142,6 +142,53 @@ export default async function TenantDetailPage({
               <div className="text-xs text-text-secondary mt-1">
                 {t("suspendedDescription")}
               </div>
+              {/* Retention countdown. suspendedAt is stamped by the
+                  operator on first transition to suspended; missing
+                  values fall through silently rather than rendering
+                  garbage (operator hasn't reconciled yet, edge case).
+                  The 60-day window is the operator's
+                  retentionAfterSuspend constant; if you change one,
+                  change both. We don't expose the constant via API —
+                  the value rarely changes and duplicating it here
+                  beats fetching a single int over the network. */}
+              {tenant.status?.suspendedAt && (() => {
+                const suspendedAt = new Date(tenant.status.suspendedAt);
+                const deletionAt = new Date(suspendedAt);
+                deletionAt.setDate(deletionAt.getDate() + 60);
+                const now = new Date();
+                const msRemaining = deletionAt.getTime() - now.getTime();
+                const daysRemaining = Math.max(
+                  0,
+                  Math.ceil(msRemaining / (1000 * 60 * 60 * 24))
+                );
+                // < 7 days: red/critical to draw attention. Otherwise
+                // amber, matching the banner.
+                const urgent = daysRemaining < 7;
+                return (
+                  <div
+                    className={`text-xs mt-2 ${
+                      urgent ? "text-red-400" : "text-text-muted"
+                    }`}
+                  >
+                    {t("suspendedSince", {
+                      date: formatDateTime(
+                        tenant.status.suspendedAt,
+                        f
+                      ),
+                    })}
+                    {" · "}
+                    {daysRemaining > 0
+                      ? t("suspendedDeletionIn", {
+                          days: daysRemaining,
+                          date: formatDateTime(
+                            deletionAt.toISOString(),
+                            f
+                          ),
+                        })
+                      : t("suspendedDeletionImminent")}
+                  </div>
+                );
+              })()}
             </div>
           </div>
         </div>

src/app/api/admin/requests/[id]/approve/route.ts

@@ -6,7 +6,7 @@ import {
   clearEncryptedSecrets,
 } from "@/lib/db";
 import { createTenant, patchTenantSpec, setTenantAnnotation } from "@/lib/k8s";
-import { sendApprovalEmail } from "@/lib/email";
+import { sendApprovalEmail, sendResumeApprovalEmail } from "@/lib/email";
 import { decryptSecrets } from "@/lib/crypto";
 import { writePackageSecrets } from "@/lib/openbao";
 import {
@@ -105,11 +105,11 @@ export async function POST(
 
       await updateTenantRequestStatus(id, "approved", { adminNotes });
 
-      await sendApprovalEmail(
+      await sendResumeApprovalEmail(
         tenantRequest.contactEmail,
         tenantRequest.contactName,
         tenantRequest.companyName
-      ).catch((e) => console.error("approval email failed:", e));
+      ).catch((e) => console.error("resume approval email failed:", e));
 
       return NextResponse.json({
         message: "Resume approved. Tenant is reactivating.",

src/app/api/admin/requests/[id]/reject/route.ts

@@ -2,7 +2,7 @@ import { NextResponse } from "next/server";
 import { requirePlatformRole } from "@/lib/session";
 import { getTenantRequestById, updateTenantRequestStatus } from "@/lib/db";
 import { setTenantAnnotation } from "@/lib/k8s";
-import { sendRejectionEmail } from "@/lib/email";
+import { sendRejectionEmail, sendResumeRejectionEmail } from "@/lib/email";
 
 /**
  * POST /api/admin/requests/[id]/reject
@@ -65,13 +65,25 @@ export async function POST(
     }
   }
 
-  // Notify customer
+  // Notify customer. Resume requests get a different email — the
+  // tenant already exists; copy needs to mention "stays suspended" and
+  // the 60-day retention deadline. Provision rejections use the
+  // original onboarding-rejection wording.
+  if (tenantRequest.requestType === "resume") {
+    await sendResumeRejectionEmail(
+      tenantRequest.contactEmail,
+      tenantRequest.contactName,
+      tenantRequest.companyName,
+      adminNotes
+    );
+  } else {
     await sendRejectionEmail(
       tenantRequest.contactEmail,
       tenantRequest.contactName,
       tenantRequest.companyName,
       adminNotes
     );
+  }
 
   return NextResponse.json({
     message: "Request rejected.",

src/lib/db.ts

@@ -63,9 +63,14 @@ const MIGRATION_SQL = `
   CREATE INDEX IF NOT EXISTS idx_tenant_requests_status ON tenant_requests(status);
   CREATE INDEX IF NOT EXISTS idx_tenant_requests_org_id ON tenant_requests(zitadel_org_id);
   CREATE INDEX IF NOT EXISTS idx_tenant_requests_org_status ON tenant_requests(zitadel_org_id, status);
-  CREATE UNIQUE INDEX IF NOT EXISTS uniq_tenant_requests_tenant_name
+  -- Note: the unique constraint on tenant_name is NOT created here.
-    ON tenant_requests(tenant_name)
+  -- Pre-Bug-37 we had a non-partial UNIQUE on tenant_name, which is
-    WHERE tenant_name IS NOT NULL;
+  -- incompatible with resume requests (same tenant_name, different
+  -- request_type). The new partial unique indexes are created
+  -- further down in the migration block, after the request_type
+  -- column has been added and backfilled. This bootstrap section
+  -- only creates indexes that are safe regardless of request_type
+  -- semantics.
 
   -- Idempotent column adds for existing databases
   ALTER TABLE tenant_requests ADD COLUMN IF NOT EXISTS encrypted_secrets BYTEA;
@@ -639,8 +644,33 @@ export async function deleteTenantRequest(id: string): Promise<void> {
 }
 
 /**
- * Sync provisioning statuses: for all requests with status "provisioning",
+ * Reconcile the portal's tenant_requests table against actual cluster
- * check if the PiecedTenant CR has reached "Ready" and update to "active".
+ * state. Three passes, walking only rows with `tenant_name` set:
+ *
+ *   1. provisioning → active: when a tenant CR's phase reaches Ready
+ *      or Running, the portal flips the row to active so the
+ *      "provisioning…" card transitions into the running tenant view.
+ *
+ *   2. active/provisioning → deleted: when the corresponding CR no
+ *      longer exists in the cluster (404), or is mid-deletion (has
+ *      metadata.deletionTimestamp set), the row gets flipped to
+ *      `deleted`. The DB is otherwise blind to operator-initiated
+ *      deletions — when the 60-day TTL fires (Bug 37b) and the
+ *      operator deletes a suspended tenant, the portal would happily
+ *      keep showing the "Your assistant is ready!" card forever.
+ *      Without this reconciliation the dashboard drifts from reality.
+ *
+ *   3. pending resume → cancelled: when a pending resume request's
+ *      tenant is no longer suspended (admin resumed it directly,
+ *      tenant was deleted, or it was never suspended in the first
+ *      place), the request is moot. Flip to 'cancelled' so the
+ *      pending-resume unique index releases for any future genuine
+ *      resume request. We pick `cancelled` over `rejected` because
+ *      the customer didn't do anything wrong — circumstances just
+ *      changed.
+ *
+ * Errors are tolerated per-row: a transient API hiccup on one tenant
+ * shouldn't fail the whole sweep. Skipped rows get retried next call.
  *
  * Slice 3 note: with multi-tenant per org, this iterates each row
  * individually (keyed by its own tenant_name), so multiple in-flight
@@ -648,25 +678,79 @@ export async function deleteTenantRequest(id: string): Promise<void> {
  */
 export async function syncProvisioningStatuses(): Promise<void> {
   await ensureSchema();
+  // Active+provisioning rows: status reflects "the tenant should
+  // exist and be running".
+  // Pending resume rows: status reflects "the tenant is suspended,
+  // awaiting reactivation".
+  // Both need cluster-side validation; we fetch them in one query
+  // and dispatch on (status, request_type).
   const result = await getPool().query<TenantRequest>(
-    "SELECT * FROM tenant_requests WHERE status = 'provisioning'"
+    `SELECT * FROM tenant_requests
+       WHERE tenant_name IS NOT NULL
+         AND (
+           status IN ('provisioning', 'active')
+           OR (status = 'pending' AND request_type = 'resume')
+         )`
   );
 
   for (const row of result.rows) {
     const mapped = mapRow(row);
     if (!mapped.tenantName) continue;
 
+    let tenant: Awaited<ReturnType<typeof getTenant>> = null;
     try {
-      const tenant = await getTenant(mapped.tenantName);
+      tenant = await getTenant(mapped.tenantName);
+    } catch {
+      // Transient API error — skip this row, retry on next sweep.
+      continue;
+    }
+
+    // Pending resume request: validity hinges on tenant being suspended.
     if (
-        tenant?.status?.phase === "Ready" ||
+      mapped.status === "pending" &&
-        tenant?.status?.phase === "Running"
+      mapped.requestType === "resume"
+    ) {
+      // Tenant doesn't exist or is being deleted: cancel the resume
+      // request (it can never be fulfilled). Don't fall through to
+      // the "deleted" branch below — that would also flip the
+      // provision row, which is the right thing for a CR-level
+      // deletion but we want this resume row specifically resolved
+      // here.
+      if (!tenant || tenant.metadata.deletionTimestamp) {
+        await updateTenantRequestStatus(mapped.id, "cancelled");
+        continue;
+      }
+      // Tenant is no longer suspended: the request is moot.
+      // Cancel it (the customer didn't do anything wrong; the
+      // condition the request was about no longer applies).
+      if (!tenant.spec.suspend) {
+        await updateTenantRequestStatus(mapped.id, "cancelled");
+        continue;
+      }
+      // Tenant still suspended, request still relevant. Leave as-is.
+      continue;
+    }
+
+    // Active or provisioning row: CR gone, or mid-deletion. Flip the
+    // row to 'deleted'. `markTenantRequestDeletedByTenantName` flips
+    // every row with this tenant_name (provision + any resume rows),
+    // which is the right thing for a CR-level deletion.
+    if (!tenant || tenant.metadata.deletionTimestamp) {
+      await markTenantRequestDeletedByTenantName(mapped.tenantName);
+      continue;
+    }
+
+    // CR exists and is healthy. Promote provisioning → active when
+    // the operator reports the tenant has reached steady state.
+    // Keep `active` rows on `active` regardless of phase — a
+    // temporarily-Reconfiguring tenant is still active from the
+    // portal's billing/visibility perspective.
+    if (
+      mapped.status === "provisioning" &&
+      (tenant.status?.phase === "Ready" || tenant.status?.phase === "Running")
     ) {
       await updateTenantRequestStatus(mapped.id, "active");
     }
-    } catch {
-      // Tenant might not exist yet — skip
-    }
   }
 }
 

src/lib/email.ts

@@ -156,6 +156,121 @@ export async function sendRejectionEmail(
   }
 }
 
+/**
+ * Bug 37a: separate email for resume request approval. The tenant
+ * already exists; the message is "we're un-suspending it" rather than
+ * "we're provisioning a new instance". Avoids confusing the customer
+ * with onboarding language for a tenant they already had.
+ */
+export async function sendResumeApprovalEmail(
+  to: string,
+  contactName: string,
+  companyName: string
+): Promise<void> {
+  const safeName = escapeHtml(contactName);
+  const safeCompany = escapeHtml(companyName);
+
+  try {
+    await getTransporter().sendMail({
+      from: getFrom(),
+      to,
+      subject: `Your PieCed AI assistant has been reactivated — ${companyName}`,
+      text: [
+        `Hello ${contactName},`,
+        "",
+        `Good news — your reactivation request for ${companyName} has been approved.`,
+        "",
+        "Your AI assistant is being brought back online and should be ready in a few minutes.",
+        "You can check the status in your dashboard at https://app.pieced.ch",
+        "",
+        "Best regards,",
+        "PieCed IT",
+      ].join("\n"),
+      html: `
+        <div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; max-width: 560px; margin: 0 auto; color: #e0e0e0; background: #1a1a1a; padding: 32px; border-radius: 12px;">
+          <h2 style="color: #ffffff; margin-top: 0;">Your AI assistant has been reactivated</h2>
+          <p>Hello ${safeName},</p>
+          <p>Good news — your reactivation request for <strong>${safeCompany}</strong> has been approved.</p>
+          <p>Your AI assistant is being brought back online and should be ready in a few minutes.</p>
+          <p>
+            <a href="https://app.pieced.ch" style="display: inline-block; padding: 10px 24px; background: #3b82f6; color: #ffffff; text-decoration: none; border-radius: 8px; font-weight: 500;">
+              Go to Dashboard
+            </a>
+          </p>
+          <hr style="border: none; border-top: 1px solid #333; margin: 24px 0;" />
+          <p style="color: #666; font-size: 12px;">PieCed IT — Hosted on-premises in Switzerland</p>
+        </div>
+      `,
+    });
+  } catch (err) {
+    console.error("Failed to send resume approval email:", err);
+  }
+}
+
+/**
+ * Bug 37a: separate email for resume request rejection. Differs from
+ * the onboarding rejection in two ways: it explicitly mentions the
+ * tenant remains suspended, and it points the customer to the
+ * 60-day retention window so they understand the deletion clock is
+ * still ticking. The latter is important — a customer reading a
+ * generic "request rejected" email might not realise their data is
+ * still on a countdown.
+ */
+export async function sendResumeRejectionEmail(
+  to: string,
+  contactName: string,
+  companyName: string,
+  adminNotes?: string
+): Promise<void> {
+  const safeName = escapeHtml(contactName);
+  const safeCompany = escapeHtml(companyName);
+  const safeNotes = adminNotes ? escapeHtml(adminNotes) : "";
+
+  try {
+    const notesBlock = adminNotes
+      ? `\nNote from our team:\n${adminNotes}\n`
+      : "";
+    const notesHtml = safeNotes
+      ? `<div style="background: #2a2a2a; border-left: 3px solid #ef4444; padding: 12px 16px; border-radius: 6px; margin: 16px 0;">
+           <p style="color: #ccc; font-size: 13px; margin: 0;"><strong>Note from our team:</strong></p>
+           <p style="color: #aaa; font-size: 13px; margin: 8px 0 0 0;">${safeNotes}</p>
+         </div>`
+      : "";
+
+    await getTransporter().sendMail({
+      from: getFrom(),
+      to,
+      subject: `Update on your reactivation request — ${companyName}`,
+      text: [
+        `Hello ${contactName},`,
+        "",
+        `Thank you for your reactivation request for ${companyName}. Unfortunately, we were unable to approve it at this time.`,
+        notesBlock,
+        "Your tenant remains suspended. As a reminder, your data is preserved for 60 days from the original cancellation date, after which it will be permanently deleted. You can submit a new reactivation request at any time before then.",
+        "",
+        "If you have questions, please reply to this email.",
+        "",
+        "Best regards,",
+        "PieCed IT",
+      ].join("\n"),
+      html: `
+        <div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; max-width: 560px; margin: 0 auto; color: #e0e0e0; background: #1a1a1a; padding: 32px; border-radius: 12px;">
+          <h2 style="color: #ffffff; margin-top: 0;">Update on your reactivation request</h2>
+          <p>Hello ${safeName},</p>
+          <p>Thank you for your reactivation request for <strong>${safeCompany}</strong>. Unfortunately, we were unable to approve it at this time.</p>
+          ${notesHtml}
+          <p>Your tenant remains suspended. As a reminder, your data is preserved for 60 days from the original cancellation date, after which it will be permanently deleted. You can submit a new reactivation request at any time before then.</p>
+          <p>If you have questions, please reply to this email.</p>
+          <hr style="border: none; border-top: 1px solid #333; margin: 24px 0;" />
+          <p style="color: #666; font-size: 12px;">PieCed IT — Hosted on-premises in Switzerland</p>
+        </div>
+      `,
+    });
+  } catch (err) {
+    console.error("Failed to send resume rejection email:", err);
+  }
+}
+
 export async function sendAdminNotificationEmail(
   companyName: string,
   contactName: string,

src/messages/de.json

@@ -166,7 +166,10 @@
     "resumeRequestPendingTitle": "Reaktivierungsanfrage ausstehend",
     "resumeRequestPendingDescription": "Eingereicht {when}. Ein Administrator wird die Anfrage in Kürze prüfen.",
     "resumeRequestPendingNoteAdmin": "Ein Inhaber hat eine Reaktivierung angefragt; Sie können direkt oben fortfahren oder die Anfrage in der Admin-Warteschlange bearbeiten.",
-    "cancelConfirmRetentionWarning": "Ihre Daten bleiben nach der Kündigung 60 Tage lang erhalten. Danach werden alle Tenant-Daten – Konfiguration, Geheimnisse, Konversationen und Dateien – endgültig gelöscht."
+    "cancelConfirmRetentionWarning": "Ihre Daten bleiben nach der Kündigung 60 Tage lang erhalten. Danach werden alle Tenant-Daten – Konfiguration, Geheimnisse, Konversationen und Dateien – endgültig gelöscht.",
+    "suspendedSince": "Gekündigt am {date}",
+    "suspendedDeletionIn": "Datenlöschung in {days, plural, one {# Tag} other {# Tagen}} ({date})",
+    "suspendedDeletionImminent": "Daten werden jetzt gelöscht"
   },
   "usage": {
     "inputTokens": "Input-Tokens",

src/messages/en.json

@@ -166,7 +166,10 @@
     "resumeRequestPendingTitle": "Reactivation request pending",
     "resumeRequestPendingDescription": "Submitted {when}. An administrator will review it shortly.",
     "resumeRequestPendingNoteAdmin": "An owner has requested reactivation; you can resume directly above or process the request from the admin queue.",
-    "cancelConfirmRetentionWarning": "Your data is preserved for 60 days after cancellation. After that, all tenant data — configuration, secrets, conversations, and files — will be permanently deleted."
+    "cancelConfirmRetentionWarning": "Your data is preserved for 60 days after cancellation. After that, all tenant data — configuration, secrets, conversations, and files — will be permanently deleted.",
+    "suspendedSince": "Suspended on {date}",
+    "suspendedDeletionIn": "data deletion in {days, plural, one {# day} other {# days}} ({date})",
+    "suspendedDeletionImminent": "data is being deleted now"
   },
   "usage": {
     "inputTokens": "Input Tokens",

src/messages/fr.json

@@ -166,7 +166,10 @@
     "resumeRequestPendingTitle": "Demande de réactivation en attente",
     "resumeRequestPendingDescription": "Soumise {when}. Un administrateur l'examinera sous peu.",
     "resumeRequestPendingNoteAdmin": "Un propriétaire a demandé la réactivation ; vous pouvez reprendre directement ci-dessus ou traiter la demande depuis la file d'attente d'administration.",
-    "cancelConfirmRetentionWarning": "Vos données sont conservées pendant 60 jours après l'annulation. Passé ce délai, toutes les données du locataire — configuration, secrets, conversations et fichiers — seront définitivement supprimées."
+    "cancelConfirmRetentionWarning": "Vos données sont conservées pendant 60 jours après l'annulation. Passé ce délai, toutes les données du locataire — configuration, secrets, conversations et fichiers — seront définitivement supprimées.",
+    "suspendedSince": "Suspendu le {date}",
+    "suspendedDeletionIn": "suppression des données dans {days, plural, one {# jour} other {# jours}} ({date})",
+    "suspendedDeletionImminent": "les données sont en cours de suppression"
   },
   "usage": {
     "inputTokens": "Tokens d'entrée",

src/messages/it.json

@@ -166,7 +166,10 @@
     "resumeRequestPendingTitle": "Richiesta di riattivazione in sospeso",
     "resumeRequestPendingDescription": "Inviata {when}. Un amministratore la esaminerà a breve.",
     "resumeRequestPendingNoteAdmin": "Un proprietario ha richiesto la riattivazione; puoi riprendere direttamente sopra o elaborare la richiesta dalla coda di amministrazione.",
-    "cancelConfirmRetentionWarning": "I tuoi dati sono conservati per 60 giorni dopo l'annullamento. Trascorso tale periodo, tutti i dati del tenant — configurazione, segreti, conversazioni e file — verranno eliminati definitivamente."
+    "cancelConfirmRetentionWarning": "I tuoi dati sono conservati per 60 giorni dopo l'annullamento. Trascorso tale periodo, tutti i dati del tenant — configurazione, segreti, conversazioni e file — verranno eliminati definitivamente.",
+    "suspendedSince": "Sospeso il {date}",
+    "suspendedDeletionIn": "eliminazione dei dati tra {days, plural, one {# giorno} other {# giorni}} ({date})",
+    "suspendedDeletionImminent": "i dati vengono eliminati ora"
   },
   "usage": {
     "inputTokens": "Token di input",

src/types/index.ts

@@ -103,6 +103,16 @@ export interface PiecedTenantStatus {
   litellmKeyAlias?: string;
   tenantNamespace?: string;
   enabledPackages?: string[];
+  /**
+   * RFC3339 timestamp of when the tenant first transitioned to
+   * suspended (Bug 37). Stamped by the operator on the first reconcile
+   * with `spec.suspend=true` and cleared when the tenant resumes. Used
+   * by the portal to render the "deleted in N days" countdown in the
+   * suspended banner. The retention policy is 60 days from this
+   * timestamp; see operator's `retentionAfterSuspend` constant for the
+   * authoritative value.
+   */
+  suspendedAt?: string;
   /**
    * Non-fatal issues from downstream resources surfaced by the operator
    * (e.g. an OpenClawInstance sub-condition reporting failure). The
@@ -134,6 +144,15 @@ export interface PiecedTenant {
     name: string;
     namespace?: string;
     creationTimestamp?: string;
+    /**
+     * Set by the API server when something issues a Delete on the CR.
+     * The CR continues to exist while finalizers run cleanup; once
+     * they all remove themselves, the API server permanently removes
+     * the CR. Used by the portal's status sync to detect tenants
+     * being torn down — the customer should see "Deleted" rather
+     * than "Ready" while the cleanup runs.
+     */
+    deletionTimestamp?: string;
     labels?: Record<string, string>;
     annotations?: Record<string, string>;
   };