Momento/memento-note/lib/ai/services/memory-echo.service.ts

import { getChatProvider } from '../factory'
import { cosineSimilarity } from '@/lib/utils'
import { embeddingService } from './embedding.service'
import { getSystemConfig } from '@/lib/config'
import prisma from '@/lib/prisma'
import { Prisma } from '@prisma/client'
import { upsertNoteEmbedding } from '@/lib/embeddings'
import {
  excerptPlainNoteContent,
  prepareNoteTextForEmbedding,
} from '@/lib/text/plain-text'
import { detectTextDirection } from '@/lib/clip/rtl-content'
import { detectUserLanguage } from '@/lib/i18n/detect-user-language'
import type { SupportedLanguage } from '@/lib/i18n/load-translations'
import {
  buildMemoryEchoInsightPrompt,
  getMemoryEchoInsightFallback,
} from '@/lib/ai/memory-echo-i18n'
import {
  SEMANTIC_SIMILARITY_FLOOR_CLIP,
  SEMANTIC_SIMILARITY_FLOOR_DEMO,
  SEMANTIC_SIMILARITY_FLOOR,
} from '@/lib/ai/semantic-proximity'

export interface NoteConnection {
  note1: {
    id: string
    title: string | null
    content: string
    createdAt: Date
  }
  note2: {
    id: string
    title: string | null
    content: string | null
    createdAt: Date
  }
  similarityScore: number
  insight: string
  daysApart: number
}

export interface MemoryEchoInsight {
  id: string
  note1Id: string
  note2Id: string
  note1: {
    id: string
    title: string | null
    content: string
  }
  note2: {
    id: string
    title: string | null
    content: string
  }
  similarityScore: number
  insight: string
  insightDate: Date
  viewed: boolean
  feedback: string | null
}

/**
 * Memory Echo Service - Proactive note connections
 * "I didn't search, it found me"
 */
export class MemoryEchoService {
  private readonly SIMILARITY_THRESHOLD = SEMANTIC_SIMILARITY_FLOOR
  private readonly SIMILARITY_THRESHOLD_DEMO = SEMANTIC_SIMILARITY_FLOOR_DEMO
  private readonly SIMILARITY_THRESHOLD_CLIP = SEMANTIC_SIMILARITY_FLOOR_CLIP
  private readonly MIN_DAYS_APART = 7 // Notes must be at least 7 days apart
  private readonly MIN_DAYS_APART_CLIP = 0 // Notes clippées (sourceUrl) : même jour OK
  private readonly MIN_DAYS_APART_DEMO = 0 // No delay for demo mode
  private readonly MAX_INSIGHTS_PER_USER = 100 // Prevent spam

  private isClippedNote(note: { sourceUrl?: string | null }): boolean {
    return Boolean(note.sourceUrl?.trim())
  }

  private passesTimeDiversityFilter(
    daysApart: number,
    noteA: { sourceUrl?: string | null },
    noteB: { sourceUrl?: string | null },
    demoMode: boolean,
  ): boolean {
    if (demoMode) return true
    const minDays =
      this.isClippedNote(noteA) || this.isClippedNote(noteB)
        ? this.MIN_DAYS_APART_CLIP
        : this.MIN_DAYS_APART
    return daysApart >= minDays
  }

  private isRtlOrClipNote(note: {
    sourceUrl?: string | null
    content?: string
    title?: string | null
  }): boolean {
    if (this.isClippedNote(note)) return true
    if (note.content?.includes('clip-article--rtl')) return true
    const sample = prepareNoteTextForEmbedding(note.title, note.content || '')
    return detectTextDirection(sample) === 'rtl'
  }

  private pairSimilarityThreshold(
    noteA: { sourceUrl?: string | null; content?: string; title?: string | null },
    noteB: { sourceUrl?: string | null; content?: string; title?: string | null },
    demoMode: boolean,
  ): number {
    if (demoMode) return this.SIMILARITY_THRESHOLD_DEMO
    if (this.isRtlOrClipNote(noteA) || this.isRtlOrClipNote(noteB)) {
      return this.SIMILARITY_THRESHOLD_CLIP
    }
    return this.SIMILARITY_THRESHOLD
  }

  /** Texte plain complet envoyé à l'API / résolution de blocs (pas de troncature). */
  private connectionPlainText(
    title: string | null,
    content: string,
  ): string {
    return prepareNoteTextForEmbedding(title, content)
  }

  private async upsertNoteEmbeddingFromNote(note: {
    id: string
    title: string | null
    content: string
  }): Promise<number[] | null> {
    const text = prepareNoteTextForEmbedding(note.title, note.content)
    if (!text.trim()) return null
    try {
      const { embedding } = await embeddingService.generateNoteEmbedding(note.title, note.content)
      if (embedding?.length) {
        await upsertNoteEmbedding(note.id, embedding)
        return embedding
      }
    } catch (error) {
      console.error(`[MemoryEcho] embedding failed for note ${note.id}:`, error)
    }
    return null
  }

  /**
   * Generate embeddings for notes that don't have one yet
   */
  private async ensureEmbeddings(userId: string): Promise<void> {
    const notes = await prisma.note.findMany({
      where: {
        userId,
        isArchived: false,
        trashedAt: null,
      },
      select: {
        id: true,
        title: true,
        content: true,
        sourceUrl: true,
        noteEmbedding: { select: { noteId: true } },
      },
    })

    if (notes.length === 0) return

    for (const note of notes) {
      if (!note.content?.trim()) continue
      const isClip = this.isClippedNote(note)
      const missing = !note.noteEmbedding
      if (!missing && !isClip) continue
      await this.upsertNoteEmbeddingFromNote(note)
    }
  }

  /**
   * Find meaningful connections between user's notes
   */
  async findConnections(userId: string, demoMode: boolean = false): Promise<NoteConnection[]> {
    // GDPR AI Consent check — compliance skip if not granted (AC6)
    const userSettings = await prisma.userAISettings.findUnique({
      where: { userId },
      select: { aiProcessingConsent: true },
    })
    if (!userSettings?.aiProcessingConsent) {
      return []
    }

    // Ensure all notes have embeddings before searching for connections
    await this.ensureEmbeddings(userId)

    // Get all user's notes with embeddings
    const notes = await prisma.note.findMany({
      where: {
        userId,
        isArchived: false,
        trashedAt: null,
        // noteEmbedding: { isNot: null } // Removed because Unsupported type cannot be used in 'where' easily
      },
      select: {
        id: true,
        title: true,
        content: true,
        sourceUrl: true,
        noteEmbedding: true,
        createdAt: true
      },
      orderBy: { createdAt: 'desc' }
    })

    if (notes.length < 2) {
      return [] // Need at least 2 notes to find connections
    }

    if (notes.length < 2) return []

    // Fetch embeddings separately using raw SQL to avoid deserialization error
    const noteIds = notes.map(n => n.id)
    const embeddings = noteIds.length === 0 ? [] : await prisma.$queryRaw<Array<{ noteId: string, embedding: any }>>(
      Prisma.sql`SELECT "noteId", "embedding"::text FROM "NoteEmbedding" WHERE "noteId" IN (${Prisma.join(noteIds)})`
    )
    const embeddingMap = new Map(embeddings.map(e => [e.noteId, e.embedding]))

    const notesWithEmbeddings = notes
      .map(note => ({
        ...note,
        embedding: embeddingMap.has(note.id)
          ? embeddingService.fromVectorString(embeddingMap.get(note.id))
          : null
      }))
      .filter(note => note.embedding && Array.isArray(note.embedding))

    const connections: NoteConnection[] = []

    // Load user feedback to adjust thresholds per note
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
      select: { note1Id: true, note2Id: true, feedback: true }
    })

    const notePenalty = new Map<string, number>() // positive = higher threshold (penalty), negative = lower (boost)
    for (const fi of feedbackInsights) {
      if (fi.feedback === 'thumbs_down') {
        notePenalty.set(fi.note1Id, (notePenalty.get(fi.note1Id) || 0) + 0.15)
        notePenalty.set(fi.note2Id, (notePenalty.get(fi.note2Id) || 0) + 0.15)
      } else if (fi.feedback === 'thumbs_up') {
        notePenalty.set(fi.note1Id, (notePenalty.get(fi.note1Id) || 0) - 0.05)
        notePenalty.set(fi.note2Id, (notePenalty.get(fi.note2Id) || 0) - 0.05)
      }
    }

    // Compare all pairs of notes
    for (let i = 0; i < notesWithEmbeddings.length; i++) {
      for (let j = i + 1; j < notesWithEmbeddings.length; j++) {
        const note1 = notesWithEmbeddings[i]
        const note2 = notesWithEmbeddings[j]

        const baseThreshold = this.pairSimilarityThreshold(note1, note2, demoMode)
        const adjustedThreshold = baseThreshold + (notePenalty.get(note1.id) || 0) + (notePenalty.get(note2.id) || 0)

        // Calculate time difference
        const daysApart = Math.abs(
          Math.floor((note1.createdAt.getTime() - note2.createdAt.getTime()) / (1000 * 60 * 60 * 24))
        )

        // Time diversity filter: notes must be from different time periods (sauf clips récents)
        if (!this.passesTimeDiversityFilter(daysApart, note1, note2, demoMode)) {
          continue
        }

        // Calculate cosine similarity — whole note level
        const similarity = cosineSimilarity(note1.embedding!, note2.embedding!)

        // Also check chunk-level similarity for more precise connections
        // This catches cases where two notes share a similar SECTION even if overall different
        let bestChunkSimilarity = similarity
        let chunkSnippet: string | undefined
        if (similarity < adjustedThreshold) {
          // Only check chunks if whole-note similarity didn't pass — saves DB queries
          try {
            const chunkRows: Array<{ content: string; embedding: string }> = await prisma.$queryRawUnsafe(
              `SELECT a.content AS content,
                      1 - (a."embedding"::vector <=> b."embedding"::vector) AS chunk_sim
               FROM "NoteEmbeddingChunk" a
               CROSS JOIN LATERAL (
                 SELECT "embedding"
                 FROM "NoteEmbeddingChunk"
                 WHERE "noteId" = $2 AND "embedding" IS NOT NULL
                 ORDER BY "embedding"::vector <=> a."embedding"::vector ASC
                 LIMIT 1
               ) b
               WHERE a."noteId" = $1 AND a."embedding" IS NOT NULL
               ORDER BY chunk_sim DESC
               LIMIT 1`,
              note1.id, note2.id
            )
            if (chunkRows.length > 0 && chunkRows[0]) {
              const row = chunkRows[0] as any
              if (row.chunk_sim && row.chunk_sim > bestChunkSimilarity) {
                bestChunkSimilarity = row.chunk_sim
                chunkSnippet = row.content?.slice(0, 200)
              }
            }
          } catch {}
        }

        if (bestChunkSimilarity >= adjustedThreshold) {
          connections.push({
            note1: {
              id: note1.id,
              title: note1.title,
              content: this.connectionPlainText(note1.title, note1.content),
              createdAt: note1.createdAt
            },
            note2: {
              id: note2.id,
              title: note2.title,
              content: this.connectionPlainText(note2.title, note2.content || ''),
              createdAt: note2.createdAt
            },
            similarityScore: bestChunkSimilarity,
            insight: '', // Will be generated by AI
            daysApart,
            ...(chunkSnippet ? { contextSnippet: chunkSnippet } : {})
          })
        }
      }
    }

    // Sort by similarity score (descending)
    connections.sort((a, b) => b.similarityScore - a.similarityScore)

    // Return top connections
    return connections.slice(0, 10)
  }

  /**
   * Generate AI explanation for the connection
   */
  async generateInsight(
    note1Title: string | null,
    note1Content: string,
    note2Title: string | null,
    note2Content: string,
    language: SupportedLanguage = 'en',
  ): Promise<string> {
    try {
      const config = await getSystemConfig()
      const provider = getChatProvider(config)

      const excerpt1 = excerptPlainNoteContent(note1Title, note1Content, 1200)
      const excerpt2 = excerptPlainNoteContent(note2Title, note2Content, 1200)
      const prompt = buildMemoryEchoInsightPrompt(language, note1Title, excerpt1, note2Title, excerpt2)

      const response = await provider.generateText(prompt)

      const insight = response
        .replace(/^["'«»]|["'«»]$/g, '')
        .replace(/^[^.]+\.\s*/, '')
        .trim()
        .substring(0, 150)

      return insight || getMemoryEchoInsightFallback(language)

    } catch (error) {
      console.error('[MemoryEcho] Failed to generate insight:', error)
      const sample = excerptPlainNoteContent(note1Title, note1Content, 200)
        + excerptPlainNoteContent(note2Title, note2Content, 200)
      const rtl = detectTextDirection(sample) === 'rtl'
      if (rtl) return getMemoryEchoInsightFallback(language === 'fa' || language === 'ar' ? language : 'fa')
      return getMemoryEchoInsightFallback(language)
    }
  }

  /**
   * Get next pending insight for user (based on frequency limit)
   */
  async getNextInsight(userId: string): Promise<MemoryEchoInsight | null> {
    // Check if Memory Echo is enabled for user
    const settings = await prisma.userAISettings.findUnique({
      where: { userId }
    })

    if (!settings || !settings.memoryEcho) {
      return null // Memory Echo disabled
    }

    const demoMode = settings.demoMode || false

    // Skip frequency checks in demo mode
    if (!demoMode) {
      // Check frequency limit
      const today = new Date()
      today.setHours(0, 0, 0, 0)

      const insightsShownToday = await prisma.memoryEchoInsight.count({
        where: {
          userId,
          insightDate: {
            gte: today
          }
        }
      })

      // Frequency limits
      const maxPerDay = settings.memoryEchoFrequency === 'daily' ? 1 :
                        settings.memoryEchoFrequency === 'weekly' ? 0 : // 1 per 7 days (handled below)
                        3 // custom = 3 per day

      if (settings.memoryEchoFrequency === 'weekly') {
        // Check if shown in last 7 days
        const weekAgo = new Date(today)
        weekAgo.setDate(weekAgo.getDate() - 7)

        const recentInsight = await prisma.memoryEchoInsight.findFirst({
          where: {
            userId,
            insightDate: {
              gte: weekAgo
            }
          }
        })

        if (recentInsight) {
          return null // Already shown this week
        }
      } else if (insightsShownToday >= maxPerDay) {
        return null // Daily limit reached
      }

      // Check total insights limit (prevent spam)
      const totalInsights = await prisma.memoryEchoInsight.count({
        where: { userId }
      })

      if (totalInsights >= this.MAX_INSIGHTS_PER_USER) {
        return null // User has too many insights
      }
    }

    // Find new connections (pass demoMode)
    const connections = await this.findConnections(userId, demoMode)

    if (connections.length === 0) {
      return null // No connections found
    }

    // Filter out already shown connections
    const existingInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId },
      select: { note1Id: true, note2Id: true }
    })

    const shownPairs = new Set(
      existingInsights.map(i => `${i.note1Id}-${i.note2Id}`)
    )

    const newConnection = connections.find(c =>
      !shownPairs.has(`${c.note1.id}-${c.note2.id}`) &&
      !shownPairs.has(`${c.note2.id}-${c.note1.id}`)
    )

    if (!newConnection) {
      return null // All connections already shown
    }

    // Generate AI insight in user's language
    const userLanguage = await detectUserLanguage()
    const insightText = await this.generateInsight(
      newConnection.note1.title,
      newConnection.note1.content,
      newConnection.note2.title,
      newConnection.note2.content || '',
      userLanguage,
    )

    // Store insight in database
    // In demo mode, add milliseconds offset to avoid @@unique([userId, insightDate]) collision
    const insightDateValue = demoMode
      ? new Date(Date.now() + Math.floor(Math.random() * 1000))
      : new Date()

    const insight = await prisma.memoryEchoInsight.create({
      data: {
        userId,
        note1Id: newConnection.note1.id,
        note2Id: newConnection.note2.id,
        similarityScore: newConnection.similarityScore,
        insight: insightText,
        insightDate: insightDateValue,
        viewed: false
      },
      include: {
        note1: {
          select: {
            id: true,
            title: true,
            content: true
          }
        },
        note2: {
          select: {
            id: true,
            title: true,
            content: true
          }
        }
      }
    })

    return insight
  }

  /**
   * Mark insight as viewed
   */
  async markAsViewed(insightId: string): Promise<void> {
    await prisma.memoryEchoInsight.update({
      where: { id: insightId },
      data: { viewed: true }
    })
  }

  /**
   * Submit feedback for insight
   */
  async submitFeedback(insightId: string, feedback: 'thumbs_up' | 'thumbs_down'): Promise<void> {
    await prisma.memoryEchoInsight.update({
      where: { id: insightId },
      data: { feedback }
    })

    // Optional: Store in AiFeedback for analytics
    const insight = await prisma.memoryEchoInsight.findUnique({
      where: { id: insightId },
      select: { userId: true, note1Id: true }
    })

    if (insight) {
      await prisma.aiFeedback.create({
        data: {
          noteId: insight.note1Id,
          userId: insight.userId,
          feedbackType: feedback,
          feature: 'memory_echo',
          originalContent: JSON.stringify({ insightId }),
          metadata: {
            timestamp: new Date().toISOString()
          } as any
        }
      })
    }
  }

  /**
   * Get all connections for a specific note
   */
  async getConnectionsForNote(noteId: string, userId: string): Promise<NoteConnection[]> {
    // Ensure all notes have embeddings before searching
    await this.ensureEmbeddings(userId)

    // Get the note with embedding
    const targetNote = await prisma.note.findUnique({
      where: { id: noteId },
      select: {
        id: true,
        title: true,
        content: true,
        sourceUrl: true,
        createdAt: true,
        userId: true
      }
    })

    if (!targetNote || targetNote.userId !== userId) {
      return [] // Note not found or doesn't belong to user
    }

    // Fetch embedding separately
    const embeddingResult: Array<{ embedding: any }> = await prisma.$queryRawUnsafe(
      `SELECT "embedding"::text FROM "NoteEmbedding" WHERE "noteId" = $1`,
      noteId
    )
    const targetEmbeddingStr = embeddingResult[0]?.embedding

    let targetEmbedding = targetEmbeddingStr
      ? embeddingService.fromVectorString(targetEmbeddingStr)
      : null

    if (!targetEmbedding && targetNote.content?.trim()) {
      targetEmbedding = await this.upsertNoteEmbeddingFromNote(targetNote)
    }

    if (!targetEmbedding) {
      return []
    }

    // Get dismissed connections for this note (to filter them out)
    const dismissedInsights = await prisma.memoryEchoInsight.findMany({
      where: {
        userId,
        dismissed: true,
        OR: [
          { note1Id: noteId },
          { note2Id: noteId }
        ]
      },
      select: {
        note1Id: true,
        note2Id: true
      }
    })

    // Create a set of dismissed note pairs for quick lookup
    const dismissedPairs = new Set(
      dismissedInsights.map(i =>
        `${i.note1Id}-${i.note2Id}`
      )
    )

    // Get all other user's notes with embeddings
    const otherNotes = await prisma.note.findMany({
      where: {
        userId,
        id: { not: noteId },
        isArchived: false,
        trashedAt: null,
      },
      select: {
        id: true,
        title: true,
        content: true,
        sourceUrl: true,
        createdAt: true
      },
      orderBy: { createdAt: 'desc' }
    })

    if (otherNotes.length === 0) {
      return []
    }

    // Fetch all other embeddings
    const otherNoteIds = otherNotes.map(n => n.id)
    const otherEmbeddings = otherNoteIds.length === 0 ? [] : await prisma.$queryRaw<Array<{ noteId: string, embedding: any }>>(
      Prisma.sql`SELECT "noteId", "embedding"::text FROM "NoteEmbedding" WHERE "noteId" IN (${Prisma.join(otherNoteIds)})`
    )
    const otherEmbeddingMap = new Map(otherEmbeddings.map(e => [e.noteId, e.embedding]))

    // Check if user has demo mode enabled
    const settings = await prisma.userAISettings.findUnique({
      where: { userId }
    })
    const demoMode = settings?.demoMode || false

    // Load user feedback to adjust thresholds
    const feedbackInsights = await prisma.memoryEchoInsight.findMany({
      where: { userId, feedback: { not: null } },
      select: { note1Id: true, note2Id: true, feedback: true }
    })
    const notePenalty = new Map<string, number>()
    for (const fi of feedbackInsights) {
      if (fi.feedback === 'thumbs_down') {
        notePenalty.set(fi.note1Id, (notePenalty.get(fi.note1Id) || 0) + 0.15)
        notePenalty.set(fi.note2Id, (notePenalty.get(fi.note2Id) || 0) + 0.15)
      } else if (fi.feedback === 'thumbs_up') {
        notePenalty.set(fi.note1Id, (notePenalty.get(fi.note1Id) || 0) - 0.05)
        notePenalty.set(fi.note2Id, (notePenalty.get(fi.note2Id) || 0) - 0.05)
      }
    }

    const connections: NoteConnection[] = []

    // Compare target note with all other notes
    for (const otherNote of otherNotes) {
      const otherEmbeddingStr = otherEmbeddingMap.get(otherNote.id)
      let otherEmbedding = otherEmbeddingStr
        ? embeddingService.fromVectorString(otherEmbeddingStr)
        : null

      if (!otherEmbedding && otherNote.content?.trim()) {
        otherEmbedding = await this.upsertNoteEmbeddingFromNote(otherNote)
      }
      if (!otherEmbedding) continue

      // Check if this connection was dismissed
      const pairKey1 = `${targetNote.id}-${otherNote.id}`
      const pairKey2 = `${otherNote.id}-${targetNote.id}`
      if (dismissedPairs.has(pairKey1) || dismissedPairs.has(pairKey2)) {
        continue
      }

      // Calculate time difference
      const daysApart = Math.abs(
        Math.floor((targetNote.createdAt.getTime() - otherNote.createdAt.getTime()) / (1000 * 60 * 60 * 24))
      )

      // Time diversity filter (clips récents autorisés sans délai de 7 jours)
      if (!this.passesTimeDiversityFilter(daysApart, targetNote, otherNote, demoMode)) {
        continue
      }

      // Calculate cosine similarity
      const similarity = cosineSimilarity(targetEmbedding, otherEmbedding)

      // Similarity threshold (adjusted by feedback)
      const baseThreshold = this.pairSimilarityThreshold(targetNote, otherNote, demoMode)
      const adjustedThreshold = baseThreshold
        + (notePenalty.get(targetNote.id) || 0)
        + (notePenalty.get(otherNote.id) || 0)
      if (similarity >= adjustedThreshold) {
        connections.push({
          note1: {
            id: targetNote.id,
            title: targetNote.title,
            content: this.connectionPlainText(targetNote.title, targetNote.content),
            createdAt: targetNote.createdAt
          },
          note2: {
            id: otherNote.id,
            title: otherNote.title,
            content: this.connectionPlainText(otherNote.title, otherNote.content || ''),
            createdAt: otherNote.createdAt
          },
          similarityScore: similarity,
          insight: '', // Will be generated on demand
          daysApart
        })
      }
    }

    // Sort by similarity score (descending)
    connections.sort((a, b) => b.similarityScore - a.similarityScore)

    return connections
  }

  /**
   * Get insights history for user
   */
  async getInsightsHistory(userId: string): Promise<MemoryEchoInsight[]> {
    const insights = await prisma.memoryEchoInsight.findMany({
      where: { userId },
      include: {
        note1: {
          select: {
            id: true,
            title: true,
            content: true
          }
        },
        note2: {
          select: {
            id: true,
            title: true,
            content: true
          }
        }
      },
      orderBy: { insightDate: 'desc' },
      take: 20
    })

    return insights
  }
}

// Export singleton instance
export const memoryEchoService = new MemoryEchoService()