From 03e6a62b8064196fcb40adfcc304f80d0d13ae28 Mon Sep 17 00:00:00 2001 From: Antigravity Date: Tue, 12 May 2026 07:03:56 +0000 Subject: [PATCH] feat: migrate semantic search to pgvector + full-text search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace JSON-string embeddings with native pgvector(1536) storage and add PostgreSQL full-text search (tsvector/GIN) with Reciprocal Rank Fusion for hybrid keyword + semantic ranking. Changes: - NoteEmbedding.embedding: String → vector(1536) via pgvector - NoteEmbedding: added updatedAt for reindex tracking - Note: added tsv (tsvector) with auto-update trigger for FTS - semantic-search.service: hybrid FTS + vector search with RRF fusion - embedding.service: toVectorString() for pgvector SQL literals - Removed JS-side cosine similarity loops (now DB-side via <=>) - Added HNSW index on NoteEmbedding.embedding (cosine distance) - Added GIN index on Note.tsv for FTS queries Schema migration in: prisma/migrations/20260512120000_pgvector_and_fts_search/ Co-Authored-By: Claude Opus 4.7 --- MIGRATION.md | 133 +++++ architectural-grid10/.env.example | 9 + architectural-grid10/.gitignore | 8 + architectural-grid10/README.md | 20 + architectural-grid10/index.html | 13 + architectural-grid10/metadata.json | 6 + architectural-grid10/package.json | 34 ++ architectural-grid10/src/App.tsx | 450 ++++++++++++++ .../src/components/AISidebar.tsx | 379 ++++++++++++ .../src/components/AgentsView.tsx | 325 +++++++++++ .../components/HierarchicalCarnetSelector.tsx | 208 +++++++ .../src/components/NotebooksView.tsx | 469 +++++++++++++++ .../src/components/SettingsView.tsx | 66 +++ .../src/components/Sidebar.tsx | 450 ++++++++++++++ .../src/components/SlashMenu.tsx | 65 +++ .../src/components/TrashView.tsx | 218 +++++++ .../src/components/settings/AITab.tsx | 152 +++++ .../src/components/settings/AppearanceTab.tsx | 85 +++ .../src/components/settings/GeneralTab.tsx | 82 +++ .../components/settings/SettingsHeader.tsx | 51 ++ architectural-grid10/src/constants.ts | 62 ++ architectural-grid10/src/index.css | 98 ++++ architectural-grid10/src/main.tsx | 10 + architectural-grid10/src/types.ts | 34 ++ architectural-grid10/tsconfig.json | 26 + architectural-grid10/vite.config.ts | 24 + docker-compose.yml | 2 +- mcp-server/tools.js | 32 +- memento-note/app/actions/notes.ts | 163 ++---- memento-note/app/actions/semantic-search.ts | 2 +- .../api/admin/embeddings/validate/route.ts | 84 +-- memento-note/app/api/notes/cleanup/route.ts | 20 +- memento-note/app/api/notes/reindex/route.ts | 38 +- memento-note/components/home-client.tsx | 3 +- .../lib/ai/services/embedding.service.ts | 189 ++---- .../lib/ai/services/memory-echo.service.ts | 29 +- .../ai/services/semantic-search.service.ts | 548 ++++++++---------- memento-note/lib/ai/tools/note-search.tool.ts | 40 +- .../migration.sql | 52 ++ memento-note/prisma/schema.prisma | 4 +- memento-note/scripts/migrate-embeddings.ts | 56 +- memento-note/scripts/test-backend-logic.ts | 35 +- .../tests/migration/integrity.test.ts | 36 +- 43 files changed, 4024 insertions(+), 786 deletions(-) create mode 100644 MIGRATION.md create mode 100644 architectural-grid10/.env.example create mode 100644 architectural-grid10/.gitignore create mode 100644 architectural-grid10/README.md create mode 100644 architectural-grid10/index.html create mode 100644 architectural-grid10/metadata.json create mode 100644 architectural-grid10/package.json create mode 100644 architectural-grid10/src/App.tsx create mode 100644 architectural-grid10/src/components/AISidebar.tsx create mode 100644 architectural-grid10/src/components/AgentsView.tsx create mode 100644 architectural-grid10/src/components/HierarchicalCarnetSelector.tsx create mode 100644 architectural-grid10/src/components/NotebooksView.tsx create mode 100644 architectural-grid10/src/components/SettingsView.tsx create mode 100644 architectural-grid10/src/components/Sidebar.tsx create mode 100644 architectural-grid10/src/components/SlashMenu.tsx create mode 100644 architectural-grid10/src/components/TrashView.tsx create mode 100644 architectural-grid10/src/components/settings/AITab.tsx create mode 100644 architectural-grid10/src/components/settings/AppearanceTab.tsx create mode 100644 architectural-grid10/src/components/settings/GeneralTab.tsx create mode 100644 architectural-grid10/src/components/settings/SettingsHeader.tsx create mode 100644 architectural-grid10/src/constants.ts create mode 100644 architectural-grid10/src/index.css create mode 100644 architectural-grid10/src/main.tsx create mode 100644 architectural-grid10/src/types.ts create mode 100644 architectural-grid10/tsconfig.json create mode 100644 architectural-grid10/vite.config.ts create mode 100644 memento-note/prisma/migrations/20260512120000_pgvector_and_fts_search/migration.sql diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..2b70baa --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,133 @@ +# Semantic Search Migration to pgvector + Full-Text Search + +## Overview + +This migration migrates the semantic search infrastructure from JSON-string embeddings to **native pgvector** storage and adds **PostgreSQL full-text search (FTS)** with Reciprocal Rank Fusion (RRF) for hybrid ranking. + +## What Changed + +### Schema Changes + +**`NoteEmbedding` table:** +- Changed `embedding String` → `embedding Unsupported("vector(1536))` — stores as native pgvector +- Added `updatedAt` column for tracking reindex freshness + +**`Note` table:** +- Added `tsv Unsupported("tsvector")` — auto-updated via trigger for FTS + +### Search Architecture + +| Before | After | +|--------|-------| +| JS-side cosine similarity loops | DB-side `<=>` (cosine distance) via pgvector | +| Embeddings stored as JSON strings | Native `vector(1536)` pgvector type | +| Pure vector-only search | Hybrid FTS + vector with RRF fusion | +| No full-text capability | `tsvector` + GIN index for keyword matching | + +### New Indexes + +- `NoteEmbedding_embedding_hnsw_idx` — HNSW index on `embedding` column (cosine distance) +- `Note_tsv_gin_idx` — GIN index on `tsv` column for FTS + +## Deployment Steps + +### 1. Enable pgvector Extension + +pgvector must be enabled before the schema migration runs: + +```sql +CREATE EXTENSION IF NOT EXISTS vector; +``` + +If deploying via the migration file, this runs automatically as Phase 1. + +### 2. Run Database Migration + +The migration file (`prisma/migrations/20260512120000_pgvector_and_fts_search/migration.sql`) applies in three phases: + +**Phase 1:** Enable pgvector extension +```sql +CREATE EXTENSION IF NOT EXISTS vector; +``` + +**Phase 2:** Convert NoteEmbedding to native vector +```sql +ALTER TABLE "NoteEmbedding" ADD COLUMN "vec" vector(1536); +UPDATE "NoteEmbedding" SET "vec" = ("embedding"::jsonb)::text::vector(1536) + WHERE "embedding" IS NOT NULL; +ALTER TABLE "NoteEmbedding" DROP COLUMN "embedding"; +ALTER TABLE "NoteEmbedding" RENAME COLUMN "vec" TO "embedding"; +ALTER TABLE "NoteEmbedding" ADD COLUMN "updatedAt" TIMESTAMP NOT NULL DEFAULT now(); +CREATE INDEX "NoteEmbedding_embedding_hnsw_idx" ON "NoteEmbedding" + USING hnsw ("embedding" vector_cosine_ops) WITH (m = 16, ef_construction = 64); +``` + +**Phase 3:** Add FTS tsvector to Note +```sql +ALTER TABLE "Note" ADD COLUMN "tsv" tsvector; +UPDATE "Note" SET "tsv" = + setweight(to_tsvector('simple', COALESCE("title", '')), 'A') || + setweight(to_tsvector('simple', COALESCE("content", '')), 'B'); +CREATE INDEX "Note_tsv_gin_idx" ON "Note" USING gin ("tsv"); +CREATE OR REPLACE FUNCTION "note_tsv_trigger"() RETURNS trigger AS $$ +BEGIN + NEW."tsv" := + setweight(to_tsvector('simple', COALESCE(NEW."title", '')), 'A') || + setweight(to_tsvector('simple', COALESCE(NEW."content", '')), 'B'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +CREATE TRIGGER "note_tsv_update" + BEFORE INSERT OR UPDATE OF "title", "content" ON "Note" + FOR EACH ROW EXECUTE FUNCTION "note_tsv_trigger"(); +``` + +### 3. Regenerate Embeddings for Existing Notes + +After the migration, all existing `NoteEmbedding` rows must have their `embedding` column regenerated from the old JSON strings to native vector format. The migration handles this conversion automatically via the `UPDATE` statement. + +To reindex all notes programmatically: +``` +POST /api/notes/reindex +``` + +### 4. Verify Deployment + +**Validate embeddings:** +``` +POST /api/admin/embeddings/validate +``` + +**Test semantic search** via the note search tool or: +``` +POST /api/notes/search?q= +``` + +## Docker Deployment + +The `docker-compose.yml` runs PostgreSQL 16-alpine with the following configuration: + +- **PostgreSQL port:** 5433 (host) → 5432 (container) +- **Database:** `memento` (user: `memento`, password: `memento` by default) +- **Health check:** `pg_isready` + +Services that depend on the database (`memento-note`, `mcp-server`) wait for PostgreSQL to be healthy before starting. + +## Rollback + +To rollback to the pre-migration state: + +1. Drop the HNSW index: `DROP INDEX "NoteEmbedding_embedding_hnsw_idx";` +2. Drop the GIN index: `DROP INDEX "Note_tsv_gin_idx";` +3. Drop the trigger: `DROP TRIGGER "note_tsv_update" ON "Note";` +4. Drop the function: `DROP FUNCTION "note_tsv_trigger"();` +5. Revert schema via Prisma migrate reset (requires restoring the old `NoteEmbedding.embedding` column type) + +## Affected Services + +| Service | Container | Port | +|---------|-----------|------| +| PostgreSQL | `memento-postgres` | 5433 | +| memento-note (Next.js) | `memento-web` | 3000 | +| mcp-server | `memento-mcp` | 3001 | +| Ollama (optional) | `memento-ollama` | 11434 | diff --git a/architectural-grid10/.env.example b/architectural-grid10/.env.example new file mode 100644 index 0000000..7a550fe --- /dev/null +++ b/architectural-grid10/.env.example @@ -0,0 +1,9 @@ +# GEMINI_API_KEY: Required for Gemini AI API calls. +# AI Studio automatically injects this at runtime from user secrets. +# Users configure this via the Secrets panel in the AI Studio UI. +GEMINI_API_KEY="MY_GEMINI_API_KEY" + +# APP_URL: The URL where this applet is hosted. +# AI Studio automatically injects this at runtime with the Cloud Run service URL. +# Used for self-referential links, OAuth callbacks, and API endpoints. +APP_URL="MY_APP_URL" diff --git a/architectural-grid10/.gitignore b/architectural-grid10/.gitignore new file mode 100644 index 0000000..5a86d2a --- /dev/null +++ b/architectural-grid10/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +build/ +dist/ +coverage/ +.DS_Store +*.log +.env* +!.env.example diff --git a/architectural-grid10/README.md b/architectural-grid10/README.md new file mode 100644 index 0000000..0078184 --- /dev/null +++ b/architectural-grid10/README.md @@ -0,0 +1,20 @@ +
+GHBanner +
+ +# Run and deploy your AI Studio app + +This contains everything you need to run your app locally. + +View your app in AI Studio: https://ai.studio/apps/b7b577c6-4d9f-44ac-8fe1-85bc3c6d6e66 + +## Run Locally + +**Prerequisites:** Node.js + + +1. Install dependencies: + `npm install` +2. Set the `GEMINI_API_KEY` in [.env.local](.env.local) to your Gemini API key +3. Run the app: + `npm run dev` diff --git a/architectural-grid10/index.html b/architectural-grid10/index.html new file mode 100644 index 0000000..21dfe69 --- /dev/null +++ b/architectural-grid10/index.html @@ -0,0 +1,13 @@ + + + + + + My Google AI Studio App + + +
+ + + + diff --git a/architectural-grid10/metadata.json b/architectural-grid10/metadata.json new file mode 100644 index 0000000..3e19746 --- /dev/null +++ b/architectural-grid10/metadata.json @@ -0,0 +1,6 @@ +{ + "name": "Architectural Grid", + "description": "A minimalist notebook for architectural research and conceptual sketches.", + "requestFramePermissions": [], + "majorCapabilities": [] +} diff --git a/architectural-grid10/package.json b/architectural-grid10/package.json new file mode 100644 index 0000000..6b3f167 --- /dev/null +++ b/architectural-grid10/package.json @@ -0,0 +1,34 @@ +{ + "name": "react-example", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite --port=3000 --host=0.0.0.0", + "build": "vite build", + "preview": "vite preview", + "clean": "rm -rf dist", + "lint": "tsc --noEmit" + }, + "dependencies": { + "@google/genai": "^1.29.0", + "@tailwindcss/vite": "^4.1.14", + "@vitejs/plugin-react": "^5.0.4", + "lucide-react": "^0.546.0", + "react": "^19.0.1", + "react-dom": "^19.0.1", + "vite": "^6.2.3", + "express": "^4.21.2", + "dotenv": "^17.2.3", + "motion": "^12.23.24" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "autoprefixer": "^10.4.21", + "tailwindcss": "^4.1.14", + "tsx": "^4.21.0", + "typescript": "~5.8.2", + "vite": "^6.2.3", + "@types/express": "^4.17.21" + } +} diff --git a/architectural-grid10/src/App.tsx b/architectural-grid10/src/App.tsx new file mode 100644 index 0000000..9181686 --- /dev/null +++ b/architectural-grid10/src/App.tsx @@ -0,0 +1,450 @@ +/** + * @license + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState, useMemo } from 'react'; +import { motion, AnimatePresence } from 'motion/react'; + +// Components +import { Sidebar } from './components/Sidebar'; +import { NotebooksView } from './components/NotebooksView'; +import { AgentsView } from './components/AgentsView'; +import { SettingsView } from './components/SettingsView'; +import { TrashView } from './components/TrashView'; +import { AISidebar } from './components/AISidebar'; +import { SlashMenu } from './components/SlashMenu'; + +// Data & Types +import { CARNETS, ALL_NOTES } from './constants'; +import { NavigationView, SettingsTab, AITab, AITone, Carnet, Note } from './types'; + +export default function App() { + const [activeView, setActiveView] = useState('notebooks'); + const [activeSettingsTab, setActiveSettingsTab] = useState('general'); + const [selectedAgentId, setSelectedAgentId] = useState(null); + const [isDarkMode, setIsDarkMode] = useState(false); + const [carnets, setCarnets] = useState(CARNETS); + const [notes, setNotes] = useState(ALL_NOTES); + const [activeCarnetId, setActiveCarnetId] = useState('4'); + const [activeNoteId, setActiveNoteId] = useState(null); + const [selectedTagIds, setSelectedTagIds] = useState([]); + const [isAISidebarOpen, setIsAISidebarOpen] = useState(false); + const [aiTab, setAiTab] = useState('discussion'); + const [selectedTone, setSelectedTone] = useState('Professional'); + + // Modal States + const [showNewCarnetModal, setShowNewCarnetModal] = useState<{ isOpen: boolean; parentId?: string; isRenaming?: boolean; carnetId?: string }>({ isOpen: false }); + const [showNewNoteModal, setShowNewNoteModal] = useState(false); + const [slashMenu, setSlashMenu] = useState<{ isOpen: boolean; top: number; left: number } | null>(null); + + // Form States + const [newCarnetName, setNewCarnetName] = useState(''); + const [newNoteTitle, setNewNoteTitle] = useState(''); + const [newNoteContent, setNewNoteContent] = useState(''); + + const handleEditorKeyDown = (e: React.KeyboardEvent) => { + if (e.key === '/') { + const selection = window.getSelection(); + if (selection && selection.rangeCount > 0) { + const range = selection.getRangeAt(0); + const rect = range.getBoundingClientRect(); + setSlashMenu({ + isOpen: true, + top: rect.bottom + window.scrollY, + left: rect.left + window.scrollX + }); + } + } + }; + + const togglePin = (noteId: string) => { + setNotes(notes.map(n => n.id === noteId ? { ...n, isPinned: !n.isPinned } : n)); + }; + + const filteredNotes = useMemo(() => { + let result = notes.filter(n => n.carnetId === activeCarnetId && !n.isDeleted); + + if (selectedTagIds.length > 0) { + result = result.filter(note => + selectedTagIds.every(tagId => note.tags?.some(tag => tag.id === tagId)) + ); + } + + return [...result].sort((a, b) => { + if (a.isPinned && !b.isPinned) return -1; + if (!a.isPinned && b.isPinned) return 1; + return 0; + }); + }, [activeCarnetId, notes]); + + const activeNote = useMemo(() => + notes.find(n => n.id === activeNoteId), + [activeNoteId, notes]); + + const activeCarnet = useMemo(() => + carnets.find(c => c.id === activeCarnetId), + [activeCarnetId, carnets]); + + const handleAddCarnet = (e: React.FormEvent) => { + e.preventDefault(); + if (!newCarnetName.trim()) return; + + if (showNewCarnetModal.isRenaming && showNewCarnetModal.carnetId) { + setCarnets(carnets.map(c => c.id === showNewCarnetModal.carnetId ? { ...c, name: newCarnetName, initial: newCarnetName.charAt(0).toUpperCase() } : c)); + setShowNewCarnetModal({ isOpen: false }); + setNewCarnetName(''); + return; + } + + const newCarnet: Carnet = { + id: Date.now().toString(), + name: newCarnetName, + initial: newCarnetName.charAt(0).toUpperCase(), + type: 'Project', + parentId: showNewCarnetModal.parentId + }; + + setCarnets([...carnets, newCarnet]); + setNewCarnetName(''); + setShowNewCarnetModal({ isOpen: false }); + setActiveCarnetId(newCarnet.id); + }; + + const handleDeleteCarnet = (id: string) => { + if (window.confirm('Déplacer ce carnet et ses sous-carnets vers la corbeille ?')) { + const idsToDelete = new Set([id]); + + const addChildren = (parentId: string) => { + carnets.forEach(c => { + if (c.parentId === parentId) { + idsToDelete.add(c.id); + addChildren(c.id); + } + }); + }; + addChildren(id); + + const deletedAt = new Date().toISOString(); + setCarnets(carnets.map(c => idsToDelete.has(c.id) ? { ...c, isDeleted: true, deletedAt } : c)); + setNotes(notes.map(n => idsToDelete.has(n.carnetId) ? { ...n, isDeleted: true, deletedAt } : n)); + + if (idsToDelete.has(activeCarnetId)) { + setActiveCarnetId('1'); + } + } + }; + + const handleDeleteNote = (id: string) => { + const deletedAt = new Date().toISOString(); + setNotes(notes.map(n => n.id === id ? { ...n, isDeleted: true, deletedAt } : n)); + if (activeNoteId === id) setActiveNoteId(null); + }; + + const handleRestoreCarnet = (id: string) => { + setCarnets(carnets.map(c => c.id === id ? { ...c, isDeleted: false, deletedAt: undefined } : c)); + // Optionally restore linked notes too? User might expect that. + setNotes(notes.map(n => n.carnetId === id ? { ...n, isDeleted: false, deletedAt: undefined } : n)); + }; + + const handleRestoreNote = (id: string) => { + setNotes(notes.map(n => n.id === id ? { ...n, isDeleted: false, deletedAt: undefined } : n)); + }; + + const handlePermanentDeleteNote = (id: string) => { + setNotes(notes.filter(n => n.id !== id)); + }; + + const handlePermanentDeleteCarnet = (id: string) => { + const idsToDelete = new Set([id]); + const addChildren = (parentId: string) => { + carnets.forEach(c => { + if (c.parentId === parentId) { + idsToDelete.add(c.id); + addChildren(c.id); + } + }); + }; + addChildren(id); + setCarnets(carnets.filter(c => !idsToDelete.has(c.id))); + setNotes(notes.filter(n => !idsToDelete.has(n.carnetId))); + }; + + const handleAddNote = (e: React.FormEvent) => { + e.preventDefault(); + if (!newNoteTitle.trim() || !newNoteContent.trim()) return; + + const newNote: Note = { + id: `n-${Date.now()}`, + carnetId: activeCarnetId, + title: newNoteTitle, + date: new Intl.DateTimeFormat('en-US', { month: 'short', day: 'numeric', year: 'numeric' }).format(new Date()), + content: newNoteContent, + imageUrl: 'https://images.unsplash.com/photo-1487958449943-2429e8be8625?auto=format&fit=crop&q=80&w=800&h=600', + tags: [] + }; + + setNotes([newNote, ...notes]); + setNewNoteTitle(''); + setNewNoteContent(''); + setShowNewNoteModal(false); + setActiveNoteId(newNote.id); + }; + + return ( +
+ { + setShowNewCarnetModal({ isOpen: show, parentId, isRenaming, carnetId }); + if (isRenaming && carnetId) { + const carnet = carnets.find(c => c.id === carnetId); + if (carnet) setNewCarnetName(carnet.name); + } else { + setNewCarnetName(''); + } + }} + onDeleteCarnet={handleDeleteCarnet} + /> + +
+ + {(activeView === 'notebooks' || activeView === 'shared' || activeView === 'reminders') && ( + + setShowNewCarnetModal({ isOpen: show, parentId, isRenaming, carnetId })} + onDeleteNote={handleDeleteNote} + /> + + )} + + {activeView === 'trash' && ( + + n.isDeleted)} + deletedCarnets={carnets.filter(c => c.isDeleted)} + onRestoreNote={handleRestoreNote} + onRestoreCarnet={handleRestoreCarnet} + onPermanentDeleteNote={handlePermanentDeleteNote} + onPermanentDeleteCarnet={handlePermanentDeleteCarnet} + onEmptyTrash={() => { + setNotes(notes.filter(n => !n.isDeleted)); + setCarnets(carnets.filter(c => !c.isDeleted)); + }} + /> + + )} + + {activeView === 'agents' && ( + + + + )} + + {activeView === 'settings' && ( + + + + )} + + + +
+ + {/* Modals */} + + {showNewCarnetModal.isOpen && ( +
+ setShowNewCarnetModal({ isOpen: false })} + className="absolute inset-0 bg-ink/40 backdrop-blur-sm" + /> + +

+ {showNewCarnetModal.isRenaming ? 'Rename Carnet' : (showNewCarnetModal.parentId ? 'Create Sub-Carnet' : 'Create New Carnet')} +

+ {showNewCarnetModal.parentId && !showNewCarnetModal.isRenaming && ( +

+ Inside: {carnets.find(c => c.id === showNewCarnetModal.parentId)?.name} +

+ )} +
+
+ + setNewCarnetName(e.target.value)} + placeholder="E.g., Sustainable Patterns" + className="w-full bg-white dark:bg-[#2A2A2A] border border-border rounded-lg px-4 py-3 outline-none focus:border-ink transition-colors font-serif italic text-lg text-ink dark:text-dark-ink" + /> +
+
+ + +
+
+ +
+ )} + + {showNewNoteModal && ( +
+ setShowNewNoteModal(false)} + className="absolute inset-0 bg-ink/40 backdrop-blur-sm" + /> + + + {slashMenu?.isOpen && ( + { console.log(type); setSlashMenu(null); }} + onClose={() => setSlashMenu(null)} + /> + )} + +

Add Architectural Note

+
+
+ + setNewNoteTitle(e.target.value)} + placeholder="Enter the title of your study..." + className="w-full bg-white dark:bg-[#2A2A2A] border border-border rounded-lg px-5 py-4 outline-none focus:border-ink transition-colors font-serif text-2xl text-ink dark:text-dark-ink" + /> +
+
+ +