import { test, expect } from '@playwright/test'; test.describe('Semantic Search Threshold Tests', () => { test.beforeEach(async ({ page }) => { // Login before each test await page.goto('http://localhost:3000'); // Wait for potential redirect or login await page.waitForLoadState('networkidle'); }); test('should use default threshold of 0.65 for semantic matching', async ({ page }) => { // This test verifies that the default threshold is applied // We'll search for something and check that only relevant results appear await page.goto('http://localhost:3000'); // Get initial note count const initialNotes = await page.locator('.note-card').count(); // Search for something specific const searchInput = page.locator('input[placeholder*="Search"]'); await searchInput.fill('programming code'); await page.waitForTimeout(500); // Wait for search debounce // Get search results const searchResults = await page.locator('.note-card').count(); // With threshold 0.65, we should have fewer false positives // This is a basic sanity check - exact assertions depend on your data expect(searchResults).toBeLessThanOrEqual(initialNotes); // Clear search await searchInput.fill(''); await page.waitForTimeout(500); }); test('should not show unrelated notes when searching for specific terms', async ({ page }) => { await page.goto('http://localhost:3000'); // Search for technical terms const searchInput = page.locator('input[placeholder*="Search"]'); await searchInput.fill('javascript programming'); await page.waitForTimeout(500); // Get all search result titles const resultTitles = await page.locator('.note-card .note-title').allTextContents(); // Check that results are related to the search (basic keyword check) // With higher threshold (0.65), semantic matches should be more relevant const hasIrrelevantResults = resultTitles.some(title => { const lowerTitle = title.toLowerCase(); // If a result mentions completely unrelated topics like cooking/food // when searching for programming, that's a false positive return lowerTitle.includes('recipe') || lowerTitle.includes('cooking') || lowerTitle.includes('food') || lowerTitle.includes('shopping'); }); // With threshold 0.65, we expect fewer false positives // This test may need adjustment based on your actual data expect(hasIrrelevantResults).toBe(false); }); test('should still show highly relevant semantic matches', async ({ page }) => { await page.goto('http://localhost:3000'); // Search for a concept and its close synonyms const searchInput = page.locator('input[placeholder*="Search"]'); await searchInput.fill('coding'); await page.waitForTimeout(500); // Get search results const searchResults = await page.locator('.note-card'); const count = await searchResults.count(); // Even with higher threshold, we should find semantic matches // for closely related terms like "programming", "development", etc. if (count > 0) { const titles = await page.locator('.note-card .note-title').allTextContents(); const hasRelevantContent = titles.some(title => { const lowerTitle = title.toLowerCase(); return lowerTitle.includes('code') || lowerTitle.includes('program') || lowerTitle.includes('develop') || lowerTitle.includes('software') || lowerTitle.includes('app'); }); // Should find semantically related content expect(hasRelevantContent).toBe(true); } }); test('threshold can be configured via SystemConfig', async ({ page }) => { // This is an integration test verifying the configuration system // In a real scenario, you'd make an API call to update the config // Default threshold should be 0.65 const response = await page.request.get('/api/admin/embeddings/validate'); // This endpoint requires admin auth, so we're just checking it exists // The actual threshold value is tested in unit tests expect(response.ok()).toBeFalsy(); // Should be 401/403 without auth }); }); test.describe('Semantic Threshold Unit Tests', () => { test('default threshold should be 0.65', () => { // This is a compile-time check // The actual value is in SEARCH_DEFAULTS.SEMANTIC_THRESHOLD const { SEARCH_DEFAULTS } = require('../../lib/config'); expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBe(0.65); }); test('threshold should be higher than old value', () => { const { SEARCH_DEFAULTS } = require('../../lib/config'); expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBeGreaterThan(0.40); }); test('threshold should be less than 1.0', () => { const { SEARCH_DEFAULTS } = require('../../lib/config'); expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBeLessThan(1.0); }); });