Momento/memento-note/tests/search/semantic-threshold.spec.ts

import { test, expect } from '@playwright/test';

test.describe('Semantic Search Threshold Tests', () => {
  test.beforeEach(async ({ page }) => {
    // Login before each test
    await page.goto('http://localhost:3000');
    // Wait for potential redirect or login
    await page.waitForLoadState('networkidle');
  });

  test('should use default threshold of 0.65 for semantic matching', async ({ page }) => {
    // This test verifies that the default threshold is applied
    // We'll search for something and check that only relevant results appear

    await page.goto('http://localhost:3000');

    // Get initial note count
    const initialNotes = await page.locator('.note-card').count();

    // Search for something specific
    const searchInput = page.locator('input[placeholder*="Search"]');
    await searchInput.fill('programming code');
    await page.waitForTimeout(500); // Wait for search debounce

    // Get search results
    const searchResults = await page.locator('.note-card').count();

    // With threshold 0.65, we should have fewer false positives
    // This is a basic sanity check - exact assertions depend on your data
    expect(searchResults).toBeLessThanOrEqual(initialNotes);

    // Clear search
    await searchInput.fill('');
    await page.waitForTimeout(500);
  });

  test('should not show unrelated notes when searching for specific terms', async ({ page }) => {
    await page.goto('http://localhost:3000');

    // Search for technical terms
    const searchInput = page.locator('input[placeholder*="Search"]');
    await searchInput.fill('javascript programming');
    await page.waitForTimeout(500);

    // Get all search result titles
    const resultTitles = await page.locator('.note-card .note-title').allTextContents();

    // Check that results are related to the search (basic keyword check)
    // With higher threshold (0.65), semantic matches should be more relevant
    const hasIrrelevantResults = resultTitles.some(title => {
      const lowerTitle = title.toLowerCase();
      // If a result mentions completely unrelated topics like cooking/food
      // when searching for programming, that's a false positive
      return lowerTitle.includes('recipe') ||
             lowerTitle.includes('cooking') ||
             lowerTitle.includes('food') ||
             lowerTitle.includes('shopping');
    });

    // With threshold 0.65, we expect fewer false positives
    // This test may need adjustment based on your actual data
    expect(hasIrrelevantResults).toBe(false);
  });

  test('should still show highly relevant semantic matches', async ({ page }) => {
    await page.goto('http://localhost:3000');

    // Search for a concept and its close synonyms
    const searchInput = page.locator('input[placeholder*="Search"]');
    await searchInput.fill('coding');
    await page.waitForTimeout(500);

    // Get search results
    const searchResults = await page.locator('.note-card');
    const count = await searchResults.count();

    // Even with higher threshold, we should find semantic matches
    // for closely related terms like "programming", "development", etc.
    if (count > 0) {
      const titles = await page.locator('.note-card .note-title').allTextContents();
      const hasRelevantContent = titles.some(title => {
        const lowerTitle = title.toLowerCase();
        return lowerTitle.includes('code') ||
               lowerTitle.includes('program') ||
               lowerTitle.includes('develop') ||
               lowerTitle.includes('software') ||
               lowerTitle.includes('app');
      });

      // Should find semantically related content
      expect(hasRelevantContent).toBe(true);
    }
  });

  test('threshold can be configured via SystemConfig', async ({ page }) => {
    // This is an integration test verifying the configuration system
    // In a real scenario, you'd make an API call to update the config

    // Default threshold should be 0.65
    const response = await page.request.get('/api/admin/embeddings/validate');

    // This endpoint requires admin auth, so we're just checking it exists
    // The actual threshold value is tested in unit tests
    expect(response.ok()).toBeFalsy(); // Should be 401/403 without auth
  });
});

test.describe('Semantic Threshold Unit Tests', () => {
  test('default threshold should be 0.65', () => {
    // This is a compile-time check
    // The actual value is in SEARCH_DEFAULTS.SEMANTIC_THRESHOLD
    const { SEARCH_DEFAULTS } = require('../../lib/config');
    expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBe(0.65);
  });

  test('threshold should be higher than old value', () => {
    const { SEARCH_DEFAULTS } = require('../../lib/config');
    expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBeGreaterThan(0.40);
  });

  test('threshold should be less than 1.0', () => {
    const { SEARCH_DEFAULTS } = require('../../lib/config');
    expect(SEARCH_DEFAULTS.SEMANTIC_THRESHOLD).toBeLessThan(1.0);
  });
});