fix: improve note interactions and markdown LaTeX support

## Bug Fixes ### Note Card Actions - Fix broken size change functionality (missing state declaration) - Implement React 19 useOptimistic for instant UI feedback - Add startTransition for non-blocking updates - Ensure smooth animations without page refresh - All note actions now work: pin, archive, color, size, checklist ### Markdown LaTeX Rendering - Add remark-math and rehype-katex plugins - Support inline equations with dollar sign syntax - Support block equations with double dollar sign syntax - Import KaTeX CSS for proper styling - Equations now render correctly instead of showing raw LaTeX ## Technical Details - Replace undefined currentNote references with optimistic state - Add optimistic updates before server actions for instant feedback - Use router.refresh() in transitions for smart cache invalidation - Install remark-math, rehype-katex, and katex packages ## Testing - Build passes successfully with no TypeScript errors - Dev server hot-reloads changes correctly
2026-01-09 22:13:49 +01:00
parent 3c4b9d6176
commit 640fcb26f7
218 changed files with 51363 additions and 902 deletions
--- a/keep-notes/tests/unit/adaptive-weighting.test.ts
+++ b/keep-notes/tests/unit/adaptive-weighting.test.ts
@@ -0,0 +1,192 @@
+import { test, expect } from '@playwright/test';
+import { detectQueryType, getSearchWeights } from '../../lib/utils';
+import { QueryType } from '../../lib/types';
+
+test.describe('Query Type Detection Tests', () => {
+  test('should detect exact queries with quotes', () => {
+    expect(detectQueryType('"Error 404"')).toBe('exact');
+    expect(detectQueryType("'exact phrase'")).toBe('exact');
+    expect(detectQueryType('"multiple words"')).toBe('exact');
+  });
+
+  test('should detect conceptual queries', () => {
+    // Question words
+    expect(detectQueryType('how to cook pasta')).toBe('conceptual');
+    expect(detectQueryType('what is python')).toBe('conceptual');
+    expect(detectQueryType('where to find')).toBe('conceptual');
+    expect(detectQueryType('why does this happen')).toBe('conceptual');
+    expect(detectQueryType('who invented')).toBe('conceptual');
+
+    // Conceptual phrases
+    expect(detectQueryType('ways to improve')).toBe('conceptual');
+    expect(detectQueryType('best way to learn')).toBe('conceptual');
+    expect(detectQueryType('guide for beginners')).toBe('conceptual');
+    expect(detectQueryType('tips for cooking')).toBe('conceptual');
+    expect(detectQueryType('learn about javascript')).toBe('conceptual');
+    expect(detectQueryType('understand recursion')).toBe('conceptual');
+
+    // Learning patterns
+    expect(detectQueryType('tutorial on react')).toBe('conceptual');
+    expect(detectQueryType('guide to typescript')).toBe('conceptual');
+    expect(detectQueryType('introduction to python')).toBe('conceptual');
+    expect(detectQueryType('overview of microservices')).toBe('conceptual');
+    expect(detectQueryType('explanation of quantum computing')).toBe('conceptual');
+    expect(detectQueryType('examples of callbacks')).toBe('conceptual');
+  });
+
+  test('should detect mixed queries', () => {
+    // Simple terms
+    expect(detectQueryType('javascript')).toBe('mixed');
+    expect(detectQueryType('programming')).toBe('mixed');
+    expect(detectQueryType('cooking')).toBe('mixed');
+
+    // Multiple words without patterns
+    expect(detectQueryType('javascript programming')).toBe('mixed');
+    expect(detectQueryType('react components')).toBe('mixed');
+    expect(detectQueryType('database design')).toBe('mixed');
+  });
+
+  test('should be case-insensitive', () => {
+    expect(detectQueryType('How To Cook')).toBe('conceptual');
+    expect(detectQueryType('WHAT IS PYTHON')).toBe('conceptual');
+    expect(detectQueryType('"Error 404"')).toBe('exact');
+  });
+
+  test('should handle empty and whitespace queries', () => {
+    expect(detectQueryType('')).toBe('mixed');
+    expect(detectQueryType('   ')).toBe('mixed');
+  });
+});
+
+test.describe('Search Weight Calculation Tests', () => {
+  test('should return correct weights for exact queries', () => {
+    const weights = getSearchWeights('exact');
+
+    expect(weights.keywordWeight).toBe(2.0);
+    expect(weights.semanticWeight).toBe(0.7);
+  });
+
+  test('should return correct weights for conceptual queries', () => {
+    const weights = getSearchWeights('conceptual');
+
+    expect(weights.keywordWeight).toBe(0.7);
+    expect(weights.semanticWeight).toBe(1.5);
+  });
+
+  test('should return correct weights for mixed queries', () => {
+    const weights = getSearchWeights('mixed');
+
+    expect(weights.keywordWeight).toBe(1.0);
+    expect(weights.semanticWeight).toBe(1.0);
+  });
+
+  test('should handle unknown query type as mixed', () => {
+    const weights = getSearchWeights('unknown' as QueryType);
+
+    expect(weights.keywordWeight).toBe(1.0);
+    expect(weights.semanticWeight).toBe(1.0);
+  });
+});
+
+test.describe('Adaptive Weighting Integration Tests', () => {
+  test('exact query boosts keyword matches', () => {
+    const queryType = detectQueryType('"exact phrase"');
+    const weights = getSearchWeights(queryType);
+
+    // Keyword matches should be 2x more important
+    expect(weights.keywordWeight).toBeGreaterThan(weights.semanticWeight);
+    expect(weights.keywordWeight / weights.semanticWeight).toBeCloseTo(2.0 / 0.7, 1);
+  });
+
+  test('conceptual query boosts semantic matches', () => {
+    const queryType = detectQueryType('how to cook');
+    const weights = getSearchWeights(queryType);
+
+    // Semantic matches should be more important
+    expect(weights.semanticWeight).toBeGreaterThan(weights.keywordWeight);
+    expect(weights.semanticWeight / weights.keywordWeight).toBeCloseTo(1.5 / 0.7, 1);
+  });
+
+  test('mixed query treats both equally', () => {
+    const queryType = detectQueryType('javascript programming');
+    const weights = getSearchWeights(queryType);
+
+    // Both should have equal weight
+    expect(weights.keywordWeight).toBe(weights.semanticWeight);
+    expect(weights.keywordWeight).toBe(1.0);
+  });
+
+  test('weights significantly affect ranking scores', () => {
+    const k = 20;
+    const rank = 5;
+
+    // Same rank with different weights
+    const exactQueryScore = (1 / (k + rank)) * 2.0; // keyword
+    const conceptualQueryScore = (1 / (k + rank)) * 1.5; // semantic
+
+    // Exact keyword match should get highest score
+    expect(exactQueryScore).toBeGreaterThan(conceptualQueryScore);
+  });
+});
+
+test.describe('Weight Impact on Scenarios', () => {
+  test('scenario 1: User searches for "Error 404"', () => {
+    const query = '"Error 404"';
+    const queryType = detectQueryType(query);
+    const weights = getSearchWeights(queryType);
+
+    // Should be exact match type
+    expect(queryType).toBe('exact');
+
+    // Keyword matches should be heavily prioritized
+    expect(weights.keywordWeight).toBe(2.0);
+
+    // Semantic matches should be deprioritized
+    expect(weights.semanticWeight).toBe(0.7);
+
+    // This ensures that notes with "Error 404" appear first,
+    // even if semantic search might suggest other error types
+  });
+
+  test('scenario 2: User searches for "how to cook pasta"', () => {
+    const query = 'how to cook pasta';
+    const queryType = detectQueryType(query);
+    const weights = getSearchWeights(queryType);
+
+    // Should be conceptual type
+    expect(queryType).toBe('conceptual');
+
+    // Semantic matches should be boosted
+    expect(weights.semanticWeight).toBe(1.5);
+
+    // Keyword matches should be reduced
+    expect(weights.keywordWeight).toBe(0.7);
+
+    // This ensures that notes about cooking, pasta, recipes appear,
+    // even if they don't contain the exact words "how to cook pasta"
+  });
+
+  test('scenario 3: User searches for "tutorial javascript"', () => {
+    const query = 'tutorial javascript';
+    const queryType = detectQueryType(query);
+    const weights = getSearchWeights(queryType);
+
+    // Should be conceptual type (starts with "tutorial")
+    expect(queryType).toBe('conceptual');
+
+    // Semantic search should be prioritized
+    expect(weights.semanticWeight).toBeGreaterThan(weights.keywordWeight);
+  });
+
+  test('scenario 4: User searches for "react hooks"', () => {
+    const query = 'react hooks';
+    const queryType = detectQueryType(query);
+    const weights = getSearchWeights(queryType);
+
+    // Should be mixed type (no specific pattern)
+    expect(queryType).toBe('mixed');
+
+    // Both should have equal weight
+    expect(weights.keywordWeight).toBe(weights.semanticWeight);
+  });
+});
--- a/keep-notes/tests/unit/embedding-validation.test.ts
+++ b/keep-notes/tests/unit/embedding-validation.test.ts
@@ -0,0 +1,136 @@
+import { test, expect } from '@playwright/test'
+import { validateEmbedding, calculateL2Norm, normalizeEmbedding } from '../../lib/utils'
+
+test.describe('Embedding Validation', () => {
+  test.describe('validateEmbedding()', () => {
+    test('should validate a normal embedding', () => {
+      const embedding = [0.1, 0.2, 0.3, 0.4, 0.5]
+      const result = validateEmbedding(embedding)
+
+      expect(result.valid).toBe(true)
+      expect(result.issues).toHaveLength(0)
+    })
+
+    test('should reject empty embedding', () => {
+      const result = validateEmbedding([])
+
+      expect(result.valid).toBe(false)
+      expect(result.issues).toContain('Embedding is empty or has zero dimensionality')
+    })
+
+    test('should reject null embedding', () => {
+      const result = validateEmbedding(null as any)
+
+      expect(result.valid).toBe(false)
+      expect(result.issues).toContain('Embedding is empty or has zero dimensionality')
+    })
+
+    test('should reject embedding with NaN values', () => {
+      const embedding = [0.1, NaN, 0.3, 0.4, 0.5]
+      const result = validateEmbedding(embedding)
+
+      expect(result.valid).toBe(false)
+      expect(result.issues).toContain('Embedding contains NaN values')
+    })
+
+    test('should reject embedding with Infinity values', () => {
+      const embedding = [0.1, 0.2, Infinity, 0.4, 0.5]
+      const result = validateEmbedding(embedding)
+
+      expect(result.valid).toBe(false)
+      expect(result.issues).toContain('Embedding contains Infinity values')
+    })
+
+    test('should reject zero vector', () => {
+      const embedding = [0, 0, 0, 0, 0]
+      const result = validateEmbedding(embedding)
+
+      expect(result.valid).toBe(false)
+      expect(result.issues).toContain('Embedding is a zero vector (all values are 0)')
+    })
+
+    test('should warn about L2 norm outside normal range', () => {
+      // Very small norm
+      const smallEmbedding = [0.01, 0.01, 0.01]
+      const result1 = validateEmbedding(smallEmbedding)
+
+      expect(result1.valid).toBe(false)
+      expect(result1.issues.some(issue => issue.includes('L2 norm'))).toBe(true)
+
+      // Very large norm
+      const largeEmbedding = [2, 2, 2]
+      const result2 = validateEmbedding(largeEmbedding)
+
+      expect(result2.valid).toBe(false)
+      expect(result2.issues.some(issue => issue.includes('L2 norm'))).toBe(true)
+    })
+
+    test('should detect multiple issues', () => {
+      const embedding = [NaN, Infinity, 0]
+      const result = validateEmbedding(embedding)
+
+      expect(result.valid).toBe(false)
+      expect(result.issues.length).toBeGreaterThan(1)
+      expect(result.issues).toContain('Embedding contains NaN values')
+      expect(result.issues).toContain('Embedding contains Infinity values')
+      // Note: NaN and Infinity are not zero, so it won't detect zero vector
+    })
+  })
+
+  test.describe('calculateL2Norm()', () => {
+    test('should calculate correct L2 norm', () => {
+      const vector = [3, 4]
+      const norm = calculateL2Norm(vector)
+
+      expect(norm).toBe(5) // sqrt(3^2 + 4^2) = 5
+    })
+
+    test('should return 0 for zero vector', () => {
+      const vector = [0, 0, 0]
+      const norm = calculateL2Norm(vector)
+
+      expect(norm).toBe(0)
+    })
+
+    test('should handle negative values', () => {
+      const vector = [-3, -4]
+      const norm = calculateL2Norm(vector)
+
+      expect(norm).toBe(5) // sqrt((-3)^2 + (-4)^2) = 5
+    })
+  })
+
+  test.describe('normalizeEmbedding()', () => {
+    test('should normalize a vector to unit L2 norm', () => {
+      const embedding = [3, 4]
+      const normalized = normalizeEmbedding(embedding)
+      const norm = calculateL2Norm(normalized)
+
+      expect(norm).toBeCloseTo(1.0, 5)
+    })
+
+    test('should preserve direction of vector', () => {
+      const embedding = [1, 2, 3]
+      const normalized = normalizeEmbedding(embedding)
+
+      // Check that ratios are preserved
+      expect(normalized[1] / normalized[0]).toBeCloseTo(embedding[1] / embedding[0], 5)
+      expect(normalized[2] / normalized[1]).toBeCloseTo(embedding[2] / embedding[1], 5)
+    })
+
+    test('should return zero vector unchanged', () => {
+      const embedding = [0, 0, 0]
+      const normalized = normalizeEmbedding(embedding)
+
+      expect(normalized).toEqual(embedding)
+    })
+
+    test('should handle already normalized vectors', () => {
+      const embedding = [0.707, 0.707] // Already approximately unit norm
+      const normalized = normalizeEmbedding(embedding)
+      const norm = calculateL2Norm(normalized)
+
+      expect(norm).toBeCloseTo(1.0, 5)
+    })
+  })
+})
--- a/keep-notes/tests/unit/rrf.test.ts
+++ b/keep-notes/tests/unit/rrf.test.ts
@@ -0,0 +1,152 @@
+import { test, expect } from '@playwright/test';
+import { calculateRRFK } from '../../lib/utils';
+
+test.describe('RRF K Calculation Tests', () => {
+  test('should return minimum k=20 for small datasets', () => {
+    // For small datasets (< 200 notes), k should be 20
+    expect(calculateRRFK(0)).toBe(20);
+    expect(calculateRRFK(10)).toBe(20);
+    expect(calculateRRFK(50)).toBe(20);
+    expect(calculateRRFK(100)).toBe(20);
+    expect(calculateRRFK(199)).toBe(20);
+  });
+
+  test('should return k=20 for exactly 200 notes', () => {
+    expect(calculateRRFK(200)).toBe(20);
+  });
+
+  test('should scale k for larger datasets', () => {
+    // k = max(20, totalNotes / 10)
+    expect(calculateRRFK(500)).toBe(50);  // 500/10 = 50
+    expect(calculateRRFK(1000)).toBe(100); // 1000/10 = 100
+    expect(calculateRRFK(250)).toBe(25);   // 250/10 = 25
+  });
+
+  test('should handle edge cases', () => {
+    expect(calculateRRFK(201)).toBe(20);   // 201/10 = 20.1 → floor → 20, max(20,20) = 20
+    expect(calculateRRFK(210)).toBe(21);   // 210/10 = 21
+    expect(calculateRRFK(10000)).toBe(1000); // 10000/10 = 1000
+  });
+
+  test('k should always be at least 20', () => {
+    // Even for very small datasets, k should not be below 20
+    for (let i = 0; i <= 200; i++) {
+      expect(calculateRRFK(i)).toBeGreaterThanOrEqual(20);
+    }
+  });
+
+  test('should be lower than old value for typical datasets', () => {
+    // For typical user datasets (< 500 notes), new k should be lower than old k=60
+    expect(calculateRRFK(100)).toBeLessThan(60);
+    expect(calculateRRFK(200)).toBeLessThan(60);
+    expect(calculateRRFK(300)).toBe(30); // 300/10 = 30
+    expect(calculateRRFK(500)).toBe(50); // 500/10 = 50
+  });
+
+  test('should surpass old value for very large datasets', () => {
+    // For very large datasets (> 600 notes), k should be higher than 60
+    expect(calculateRRFK(700)).toBe(70); // 700/10 = 70
+    expect(calculateRRFK(1000)).toBe(100);
+  });
+});
+
+test.describe('RRF Ranking Behavior Tests', () => {
+  test('RRF with lower k penalizes low ranks more', () => {
+    // Simulate RRF scores for a note at rank 5
+    // Formula: score = 1 / (k + rank)
+
+    const rank = 5;
+    const scoreWithK20 = 1 / (20 + rank);  // 1/25 = 0.04
+    const scoreWithK60 = 1 / (60 + rank);  // 1/65 = 0.015
+
+    // Lower k gives higher score to low ranks
+    expect(scoreWithK20).toBeGreaterThan(scoreWithK60);
+  });
+
+  test('RRF favors items ranked high in both lists', () => {
+    // Note A: rank 1 in both lists
+    const scoreA_K20 = 1 / (20 + 1) + 1 / (20 + 1); // 2/21 ≈ 0.095
+
+    // Note B: rank 1 in one list, rank 10 in other
+    const scoreB_K20 = 1 / (20 + 1) + 1 / (20 + 10); // 1/21 + 1/30 ≈ 0.081
+
+    // Note C: rank 5 in both lists
+    const scoreC_K20 = 1 / (20 + 5) + 1 / (20 + 5); // 2/25 = 0.08
+
+    // Note A should have highest score (consistently high)
+    expect(scoreA_K20).toBeGreaterThan(scoreB_K20);
+    expect(scoreA_K20).toBeGreaterThan(scoreC_K20);
+  });
+
+  test('k=20 vs k=60 ranking difference', () => {
+    // Note at rank 20
+    const rank = 20;
+
+    const scoreWithK20 = 1 / (20 + rank); // 1/40 = 0.025
+    const scoreWithK60 = 1 / (60 + rank); // 1/80 = 0.0125
+
+    // With k=20, rank 20 is scored 2x higher than with k=60
+    expect(scoreWithK20 / scoreWithK60).toBeCloseTo(2.0, 1);
+  });
+
+  test('RRF score should decrease as rank increases', () => {
+    const k = 20;
+
+    const scoreRank1 = 1 / (k + 1);
+    const scoreRank5 = 1 / (k + 5);
+    const scoreRank10 = 1 / (k + 10);
+    const scoreRank50 = 1 / (k + 50);
+
+    expect(scoreRank1).toBeGreaterThan(scoreRank5);
+    expect(scoreRank5).toBeGreaterThan(scoreRank10);
+    expect(scoreRank10).toBeGreaterThan(scoreRank50);
+  });
+
+  test('RRF handles missing ranks gracefully', () => {
+    // If a note is not in a list, it gets the max rank
+    const k = 20;
+    const totalNotes = 100;
+    const missingRank = totalNotes; // Treated as worst rank
+
+    const scoreWithMissing = 1 / (k + missingRank);
+    const scoreWithRank50 = 1 / (k + 50);
+
+    // Missing rank should give much lower score
+    expect(scoreWithMissing).toBeLessThan(scoreWithRank50);
+  });
+});
+
+test.describe('RRF Adaptive Behavior Tests', () => {
+  test('adaptive k provides better rankings for small datasets', () => {
+    // For 50 notes: k=20 (adaptive) vs k=60 (old)
+    const totalNotes = 50;
+    const kAdaptive = calculateRRFK(totalNotes); // 20
+    const kOld = 60;
+
+    // Compare scores for rank 10 (20% of dataset)
+    const rank = 10;
+    const scoreAdaptive = 1 / (kAdaptive + rank);
+    const scoreOld = 1 / (kOld + rank);
+
+    // Adaptive k gives higher score to mid-rank items in small datasets
+    expect(scoreAdaptive).toBeGreaterThan(scoreOld);
+  });
+
+  test('adaptive k scales appropriately', () => {
+    const datasets = [10, 50, 100, 200, 500, 1000];
+
+    datasets.forEach(notes => {
+      const k = calculateRRFK(notes);
+
+      // k should always be at least 20
+      expect(k).toBeGreaterThanOrEqual(20);
+
+      // k should scale with dataset size
+      if (notes < 200) {
+        expect(k).toBe(20);
+      } else {
+        expect(k).toBe(Math.floor(notes / 10));
+      }
+    });
+  });
+});