chore: clean up repo for public release
- Remove BMAD framework, IDE configs, dev screenshots, test files, internal docs, and backup files - Rename keep-notes/ to memento-note/ - Update all references from keep-notes to memento-note - Add Apache 2.0 license with Commons Clause (non-commercial restriction) - Add clean .gitignore and .env.docker.example
This commit is contained in:
59
memento-note/app/actions/scrape.ts
Normal file
59
memento-note/app/actions/scrape.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
'use server'
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
|
||||
export interface LinkMetadata {
|
||||
url: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
imageUrl?: string;
|
||||
siteName?: string;
|
||||
}
|
||||
|
||||
export async function fetchLinkMetadata(url: string): Promise<LinkMetadata | null> {
|
||||
try {
|
||||
// Add protocol if missing
|
||||
let targetUrl = url;
|
||||
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
||||
targetUrl = 'https://' + url;
|
||||
}
|
||||
|
||||
const response = await fetch(targetUrl, {
|
||||
headers: {
|
||||
// Use a real browser User-Agent to avoid 403 Forbidden from strict sites
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5'
|
||||
},
|
||||
next: { revalidate: 3600 } // Cache for 1 hour
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const getMeta = (prop: string) =>
|
||||
$(`meta[property="${prop}"]`).attr('content') ||
|
||||
$(`meta[name="${prop}"]`).attr('content');
|
||||
|
||||
// Robust extraction with fallbacks
|
||||
const title = getMeta('og:title') || $('title').text() || getMeta('twitter:title') || url;
|
||||
const description = getMeta('og:description') || getMeta('description') || getMeta('twitter:description') || '';
|
||||
const imageUrl = getMeta('og:image') || getMeta('twitter:image') || $('link[rel="image_src"]').attr('href');
|
||||
const siteName = getMeta('og:site_name') || '';
|
||||
|
||||
return {
|
||||
url: targetUrl,
|
||||
title: title.substring(0, 100),
|
||||
description: description.substring(0, 200),
|
||||
imageUrl,
|
||||
siteName
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`[Scrape] Error fetching ${url}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user