diff --git a/.github/workflows/import-rss-feeds.yml b/.github/workflows/import-rss-feeds.yml new file mode 100644 index 0000000..42e1af1 --- /dev/null +++ b/.github/workflows/import-rss-feeds.yml @@ -0,0 +1,51 @@ +name: Import RSS Feeds + +on: + # Run daily at 6 AM UTC + schedule: + - cron: '0 6 * * *' + # Allow manual trigger + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + import-feeds: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '22' + + - name: Run RSS feed importer + run: node scripts/import-rss-feeds.mjs + + - name: Check for changes + id: check_changes + run: | + if [[ -n $(git status --porcelain src/content/stories) ]]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Open pull request with new stories + if: steps.check_changes.outputs.has_changes == 'true' + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "chore: import new RSS feed stories" + branch: automated/import-rss-feeds + title: "chore: import new RSS feed stories" + body: | + This pull request was automatically generated by the RSS feed importer. + + It contains new stories fetched from AI4EOSC RSS feeds. Please review before merging. + labels: automated + add-paths: src/content/stories/*.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..403332d --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,44 @@ +# RSS Feed Importer + +This directory contains the RSS feed importer script that automatically fetches news from AI4EOSC domains and creates story files. + +## Script + +- `import-rss-feeds.mjs`: Node.js script that fetches RSS feeds and creates story markdown files + +## How it works + +1. **Fetches RSS feeds** from: + - https://origin.ai4eosc.eu/feed/ + - https://fluid.ai4eosc.eu/feed/ + - https://arena.ai4eosc.eu/feed/ + +2. **Parses XML** to extract: + - Title + - Link (used as external URL) + - Publication date + - Description (used as excerpt) + - Featured image + +3. **Checks for duplicates** by comparing external URLs with existing stories + +4. **Creates new story files** in `src/content/stories/` with: + - Filename format: `YYYY-MM-DD-slug.md` + - Frontmatter with title, type (News), date, excerpt, image, and externalUrl + - Empty body (since these are external stories) + +## GitHub Action + +The script is automatically run daily at 6 AM UTC via the `.github/workflows/import-rss-feeds.yml` workflow. + +You can also trigger it manually from the GitHub Actions tab. + +## Manual usage + +To run the script manually: + +```bash +node scripts/import-rss-feeds.mjs +``` + +This will fetch the latest RSS feeds and create story files for any new entries that don't already exist. diff --git a/scripts/import-rss-feeds.mjs b/scripts/import-rss-feeds.mjs new file mode 100644 index 0000000..4ff6d51 --- /dev/null +++ b/scripts/import-rss-feeds.mjs @@ -0,0 +1,233 @@ +#!/usr/bin/env node + +/** + * RSS Feed Importer + * + * Fetches RSS feeds from AI4EOSC domains and creates story files + * for new entries in src/content/stories + */ + +import { readdir, readFile, writeFile } from 'fs/promises'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// RSS feed URLs +const RSS_FEEDS = [ + 'https://origin.ai4eosc.eu/feed/', + 'https://fluid.ai4eosc.eu/feed/', + 'https://arena.ai4eosc.eu/feed/' +]; + +// Path to stories directory +const STORIES_DIR = join(__dirname, '..', 'src', 'content', 'stories'); + +/** + * Fetch and parse RSS feed + */ +async function fetchRssFeed(url) { + try { + console.log(`Fetching RSS feed from ${url}`); + const response = await fetch(url); + if (!response.ok) { + console.error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + return []; + } + + const text = await response.text(); + return parseRssFeed(text, url); + } catch (error) { + console.error(`Error fetching ${url}:`, error.message); + return []; + } +} + +/** + * Parse RSS feed XML + */ +function parseRssFeed(xml, feedUrl) { + const items = []; + + // Extract items from RSS feed + const itemRegex = /([\s\S]*?)<\/item>/g; + let match; + + while ((match = itemRegex.exec(xml)) !== null) { + const itemXml = match[1]; + + // Extract fields + const title = extractTag(itemXml, 'title'); + const link = extractTag(itemXml, 'link'); + const pubDate = extractTag(itemXml, 'pubDate'); + const description = extractTag(itemXml, 'description'); + const content = extractTag(itemXml, 'content:encoded') || description; + + // Extract image from content or description + let imageUrl = ''; + const imgMatch = (content || description).match(/]+src=["']([^"']+)["']/i); + if (imgMatch) { + imageUrl = imgMatch[1]; + } + + if (title && link && pubDate) { + items.push({ + title: cleanText(title), + link: link.trim(), + pubDate: new Date(pubDate), + description: cleanText(description), + image: imageUrl, + feedUrl + }); + } + } + + console.log(`Found ${items.length} items in ${feedUrl}`); + return items; +} + +/** + * Extract content from XML tag + */ +function extractTag(xml, tagName) { + const regex = new RegExp(`<${tagName}(?:[^>]*)><\/${tagName}>|<${tagName}(?:[^>]*)>([\\s\\S]*?)<\/${tagName}>`, 'i'); + const match = xml.match(regex); + return match ? (match[1] || match[2] || '').trim() : ''; +} + +/** + * Clean HTML and entities from text + */ +function cleanText(text) { + return text + .replace(//g, '$1') + .replace(/<[^>]+>/g, '') + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Generate filename from title and date + */ +function generateFilename(title, date) { + const dateStr = date.toISOString().split('T')[0]; // YYYY-MM-DD + const slug = title + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .substring(0, 50); + + return `${dateStr}-${slug}.md`; +} + +/** + * Create story frontmatter + */ +function createStoryContent(item) { + const dateStr = item.pubDate.toISOString().split('T')[0]; + + // Truncate description to a reasonable length for excerpt + let excerpt = item.description; + if (excerpt.length > 200) { + excerpt = excerpt.substring(0, 197) + '...'; + } + + const frontmatter = [ + '---', + `title: "${item.title.replace(/"/g, '\\"')}"`, + `type: News`, + `date: ${dateStr}`, + `excerpt: ${excerpt.replace(/"/g, '\\"')}`, + ]; + + if (item.image) { + frontmatter.push(`image: ${item.image}`); + } + + frontmatter.push(`externalUrl: ${item.link}`); + frontmatter.push('---'); + frontmatter.push(''); + + return frontmatter.join('\n'); +} + +/** + * Get existing story files + */ +async function getExistingStories() { + try { + const files = await readdir(STORIES_DIR); + const stories = new Set(); + + for (const file of files) { + if (file.endsWith('.md') && file !== '_template.md') { + const content = await readFile(join(STORIES_DIR, file), 'utf-8'); + + // Extract externalUrl from frontmatter + const urlMatch = content.match(/externalUrl:\s*(.+)/); + if (urlMatch) { + stories.add(urlMatch[1].trim()); + } + } + } + + return stories; + } catch (error) { + console.error('Error reading existing stories:', error.message); + return new Set(); + } +} + +/** + * Main function + */ +async function main() { + console.log('Starting RSS feed import...'); + + // Get existing stories + const existingUrls = await getExistingStories(); + console.log(`Found ${existingUrls.size} existing stories`); + + // Fetch all RSS feeds + const allItems = []; + for (const feedUrl of RSS_FEEDS) { + const items = await fetchRssFeed(feedUrl); + allItems.push(...items); + } + + console.log(`Total items fetched: ${allItems.length}`); + + // Filter out existing stories + const newItems = allItems.filter(item => !existingUrls.has(item.link)); + console.log(`New items to add: ${newItems.length}`); + + // Create story files for new items + let createdCount = 0; + for (const item of newItems) { + const filename = generateFilename(item.title, item.pubDate); + const filepath = join(STORIES_DIR, filename); + const content = createStoryContent(item); + + try { + await writeFile(filepath, content, 'utf-8'); + console.log(`Created: ${filename}`); + createdCount++; + } catch (error) { + console.error(`Error creating ${filename}:`, error.message); + } + } + + console.log(`\nImport complete: ${createdCount} new stories created`); +} + +main().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +});