From a78b4b4ef5408044c7c64ae9e10784eccd64f225 Mon Sep 17 00:00:00 2001 From: chamik Date: Fri, 5 Jun 2026 15:39:41 +0200 Subject: [PATCH] added proxy for rss requests, individual delay for feeds --- thorn/Config/FeedConfig.cs | 1 + thorn/Config/config.example.json | 1 + thorn/Config/feeds.example.json | 15 ++++-- thorn/Jobs/RssJob.cs | 90 ++++++++++++++++++++++++-------- thorn/Program.cs | 25 +++++++-- 5 files changed, 101 insertions(+), 31 deletions(-) diff --git a/thorn/Config/FeedConfig.cs b/thorn/Config/FeedConfig.cs index 377f90c..edbd69c 100644 --- a/thorn/Config/FeedConfig.cs +++ b/thorn/Config/FeedConfig.cs @@ -9,4 +9,5 @@ public class FeedConfig public string CustomDescription { get; set; } public bool NewPageAnnouncement { get; set; } public uint EmbedColor { get; set; } + public int Delay { get; set; } } \ No newline at end of file diff --git a/thorn/Config/config.example.json b/thorn/Config/config.example.json index 5d714c0..9c7d2a1 100644 --- a/thorn/Config/config.example.json +++ b/thorn/Config/config.example.json @@ -1,5 +1,6 @@ { "token": "YOUR-TOKEN", + "rssProxy": "socks5://10.64.0.1:1080", "helpText": { "translate": "[Návod pro Překladatele]()\n[Rychlý Start pro Překladatele]()\n[Potřebné Překlady]()\n[Překladatelské Stránky]()", "write": "[Jak napsat SCP]()\n[Pravidla Kritizování]()\n[Autorské Stránky]()", diff --git a/thorn/Config/feeds.example.json b/thorn/Config/feeds.example.json index dfe383c..9f12a4b 100644 --- a/thorn/Config/feeds.example.json +++ b/thorn/Config/feeds.example.json @@ -5,14 +5,16 @@ "Filter": ["nová stránka"], "FilterIgnore": ["fragment:"], "EmbedColor": 16711680, - "NewPageAnnouncement": true + "NewPageAnnouncement": true, + "Delay": 120 }, { "Link": "http://scp-cs.wikidot.com/feed/site-changes.xml", "ChannelIds": [710575651664429127], "EmbedColor": 16711680, - "NewPageAnnouncement": false + "NewPageAnnouncement": false, + "Delay": 120 }, { "Link": "http://o5command-cs.wikidot.com/feed/site-changes.xml", @@ -25,21 +27,24 @@ "ChannelIds": [710575651664429127], "Filter": ["nová stránka"], "EmbedColor": 3329330, - "NewPageAnnouncement": false + "NewPageAnnouncement": false, + "Delay": 300 }, { "Link": "http://scp-cs-sandbox-2.wikidot.com/feed/site-changes.xml", "ChannelIds": [710575651664429127], "Filter": ["nová stránka"], "EmbedColor": 3329330, - "NewPageAnnouncement": false + "NewPageAnnouncement": false, + "Delay": 300 }, { "Link": "http://scp-cs-sandbox-3.wikidot.com/feed/site-changes.xml", "ChannelIds": [710575651664429127], "Filter": ["nová stránka"], "EmbedColor": 3329330, - "NewPageAnnouncement": false + "NewPageAnnouncement": false, + "Delay": 300 }, { "Link": "http://scp-cs.wikidot.com/feed/forum/threads.xml", diff --git a/thorn/Jobs/RssJob.cs b/thorn/Jobs/RssJob.cs index d7d546d..60c90a8 100644 --- a/thorn/Jobs/RssJob.cs +++ b/thorn/Jobs/RssJob.cs @@ -3,6 +3,8 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -10,6 +12,7 @@ using Discord; using Discord.WebSocket; using Html2Markdown; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Quartz; @@ -17,18 +20,24 @@ namespace thorn.Jobs; -public class RssJob : IJob +[DisallowConcurrentExecution] // there is shared state in this object +public partial class RssJob : IJob { + public const int DefaultDelaySeconds = 10800; + public const string FeedIndexKey = "feedIndex"; + private readonly ILogger _logger; private readonly List _configs; private readonly Dictionary _channels; - private Dictionary _lastUpdates; + private readonly HttpClient _httpClient; + private readonly Dictionary _lastUpdates; - public RssJob(ILogger logger, DiscordSocketClient client) + public RssJob(ILogger logger, DiscordSocketClient client, IConfiguration configuration) { _logger = logger; _channels = new Dictionary(); _lastUpdates = new Dictionary(); + _httpClient = BuildHttpClient(configuration["rssProxy"]); _configs = JsonConvert.DeserializeObject>(File.ReadAllText("Config/feeds.json")); @@ -43,28 +52,61 @@ public RssJob(ILogger logger, DiscordSocketClient client) _lastUpdates.Add(feedConfig, null); } + private HttpClient BuildHttpClient(string proxy) + { + if (string.IsNullOrWhiteSpace(proxy)) + { + _logger.LogInformation("RSS proxy not configured"); + return new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + } + + _logger.LogInformation("RSS routed through proxy {Proxy}", proxy); + var handler = new SocketsHttpHandler + { + Proxy = new WebProxy(proxy), + UseProxy = true + }; + return new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(10) }; + } + public async Task Execute(IJobExecutionContext context) { - foreach (var config in _configs) + var index = context.MergedJobDataMap.GetInt(FeedIndexKey); + if (index < 0 || index >= _configs.Count) + { + _logger.LogError("RssJob fired with out-of-range feed index {Index}", index); + return; + } + + var config = _configs[index]; + var newItems = await GetNewItems(config); + if (newItems is null) return; + + foreach (var feedItem in newItems) + foreach (var channelId in config.ChannelIds) { - var newItems = await GetNewItems(config); - if (newItems is null) continue; - - foreach (var feedItem in newItems) - foreach (var channelId in config.ChannelIds) - { - var channel = _channels[channelId]; - if (channel is null) continue; - - await channel.SendMessageAsync(embed: GetEmbed(feedItem, config)); - _logger.LogInformation("Sent RSS feed '{Title}' to #{Channel}", feedItem.Title, channel); - } + var channel = _channels[channelId]; + if (channel is null) continue; + + await channel.SendMessageAsync(embed: GetEmbed(feedItem, config)); + _logger.LogInformation("Sent RSS feed '{Title}' to #{Channel}", feedItem.Title, channel); } } private async Task> GetNewItems(FeedConfig feedConfig) { - var feed = await FeedReader.ReadAsync(feedConfig.Link); + Feed feed; + try + { + var content = await _httpClient.GetStringAsync(feedConfig.Link); + feed = FeedReader.ReadFromString(content); + } + catch (Exception e) + { + _logger.LogWarning(e, "Failed to fetch RSS feed {Link}", feedConfig.Link); + return null; + } + var lastUpdate = _lastUpdates[feedConfig]; if (lastUpdate is null) @@ -122,7 +164,7 @@ private Embed GetEmbed(FeedItem feedItem, FeedConfig feedConfig) private Embed GetAnnouncementEmbed(FeedItem feedItem, FeedConfig feedConfig, string text) { - var title = Regex.Match(feedItem.Title, "\"(.*)\" - .*").Groups[1].Value; + var title = GetTitle(feedItem.Title); var author = GetUsername(text); var description = new StringBuilder("Nový článek na wiki! Yay! \\o/\n"); @@ -142,8 +184,12 @@ private Embed GetAnnouncementEmbed(FeedItem feedItem, FeedConfig feedConfig, str }.Build(); } - private string GetUsername(string source) - { - return Regex.Match(source, "user:info\\/([^)]*)").Groups[1].Value; - } + private string GetTitle(string source) => TitleRegex().Match(source).Groups[1].Value; + private string GetUsername(string source) => UsernameRegex().Match(source).Groups[1].Value; + + [GeneratedRegex("user:info\\/([^)]*)")] + private static partial Regex UsernameRegex(); + + [GeneratedRegex("\"(.*)\" - .*")] + private static partial Regex TitleRegex(); } \ No newline at end of file diff --git a/thorn/Program.cs b/thorn/Program.cs index f883f2a..157d9eb 100644 --- a/thorn/Program.cs +++ b/thorn/Program.cs @@ -1,4 +1,7 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; using Discord; using Discord.Addons.Hosting; using Discord.WebSocket; @@ -6,9 +9,11 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Newtonsoft.Json; using Quartz; using Serilog; using Serilog.Events; +using thorn.Config; using thorn.Jobs; using thorn.Services; @@ -68,10 +73,22 @@ private static async Task Main() builder.Services.AddQuartz(configure => { - var tenMinSchedule = CronScheduleBuilder.CronSchedule("0 */10 * ? * *"); + var feeds = JsonConvert.DeserializeObject>(File.ReadAllText("Config/feeds.json")); var rssJob = new JobKey(nameof(RssJob)); - configure.AddJob(rssJob) - .AddTrigger(t => t.ForJob(rssJob).WithSchedule(tenMinSchedule)); + configure.AddJob(rssJob); + + // create a trigger for each feed + for (var i = 0; i < feeds.Count; i++) + { + var index = i; + var delay = feeds[i].Delay > 0 ? feeds[i].Delay : RssJob.DefaultDelaySeconds; + var startAt = DateTimeOffset.UtcNow.AddSeconds(5 + index * 5); + configure.AddTrigger(t => t + .ForJob(rssJob) + .UsingJobData(RssJob.FeedIndexKey, index) + .StartAt(startAt) + .WithSimpleSchedule(s => s.WithIntervalInSeconds(delay).RepeatForever())); + } var dailySchedule = CronScheduleBuilder.CronSchedule("0 0 0 * * ?"); var reminderJob = new JobKey(nameof(ReminderJob));