diff --git a/GenHub/GenHub.Core/Constants/ContentConstants.cs b/GenHub/GenHub.Core/Constants/ContentConstants.cs index 0bc3c907c..96d6a6db0 100644 --- a/GenHub/GenHub.Core/Constants/ContentConstants.cs +++ b/GenHub/GenHub.Core/Constants/ContentConstants.cs @@ -94,4 +94,31 @@ public static class ContentConstants /// Maximum allowed size for the content catalog in bytes (10 MB). /// public const long MaxCatalogSizeBytes = 10 * ConversionConstants.BytesPerMegabyte; + + /// + /// Shared resolver/display metadata key for map player counts. + /// Builtin and catalog publishers should set this so download cards can render a consistent badge. + /// + public const string PlayerCountMetadataKey = "playerCount"; + + /// + /// Shared resolver/display metadata key for content categories (AOA, Compstomp, ModDB category, etc.). + /// + public const string CategoryMetadataKey = "category"; + + /// + /// Display metadata key for a comma-separated list of included/required content names + /// (e.g. catalog ContentBundle dependencies resolved to friendly titles). + /// + public const string IncludesSummaryMetadataKey = "includesSummary"; + + /// + /// Number of recent releases and addons to eagerly preload extended details for. + /// + public const int PreloadRecentItemsLimit = 5; + + /// + /// Maximum concurrent background requests when preloading recent item details. + /// + public const int PreloadConcurrencyLimit = 3; } \ No newline at end of file diff --git a/GenHub/GenHub.Core/Constants/DirectoryNames.cs b/GenHub/GenHub.Core/Constants/DirectoryNames.cs index 47097cc18..19b341d80 100644 --- a/GenHub/GenHub.Core/Constants/DirectoryNames.cs +++ b/GenHub/GenHub.Core/Constants/DirectoryNames.cs @@ -59,4 +59,14 @@ public static class DirectoryNames /// Directory for storing tool workspaces. /// public const string ToolWorkspaces = "ToolWorkspaces"; + + /// + /// Directory for persistent Playwright browser profiles (cookies/storage for bot-protected sites). + /// + public const string BrowserProfiles = "BrowserProfiles"; + + /// + /// Directory for the app-owned Playwright Chromium runtime (not the system Chrome/Edge install). + /// + public const string BrowserRuntime = "BrowserRuntime"; } diff --git a/GenHub/GenHub.Core/Constants/ModDBConstants.cs b/GenHub/GenHub.Core/Constants/ModDBConstants.cs index 2deed935c..f8b9c8d3d 100644 --- a/GenHub/GenHub.Core/Constants/ModDBConstants.cs +++ b/GenHub/GenHub.Core/Constants/ModDBConstants.cs @@ -72,6 +72,12 @@ public static class ModDBConstants /// ModDB website URL. public const string PublisherWebsite = BaseUrl; + /// + /// On-disk Playwright browser profile name used to persist the Cloudflare clearance cookie so + /// the user only solves the bot challenge once per session (and across restarts until expiry). + /// + public const string BrowserProfileName = "moddb"; + /// Short description for publisher card display. public const string ShortDescription = "Community mods, maps, and content from ModDB"; @@ -184,6 +190,29 @@ public static class ModDBConstants /// Value for filter parameter when enabled. public const string FilterEnabledValue = "t"; + // ===== Sort Values ===== + + /// Sort: Date descending (newest first). + public const string SortDateDesc = "date-desc"; + + /// Sort: Date ascending (oldest first). + public const string SortDateAsc = "date-asc"; + + /// Sort: Visits / Popularity descending. + public const string SortVisitDesc = "visit-desc"; + + /// Sort: Rating descending. + public const string SortRatingDesc = "rating-desc"; + + /// Sort: Name ascending (A-Z). + public const string SortNameAsc = "name-asc"; + + /// Sort: Name descending (Z-A). + public const string SortNameDesc = "name-desc"; + + /// Default sort value for ModDB searches and listings (newest first). + public const string DefaultSort = SortDateDesc; + // ===== Category Values ===== // Downloads Section - Releases @@ -356,6 +385,34 @@ public static class ModDBConstants /// Metadata key for original category. public const string OriginalCategoryMetadataKey = "moddbCategory"; + /// Metadata key for identifying if content is a mod. + public const string IsModMetadataKey = "IsMod"; + + /// Metadata key for parent mod URL. + public const string ParentModUrlMetadataKey = "ParentModUrl"; + + // ===== Playwright / Scraping Constants ===== + + /// Default timeout for page navigation (ms). + public const int DefaultGotoTimeout = 30000; + + /// + /// Default timeout for waiting for a selector (ms). ModDB sits behind Cloudflare; the headed + /// browser persistent profile usually receives the clearance cookie after verification, but 15 s gives a safe margin + /// for manual challenge solves before the scraper parses whatever it has. + /// + public const int DefaultSelectorTimeout = 15000; + + /// + /// How long (ms) the listing scrape waits for the user to solve a Cloudflare challenge in the + /// visible browser before giving up. Long enough for a manual "I am not a robot" click; the + /// page stays open after the deadline so the user can finish and retry. + /// + public const int VerificationWaitTimeoutMs = 120000; + + /// Selector for content items in listing pages (Fallback). + public const string DefaultListItemSelector = "div.row.rowcontent, div.table tr"; + // ===== Error Messages ===== /// Error message for invalid URL. diff --git a/GenHub/GenHub.Core/Constants/ModDBParserConstants.cs b/GenHub/GenHub.Core/Constants/ModDBParserConstants.cs index 55ab7fa3c..4f61403a9 100644 --- a/GenHub/GenHub.Core/Constants/ModDBParserConstants.cs +++ b/GenHub/GenHub.Core/Constants/ModDBParserConstants.cs @@ -17,6 +17,9 @@ public static class ModDBParserConstants /// Selector for developer/publisher links. public const string DeveloperSelector = "a[href*='/members/'], a[href*='/company/']"; + /// Selector for the profile/mod info box that carries the real developer name. + public const string DeveloperProfileSelector = "#modsinfo a[href*='/members/'], #modsinfo a[href*='/company/'], .sidecolumn a[href*='/members/'], .sidecolumn a[href*='/company/']"; + /// Selector for release date. public const string ReleaseDateSelector = "time[datetime], .date, .released"; @@ -59,7 +62,7 @@ public static class ModDBParserConstants public const string FileMetadataValueSelector = "td:last-child"; /// Selector for the main download button on file pages. - public const string MainDownloadButtonSelector = "a.download, a.downloadarea, .downloadbutton a, a[href*='/downloads/start/']"; + public const string MainDownloadButtonSelector = "a.download, a.downloadarea, .downloadbutton a, a[href*='/downloads/start/'], a[href*='/addons/start/']"; /// Selector for download size on the button. public const string DownloadSizeSelector = ".download .size, .downloadbutton .size"; @@ -87,10 +90,13 @@ public static class ModDBParserConstants // ===== Description/Summary Selectors ===== /// Selector for full description content. - public const string FullDescriptionSelector = "#articlebrowse, .summary .content, .description .content, .modtext"; + public const string FullDescriptionSelector = "#downloaddescription, #downloadsummary, #articlebrowse .articlebody, .articlebody, #modsummary, .modtext, #profile .description, #description, #articlebrowse, .summary .content, .description .content"; + + /// Selector for the file-page body copy (not the breadcrumb .summary trail). + public const string FileDescriptionSelector = "#downloaddescription, #downloadsummary"; - /// Selector for truncated summary. - public const string SummarySelector = ".summary p, .description p"; + /// Selector for summary or description container. + public const string SummarySelector = ".description, .rubric, p[itemprop='description']"; // ===== Legacy File Selectors ===== @@ -98,28 +104,31 @@ public static class ModDBParserConstants public const string FilesTableSelector = "table.filelist, .table.files, #files"; /// Selector for individual file rows. - public const string FileRowSelector = "tr.file, .row.file, .file"; + public const string FileRowSelector = "tr.file, .row.file, .file, .row.rowcontent"; /// Selector for file name. - public const string FileNameSelector = "h5, h4, .name, .title"; + public const string FileNameSelector = "h4 a, h5 a, h3 a, .heading a, .title a, a.title, .name a, h5, h4, .name, .title"; /// Selector for file version. public const string FileVersionSelector = ".version, .ver"; /// Selector for file size. - public const string FileSizeSelector = ".size, .filesize"; + public const string FileSizeSelector = ".size, .filesize, .filesizes, span.size"; + + /// Selector for file subheading or metadata row. + public const string FileSubheadingSelector = ".subheading, span.subheading, .meta, .details, .info, p.summary, .summary"; /// Selector for file upload date. - public const string FileDateSelector = "time[datetime], .date, .uploaded"; + public const string FileDateSelector = "time[datetime], .date, .uploaded, time"; /// Selector for file category. - public const string FileCategorySelector = ".category, .type"; + public const string FileCategorySelector = ".category, .type, span.category"; /// Selector for file uploader. - public const string FileUploaderSelector = ".uploader, .author, a[href*='/members/']"; + public const string FileUploaderSelector = ".uploader, .author, a[href*='/members/'], a[href*='/company/']"; /// Selector for file download link (robust). - public const string FileDownloadSelector = "a.button.download, a[href*='/downloads/start/'], .download a"; + public const string FileDownloadSelector = "a.button, a.buttonlarge, a.download, a.btn, a[href*='/downloads/start/'], a[href*='/addons/start/'], a[href*='/downloads/'], a[href*='/addons/'], .download a, .actions a"; /// Selector for file MD5 hash. public const string FileMd5Selector = ".md5, .hash"; @@ -130,22 +139,41 @@ public static class ModDBParserConstants // ===== Videos Section Selectors ===== /// Selector for embedded video iframes. - public const string VideoSelector = "iframe[src*='youtube'], iframe[src*='vimeo'], iframe[src*='youtu.be']"; + public const string VideoSelector = "iframe[src*='youtube'], iframe[src*='youtube-nocookie'], iframe[src*='youtu.be'], iframe[src*='vimeo'], iframe[src*='dailymotion'], iframe[src*='moddb.com/media/iframe'], iframe[src*='moddb.com/media/embed'], iframe[src*='moddb.com/videos/iframe'], iframe[src*='moddb.com/videos/embed']"; + + /// Selector for video gallery containers and items. + public const string VideoGallerySelector = "#videobox, #videosbrowse, #mediabrowse, .mediarow, .mediabox"; + + /// Selector for video links. + public const string VideoLinkSelector = "a[href*='/videos/'], a[href*='youtube.com/watch'], a[href*='youtu.be/'], a[href*='vimeo.com/']"; /// Selector for video thumbnails. - public const string VideoThumbnailSelector = ".thumbnail img, .preview img"; + public const string VideoThumbnailSelector = ".thumbnail img, .preview img, img"; /// Selector for video titles. - public const string VideoTitleSelector = ".title, h3, h4"; + public const string VideoTitleSelector = ".title, h3, h4, h5, .caption"; + + /// Selector for recommendation and related content sections. + public const string RecommendationsSelector = "#recommendations, .recommendations, #related, .related, #similar, .similar, #fansalsoviewed, .fansalsoviewed, .youmayalso, [class*='recommend'], [id*='recommend'], [class*='similar'], [id*='similar']"; // ===== Images Section Selectors ===== /// Selector for image gallery container. - public const string ImageGallerySelector = ".mediarow, .screenshot, .imagebox, .gallery"; + public const string ImageGallerySelector = "#imagebox, #mediaimage, #imagebrowse, #mediabrowse, .mediarow"; + + /// + /// Selector for gallery images only. Deliberately excludes a blanket + /// img[src*='media.moddb.com'] match, which previously pulled game icons, member + /// avatars, and file-page chrome into the Media tab. + /// + public const string GalleryImageSelector = "#imagebox img, #mediaimage img, #imagebrowse img, #mediabrowse img, .mediarow img, .media .holder img, #downloadsummary img, #downloaddescription img, .preview img, a[href*='/mods/'][href*='/images/'] img"; /// Selector for individual images. public const string ImageSelector = "img"; + /// Sidebar/profile containers whose images are icons and avatars, not gallery media. + public const string ImageSidebarSelector = "#modsinfo, #downloadsprofilemenu, #profile, .sidecolumn, aside"; + /// Selector for image thumbnails. public const string ImageThumbnailSelector = ".thumbnail img, .thumb img"; @@ -197,20 +225,24 @@ public static class ModDBParserConstants // ===== Comments Section Selectors ===== - /// Selector for comments container. - public const string CommentsSelector = ".comment, .post, .comments"; + /// Selector for comments container. Do not use #commentform — that is the composer. + public const string CommentsSelector = "#commentsbrowse"; - /// Selector for individual comment rows. - public const string CommentRowSelector = ".comment, .post"; + /// + /// Selector for posted comment rows. Requires the exact rowcomment class so the + /// composer rows (rowcommentguest, rowcommentsummary, rowcommentemail) + /// and #commentform are not treated as comments. + /// + public const string CommentRowSelector = ".row.rowcomment, .rowcomment"; /// Selector for comment authors. - public const string CommentAuthorSelector = ".author, .username, a[href*='/members/']"; + public const string CommentAuthorSelector = ".author, .username, .heading a, a[href*='/members/']"; - /// Selector for comment content. - public const string CommentContentSelector = ".content, .body, .text"; + /// Selector for comment content. Avoids bare p which matches login chrome and CSS blobs. + public const string CommentContentSelector = ":scope > .commentbody, .commentbody, p.comment"; /// Selector for comment dates. - public const string CommentDateSelector = "time[datetime], .date"; + public const string CommentDateSelector = "time[datetime], time, .date, .datetime, span.subheading"; /// Selector for comment karma/votes. public const string CommentKarmaSelector = ".karma, .votes, .goodkarma, .badkarma"; @@ -245,4 +277,100 @@ public static class ModDBParserConstants /// Pattern for games URLs. public const string GamesUrlPattern = "/games/"; + + // ===== Mod Detail Page Selectors ===== + + /// Selector for the downloads section on mod pages. + public const string DownloadsSectionSelector = "#downloads, .downloads, .files"; + + /// Selector for the addons section on mod pages. + public const string AddonsSectionSelector = "#addons, .addons"; + + /// Selector for the tabs/navigation on mod pages. + public const string TabsSelector = ".tabs, .navigation, nav"; + + /// Selector for individual tab links. + public const string TabLinkSelector = "a[href*='/downloads'], a[href*='/addons']"; + + // ===== Metadata Keys (Internal/Normalized) ===== + + /// Metadata key for filename. + public const string MetadataFilename = "filename"; + + /// Alternative metadata key for filename. + public const string MetadataFileNameAlt = "file name"; + + /// Alternative metadata key for file. + public const string MetadataFileAlt = "file"; + + /// Metadata key for size. + public const string MetadataSize = "size"; + + /// Alternative metadata key for size. + public const string MetadataFileSizeAlt = "file size"; + + /// Metadata key for uploader. + public const string MetadataUploader = "uploader"; + + /// Alternative metadata key for uploaded by. + public const string MetadataUploadedBy = "uploaded by"; + + /// Alternative metadata key for author. + public const string MetadataAuthor = "author"; + + /// Metadata key for category. + public const string MetadataCategory = "category"; + + /// Alternative metadata key for file category. + public const string MetadataFileCategory = "file category"; + + /// Alternative metadata key for type. + public const string MetadataType = "type"; + + /// Metadata key for MD5 hash. + public const string MetadataMd5Hash = "md5 hash"; + + /// Metadata key for MD5 hash (alternative). + public const string MetadataMd5HashAlt = "md5hash"; + + /// Alternative metadata key for MD5 checksum. + public const string MetadataMd5Checksum = "md5 checksum"; + + /// Alternative metadata key for MD5. + public const string MetadataMd5 = "md5"; + + /// Alternative metadata key for hash. + public const string MetadataHash = "hash"; + + /// Alternative metadata key for checksum. + public const string MetadataChecksum = "checksum"; + + /// Metadata key for total downloads. + public const string MetadataTotalDownloads = "total downloads"; + + /// Alternative metadata key for download count. + public const string MetadataDownloadCount = "download count"; + + /// Metadata key for added date. + public const string MetadataAdded = "added"; + + /// Metadata key for updated date. + public const string MetadataUpdated = "updated"; + + // ===== Additional Selectors ===== + + /// Selector for fallback titles (h1, h2, etc). + public const string FallbackTitleSelector = "h2 a, h1 a, h2, h1"; + + /// Selector for file detail page title heading outside the global headerbox. + public const string FilePageTitleSelector = ".midcolumn h2, .columncenter h2, #downloadsfiles h2, #downloadsinfo h2, #downloads h2, .heading h2, .title h2, h2.title, .midcolumn h3, .heading h3"; + + /// Selector for file detail page preview images. + public const string FilePreviewImagesSelector = "#downloadmedia img, #downloadsmedia img, #preview img, #media img, .mediagallery img, .imagebox img, #imagebox img, #downloaddescription img, #downloadsummary img, a[href*='/images/'] img, .previewholder img, .media .holder img"; + + /// Selector for file description container elements. + public const string FileDescriptionContainerSelector = "#downloaddescription, #downloadsummary, #description, .description, .articlebody, #profiletotal"; + + /// Regex pattern for extracting parent mod path. + public const string ParentModPathRegex = @"(/mods/[^/]+)/(?:downloads|addons)/"; } diff --git a/GenHub/GenHub.Core/Helpers/HtmlTextHelper.cs b/GenHub/GenHub.Core/Helpers/HtmlTextHelper.cs new file mode 100644 index 000000000..b19f11d94 --- /dev/null +++ b/GenHub/GenHub.Core/Helpers/HtmlTextHelper.cs @@ -0,0 +1,145 @@ +using System; +using System.Net; +using System.Text.RegularExpressions; + +namespace GenHub.Core.Helpers; + +/// +/// Provides high-performance utilities for stripping HTML tags, decoding HTML entities, +/// and normalizing text descriptions for display across the application. +/// +public static partial class HtmlTextHelper +{ + /// + /// Converts an HTML snippet or formatted description into clean, normalized plain text: + /// - Replaces <br> and block element closures (</p>, </div>, etc.) with line breaks. + /// - Strips all remaining HTML tags. + /// - Decodes HTML entities (e.g., &amp;, &quot;, &gt;, &nbsp;). + /// - Normalizes whitespace and excessive blank lines. + /// - Uses the platform newline format. + /// + /// The raw HTML or formatted text string to normalize. + /// Normalized plain text, or empty string if input is null or whitespace. + public static string NormalizeHtml(string? html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + // 0. Remove script and style elements along with their contents + var text = ScriptTagRegex().Replace(html, string.Empty); + text = StyleTagRegex().Replace(text, string.Empty); + + // 1. Convert
tags to newline + text = BrTagRegex().Replace(text, "\n"); + + // 2. Convert paragraph closing tags to double newline for paragraph separation + text = ParagraphCloseTagRegex().Replace(text, "\n\n"); + + // 3. Convert other block-level closing tags and
tags to newline + text = BlockCloseTagRegex().Replace(text, "\n"); + + // 4. Strip all remaining HTML/XML tags + text = HtmlTagRegex().Replace(text, string.Empty); + + // 5. Decode HTML entities ( , >, ", ', numeric entities, etc.) + text = WebUtility.HtmlDecode(text); + + // 6. Normalize non-breaking spaces and line endings + text = text.Replace('\u00A0', ' ') + .Replace("\r\n", "\n") + .Replace('\r', '\n'); + + // 7. Clean trailing whitespace on lines and collapse excess blank lines + text = TrailingWhitespaceBeforeNewlineRegex().Replace(text, "\n"); + text = ExcessBlankLinesRegex().Replace(text, "\n\n"); + + // 8. Trim and unify with environment newline + text = text.Trim(); + text = text.Replace("\n", Environment.NewLine); + + return text; + } + + /// + /// Converts an HTML snippet or multi-line text into a single-line summary without HTML tags, + /// collapsing all whitespace runs into a single space, and optionally truncating with an ellipsis. + /// + /// The input HTML or text string. + /// Optional maximum character length including ellipsis. + /// A single-line plain text summary. + public static string CleanToSingleLine(string? htmlOrText, int? maxLength = null) + { + if (string.IsNullOrWhiteSpace(htmlOrText)) + { + return string.Empty; + } + + // Strip HTML if tags exist, decode entities, and normalize + var text = NormalizeHtml(htmlOrText); + + // Collapse all newlines, tabs, and multiple spaces into a single space + text = MultiWhitespaceRegex().Replace(text, " ").Trim(); + + if (maxLength.HasValue && maxLength.Value > 0 && text.Length > maxLength.Value) + { + return TruncateWithEllipsis(text, maxLength.Value); + } + + return text; + } + + /// + /// Truncates a string to a specified maximum length and appends an ellipsis ("...") if truncated. + /// + /// The text to truncate. + /// The maximum allowed length (including the ellipsis). + /// The truncated text with an ellipsis if it exceeded maxLength, or the original text. + public static string TruncateWithEllipsis(string? text, int maxLength) + { + if (string.IsNullOrWhiteSpace(text) || maxLength <= 0) + { + return string.Empty; + } + + if (text.Length <= maxLength) + { + return text; + } + + if (maxLength <= 3) + { + return text[..maxLength]; + } + + return string.Concat(text.AsSpan(0, maxLength - 3), "..."); + } + + [GeneratedRegex(@"]*>[\s\S]*?", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ScriptTagRegex(); + + [GeneratedRegex(@"]*>[\s\S]*?", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex StyleTagRegex(); + + [GeneratedRegex(@"", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex BrTagRegex(); + + [GeneratedRegex(@"", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ParagraphCloseTagRegex(); + + [GeneratedRegex(@"]*>", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex BlockCloseTagRegex(); + + [GeneratedRegex(@"]*>", RegexOptions.CultureInvariant)] + private static partial Regex HtmlTagRegex(); + + [GeneratedRegex(@"[ \t]+\n", RegexOptions.CultureInvariant)] + private static partial Regex TrailingWhitespaceBeforeNewlineRegex(); + + [GeneratedRegex(@"(?:\n){3,}", RegexOptions.CultureInvariant)] + private static partial Regex ExcessBlankLinesRegex(); + + [GeneratedRegex(@"\s+", RegexOptions.CultureInvariant)] + private static partial Regex MultiWhitespaceRegex(); +} diff --git a/GenHub/GenHub.Core/Interfaces/Parsers/IWebPageParser.cs b/GenHub/GenHub.Core/Interfaces/Parsers/IWebPageParser.cs index 136018297..2d75645fe 100644 --- a/GenHub/GenHub.Core/Interfaces/Parsers/IWebPageParser.cs +++ b/GenHub/GenHub.Core/Interfaces/Parsers/IWebPageParser.cs @@ -1,3 +1,8 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using GenHub.Core.Models.Parsers; namespace GenHub.Core.Interfaces.Parsers; @@ -36,4 +41,56 @@ public interface IWebPageParser /// Cancellation token. /// A parsed web page with all extracted content sections. Task ParseAsync(string url, string html, CancellationToken cancellationToken = default); + + /// + /// Parses a specific file or item detail page. + /// Default implementation delegates to . + /// + /// The detail page URL. + /// Cancellation token. + /// A parsed web page containing the detailed file information. + Task ParseFileDetailAsync(string url, CancellationToken cancellationToken = default) + => ParseAsync(url, cancellationToken); + + /// + /// Parses multiple file or item detail pages in a batch. + /// Default implementation delegates to . + /// + /// The detail page URLs to parse. + /// Cancellation token. + /// A dictionary mapping each URL to its parsed web page result. + async Task> ParseFileDetailsManyAsync( + IReadOnlyList urls, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(urls); + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var url in urls.Distinct(StringComparer.OrdinalIgnoreCase)) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var page = await ParseFileDetailAsync(url, cancellationToken); + results[url] = page; + } + catch (HttpRequestException) + { + // soft failure per url in batch + } + catch (IOException) + { + // soft failure per url in batch + } + catch (InvalidOperationException) + { + // soft failure per url in batch + } + catch (FormatException) + { + // soft failure per url in batch + } + } + + return results; + } } diff --git a/GenHub/GenHub.Core/Interfaces/Tools/IPlaywrightService.cs b/GenHub/GenHub.Core/Interfaces/Tools/IPlaywrightService.cs index f2007576a..df73f9827 100644 --- a/GenHub/GenHub.Core/Interfaces/Tools/IPlaywrightService.cs +++ b/GenHub/GenHub.Core/Interfaces/Tools/IPlaywrightService.cs @@ -6,6 +6,7 @@ using GenHub.Core.Models.Results; using Microsoft.Playwright; using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -25,6 +26,31 @@ public interface IPlaywrightService /// A new IPage instance. Task CreatePageAsync(BrowserNewContextOptions? options = null, CancellationToken cancellationToken = default); + /// + /// Creates a page in a persistent, headed browser context whose cookies and storage survive + /// across calls. Use this for bot-protected sites (e.g. ModDB's Cloudflare): the user solves + /// the challenge once, the resulting clearance cookie is persisted to disk, and subsequent + /// pages in the same session (and across app restarts, until the cookie expires) load without + /// another challenge. A real browser window is shown while the challenge is pending. + /// + /// The on-disk profile name (scoped under the app data browser-profile root). + /// Cancellation token. + /// A new in the persistent context. + Task CreatePersistentPageAsync(string profileName, CancellationToken cancellationToken = default); + + /// + /// Closes a page from and shuts down the headed Chromium + /// window when no active pages remain. Prefer this over page.CloseAsync alone so + /// callers do not leave an about:blank window open after a successful ModDB scrape. + /// + /// The persistent-context page to close. + /// + /// When , leaves the page open (e.g. so the user can finish a Cloudflare + /// challenge) without closing the browser. + /// + /// A task representing the asynchronous operation. + Task ClosePersistentPageAsync(IPage page, bool keepOpen = false); + /// /// Fetches HTML content from a URL using Playwright. /// @@ -41,6 +67,35 @@ public interface IPlaywrightService /// A parsed AngleSharp IDocument. Task FetchAndParseAsync(string url, CancellationToken cancellationToken = default); + /// + /// Fetches and parses a web page in a persistent, headed browser context whose cookies survive + /// across calls. Use this for bot-protected URLs (e.g. ModDB) so the Cloudflare clearance cookie + /// obtained from a single manual challenge solve is reused. + /// + /// The on-disk profile name (scoped under the app data browser-profile root). + /// The URL to fetch and parse. + /// Cancellation token. + /// A parsed AngleSharp IDocument. + Task FetchAndParsePersistentAsync(string profileName, string url, CancellationToken cancellationToken = default); + + /// + /// Fetches and parses multiple URLs in one persistent headed page — open once, navigate each + /// URL in order, then close. Use this for ModDB section sweeps so Chromium does not spawn a + /// new window per section (and so concurrent NewPage/Close races cannot tear down the context + /// mid-navigation). + /// + /// The on-disk profile name (scoped under the app data browser-profile root). + /// URLs to fetch in order. Duplicates are fetched once; order of first occurrence is kept. + /// Cancellation token. + /// + /// A map of URL → parsed document for every URL that loaded successfully. Failed URLs are omitted; + /// callers should treat a missing key as a soft failure for that section. + /// + Task> FetchAndParsePersistentManyAsync( + string profileName, + IReadOnlyList urls, + CancellationToken cancellationToken = default); + /// /// Downloads a file using Playwright to handle complex scenarios (like anti-bot protections). /// @@ -48,4 +103,27 @@ public interface IPlaywrightService /// Cancellation token. /// A DownloadResult indicating success or failure. Task DownloadFileAsync(DownloadConfiguration configuration, CancellationToken cancellationToken = default); + + /// + /// Executes an operation within a scoped persistent browser context session. + /// The persistent browser window stays open for the duration of the operation and closes + /// immediately when the operation completes, avoiding multiple window launches and idle delays. + /// + /// The return type of the operation. + /// The on-disk profile name. + /// The asynchronous operation to execute. + /// Cancellation token. + /// The result of the operation. + Task ExecuteInPersistentContextAsync( + string profileName, + Func> operation, + CancellationToken cancellationToken = default); + + /// + /// Asynchronously pre-warms the Playwright driver runtime in the background so subsequent + /// browser operations launch with minimal latency. + /// + /// Cancellation token. + /// A task representing the background warmup operation. + Task WarmupAsync(CancellationToken cancellationToken = default); } diff --git a/GenHub/GenHub.Core/Models/ModDB/MapDetails.cs b/GenHub/GenHub.Core/Models/ModDB/MapDetails.cs index 347634147..979071455 100644 --- a/GenHub/GenHub.Core/Models/ModDB/MapDetails.cs +++ b/GenHub/GenHub.Core/Models/ModDB/MapDetails.cs @@ -37,4 +37,4 @@ public record MapDetails( string? FileType = null, float? Rating = null, string? RefererUrl = null, - List? AdditionalFiles = null); + List? AdditionalFiles = null); diff --git a/GenHub/GenHub.Core/Models/ModDB/ModDBCategoryMapper.cs b/GenHub/GenHub.Core/Models/ModDB/ModDBCategoryMapper.cs index cf2fc5920..c0f1e9a1f 100644 --- a/GenHub/GenHub.Core/Models/ModDB/ModDBCategoryMapper.cs +++ b/GenHub/GenHub.Core/Models/ModDB/ModDBCategoryMapper.cs @@ -19,8 +19,8 @@ public static ContentType MapCategory(string? categoryCode) // Releases (Mods) "2" => ContentType.Mod, // Full Version "3" => ContentType.Mod, // Demo - "4" => ContentType.Patch, // Patch - "28" => ContentType.Patch, // Script + "4" => ContentType.Mod, // Patch (mod release/update) + "28" => ContentType.Mod, // Script (mod script/release) "29" => ContentType.Addon, // Trainer // Media @@ -60,11 +60,11 @@ public static ContentType MapCategory(string? categoryCode) "131" => ContentType.Addon, // Model Pack // Addons - Skins - "112" => ContentType.Skin, // Player Skin - "133" => ContentType.Skin, // Prop Skin - "113" => ContentType.Skin, // Vehicle Skin - "114" => ContentType.Skin, // Weapon Skin - "134" => ContentType.Skin, // Skin Pack + "112" => ContentType.Addon, // Player Skin + "133" => ContentType.Addon, // Prop Skin + "113" => ContentType.Addon, // Vehicle Skin + "114" => ContentType.Addon, // Weapon Skin + "134" => ContentType.Addon, // Skin Pack // Addons - Audio "117" => ContentType.Addon, // Music @@ -75,8 +75,8 @@ public static ContentType MapCategory(string? categoryCode) // Addons - Graphics "124" => ContentType.Addon, // Decal "136" => ContentType.Addon, // Effects GFX - "125" => ContentType.Skin, // GUI - "126" => ContentType.Skin, // HUD + "125" => ContentType.Addon, // GUI + "126" => ContentType.Addon, // HUD "128" => ContentType.Addon, // Sprite "129" => ContentType.Addon, // Texture @@ -103,10 +103,15 @@ public static ContentType MapCategoryByName(string? categoryName) { var s when s.Contains("full version") => ContentType.Mod, var s when s.Contains("demo") => ContentType.Mod, - var s when s.Contains("patch") => ContentType.Patch, - var s when s.Contains("script") => ContentType.Patch, + var s when s.Contains("patch") => ContentType.Mod, + var s when s.Contains("script") => ContentType.Mod, var s when s.Contains("trainer") => ContentType.Addon, + var s when s.Contains("tool") => ContentType.ModdingTool, + var s when s.Contains("sdk") => ContentType.ModdingTool, + var s when s.Contains("ide") => ContentType.ModdingTool, + var s when s.Contains("source code") => ContentType.ModdingTool, + var s when s.Contains("trailer") => ContentType.Video, var s when s.Contains("movie") => ContentType.Video, var s when s.Contains("video") => ContentType.Video, @@ -116,17 +121,12 @@ var s when s.Contains("singleplayer map") => ContentType.Map, var s when s.Contains("map") => ContentType.Map, var s when s.Contains("prefab") => ContentType.Map, - var s when s.Contains("skin") => ContentType.Skin, - var s when s.Contains("gui") => ContentType.Skin, - var s when s.Contains("hud") => ContentType.Skin, + var s when s.Contains("skin") => ContentType.Addon, + var s when s.Contains("gui") => ContentType.Addon, + var s when s.Contains("hud") => ContentType.Addon, var s when s.Contains("language") => ContentType.LanguagePack, - var s when s.Contains("tool") => ContentType.ModdingTool, - var s when s.Contains("sdk") => ContentType.ModdingTool, - var s when s.Contains("ide") => ContentType.ModdingTool, - var s when s.Contains("source code") => ContentType.ModdingTool, - _ => ContentType.Addon, }; } diff --git a/GenHub/GenHub.Core/Models/ModDB/ModDBFilter.cs b/GenHub/GenHub.Core/Models/ModDB/ModDBFilter.cs index a977a5f08..88232974f 100644 --- a/GenHub/GenHub.Core/Models/ModDB/ModDBFilter.cs +++ b/GenHub/GenHub.Core/Models/ModDB/ModDBFilter.cs @@ -1,3 +1,7 @@ +using System; +using System.Collections.Generic; +using GenHub.Core.Constants; + namespace GenHub.Core.Models.ModDB; /// @@ -21,7 +25,7 @@ public class ModDBFilter public string? Licence { get; set; } /// Gets or sets the sort parameter. - public string? Sort { get; set; } + public string? Sort { get; set; } = ModDBConstants.DefaultSort; /// Gets or sets the page number (1-based). public int Page { get; set; } = 1; diff --git a/GenHub/GenHub.Core/Models/Parsers/Comment.cs b/GenHub/GenHub.Core/Models/Parsers/Comment.cs index 645e1fd5a..dd1532318 100644 --- a/GenHub/GenHub.Core/Models/Parsers/Comment.cs +++ b/GenHub/GenHub.Core/Models/Parsers/Comment.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; + namespace GenHub.Core.Models.Parsers; /// @@ -8,9 +11,13 @@ namespace GenHub.Core.Models.Parsers; /// The comment date (optional). /// The karma/vote score (optional). /// Whether the comment is from the content creator (optional). +/// Indentation depth level for reply threads (optional). +/// Child replies to this comment (optional). public record Comment( string? Author = null, string? Content = null, DateTime? Date = null, int? Karma = null, - bool? IsCreator = null) : ContentSection(SectionType.Comment, "Comment"); + bool? IsCreator = null, + int IndentLevel = 0, + IReadOnlyList? Replies = null) : ContentSection(SectionType.Comment, "Comment"); diff --git a/GenHub/GenHub.Core/Models/Parsers/DownloadableFile.cs b/GenHub/GenHub.Core/Models/Parsers/DownloadableFile.cs new file mode 100644 index 000000000..0df14e67a --- /dev/null +++ b/GenHub/GenHub.Core/Models/Parsers/DownloadableFile.cs @@ -0,0 +1,42 @@ +namespace GenHub.Core.Models.Parsers; + +/// +/// Represents a downloadable file extracted from a web page. +/// +/// The file name. +/// The file version (optional). +/// File size in bytes (optional). +/// Human-readable file size (optional). +/// The upload date (optional). +/// The file category (optional). +/// The uploader name (optional). +/// The download URL (optional). +/// The MD5 hash of the file (optional). +/// Number of comments (optional). +/// The thumbnail image URL (optional). +/// Number of downloads (optional). +/// The file section type (Downloads or Addons). +/// The release date (optional, may differ from upload date). +/// The web page details URL (optional). +/// The full description or release notes (optional). +/// List of preview image URLs (optional). +/// The actual file archive name (optional). +public record DownloadableFile( + string Name, + string? Version = null, + long? SizeBytes = null, + string? SizeDisplay = null, + DateTime? UploadDate = null, + string? Category = null, + string? Uploader = null, + string? DownloadUrl = null, + string? Md5Hash = null, + int? CommentCount = null, + string? ThumbnailUrl = null, + int? DownloadCount = null, + FileSectionType FileSectionType = FileSectionType.Downloads, + DateTime? ReleaseDate = null, + string? DetailsUrl = null, + string? Description = null, + System.Collections.Generic.IReadOnlyList? PreviewImages = null, + string? Filename = null) : ContentSection(SectionType.File, Name); diff --git a/GenHub/GenHub.Core/Models/Parsers/FileSectionType.cs b/GenHub/GenHub.Core/Models/Parsers/FileSectionType.cs new file mode 100644 index 000000000..9fddb1afb --- /dev/null +++ b/GenHub/GenHub.Core/Models/Parsers/FileSectionType.cs @@ -0,0 +1,13 @@ +namespace GenHub.Core.Models.Parsers; + +/// +/// Represents the type of file section, distinguishing between main releases and addon files. +/// +public enum FileSectionType +{ + /// Files from the main releases/downloads section. + Downloads, + + /// Files from the addons section. + Addons, +} diff --git a/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/ModDB/ModDBCategoryMapperTests.cs b/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/ModDB/ModDBCategoryMapperTests.cs new file mode 100644 index 000000000..73e18587f --- /dev/null +++ b/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/ModDB/ModDBCategoryMapperTests.cs @@ -0,0 +1,63 @@ +using GenHub.Core.Models.Enums; +using GenHub.Core.Models.ModDB; +using Xunit; +using ContentType = GenHub.Core.Models.Enums.ContentType; + +namespace GenHub.Tests.Core.Features.Content.ModDB; + +/// +/// Unit tests for . +/// +public class ModDBCategoryMapperTests +{ + /// + /// Verifies that MapCategory maps ModDB category codes correctly, especially mapping patches and scripts to Mod. + /// + /// The category code to map. + /// The expected content type. + [Theory] + [InlineData("2", ContentType.Mod)] + [InlineData("3", ContentType.Mod)] + [InlineData("4", ContentType.Mod)] + [InlineData("28", ContentType.Mod)] + [InlineData("29", ContentType.Addon)] + [InlineData("7", ContentType.Video)] + [InlineData("8", ContentType.Video)] + [InlineData("101", ContentType.Map)] + [InlineData("102", ContentType.Map)] + [InlineData("112", ContentType.Addon)] + [InlineData("125", ContentType.Addon)] + [InlineData("126", ContentType.Addon)] + [InlineData("20", ContentType.ModdingTool)] + [InlineData("30", ContentType.LanguagePack)] + public void MapCategory_MapsCategoryCodesCorrectly(string categoryCode, ContentType expected) + { + var result = ModDBCategoryMapper.MapCategory(categoryCode); + Assert.Equal(expected, result); + } + + /// + /// Verifies that MapCategoryByName maps category names correctly, mapping patch and script names to Mod. + /// + /// The category name to map. + /// The expected content type. + [Theory] + [InlineData("Full Version", ContentType.Mod)] + [InlineData("Demo", ContentType.Mod)] + [InlineData("Patch", ContentType.Mod)] + [InlineData("v1.01 Patch", ContentType.Mod)] + [InlineData("Script", ContentType.Mod)] + [InlineData("Multiplayer Map", ContentType.Map)] + [InlineData("Singleplayer Map", ContentType.Map)] + [InlineData("Player Skin", ContentType.Addon)] + [InlineData("GUI", ContentType.Addon)] + [InlineData("HUD", ContentType.Addon)] + [InlineData("Mapping Tool", ContentType.ModdingTool)] + [InlineData("Language Pack", ContentType.LanguagePack)] + [InlineData("Trailer", ContentType.Video)] + public void MapCategoryByName_MapsNamesCorrectly(string categoryName, ContentType expected) + { + var result = ModDBCategoryMapper.MapCategoryByName(categoryName); + Assert.Equal(expected, result); + } +} diff --git a/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/Parsers/ModDBPageParserTests.cs b/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/Parsers/ModDBPageParserTests.cs new file mode 100644 index 000000000..a630b8b5c --- /dev/null +++ b/GenHub/GenHub.Tests/GenHub.Tests.Core/Features/Content/Parsers/ModDBPageParserTests.cs @@ -0,0 +1,1691 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AngleSharp; +using AngleSharp.Dom; +using GenHub.Core.Constants; +using GenHub.Core.Interfaces.Tools; +using GenHub.Core.Models.Parsers; +using GenHub.Features.Content.Services.Parsers; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace GenHub.Tests.Core.Features.Content.Parsers; + +/// +/// Regression tests for the current ModDB detail markup and Cloudflare-aware section loading. +/// +public sealed class ModDBPageParserTests +{ + /// + /// Verifies the current game-addon detail page maps its metadata and /addons/start route into + /// a usable file rather than returning an empty download URL. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_CurrentAddonDetailMarkup_ExtractsArchiveNameAndAddonStartUrlAsync() + { + // Arrange + var playwright = CreatePlaywrightMock(); + var pageUrl = "https://www.moddb.com/games/cc-generals-zero-hour/addons/lemuria-2026-fixes"; + var doc = await CreateDocumentAsync(""" + +
+
Filename
Lemuria_2026_Fixes.rar
+ +
Uploader
BagaturKhan
+
Added
+
Size
1.07mb (1,125,450 bytes)
+ +
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var file = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("Lemuria_2026_Fixes.rar", file.Name); + Assert.Equal("https://www.moddb.com/addons/start/302328", file.DownloadUrl); + Assert.Equal("Singleplayer Map", file.Category); + Assert.Equal(1_125_450, file.SizeBytes); + } + + /// + /// Game-scoped FileDetail URLs (from the ModDB downloads listing) have no parent /mods/ page + /// to sweep, so the detail view must still populate Community from comments on the file page + /// itself instead of leaving only a single Releases row. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_GameFileDetail_ExtractsOnPageCommentsWithoutParentSweepAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/games/cc-generals-zero-hour/downloads/genbigeditbig-editor"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
+
Filename
GenBigEdit.zip
+
Size
174.33mb (182,801,143 bytes)
+ +
+
+
+
mah_boi May 30 2026
+
Please, provide us the source code of this program.
+
+
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + Assert.Equal(PageType.FileDetail, parsed.PageType); + var file = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("GenBigEdit.zip", file.Name); + Assert.Equal("https://www.moddb.com/downloads/start/310120", file.DownloadUrl); + + var comment = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("mah_boi", comment.Author); + Assert.Equal("Please, provide us the source code of this program.", comment.Content); + + // Must not attempt a parent-mod section sweep for /games/... FileDetail URLs (fetches only the single URL). + playwright.Verify( + service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.Is>(urls => urls.Count == 1 && urls[0] == pageUrl), + It.IsAny()), + Times.Once); + } + + /// + /// Verifies an addons-list row retains its ModDB category so a map does not become a generic + /// add-on later in the resolver and manifest pipeline. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_AddonsListRow_ExtractsSingleplayerMapCategoryAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/games/cc-generals-zero-hour/addons"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
+

Lemuria 2026

+ Singleplayer Map + 1.07 MB + Download +
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var file = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("Singleplayer Map", file.Category); + Assert.Equal(FileSectionType.Addons, file.FileSectionType); + Assert.Equal("https://www.moddb.com/addons/start/302328", file.DownloadUrl); + } + + /// + /// Verifies that rich ModDB sections use the verified persistent Chromium profile instead of + /// a separate headless browser that loses Cloudflare clearance. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_ModDetail_UsesPersistentProfileForDownloadsAndAddonsAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/example-mod"; + var documents = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [pageUrl] = await CreateDocumentAsync("

Example Mod

"), + [pageUrl + "/downloads"] = await CreateDocumentAsync(""" +

Example Release

4 MBDownload
+ """), + [pageUrl + "/addons"] = await CreateDocumentAsync(""" +

Example Addon

2 MBDownload
+ """), + [pageUrl + "/videos"] = await CreateDocumentAsync(""), + [pageUrl + "/images"] = await CreateDocumentAsync(""), + [pageUrl + "/reviews"] = await CreateDocumentAsync(""), + [pageUrl + "/articles"] = await CreateDocumentAsync(""), + }; + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .Returns((string _, IReadOnlyList urls, CancellationToken _) => + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var url in urls) + { + if (documents.TryGetValue(url, out var d)) + { + result[url] = d; + } + } + + return Task.FromResult>(result); + }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var files = parsed.Sections.OfType().ToList(); + Assert.Contains(files, file => file.Name == "Example Release" && file.DownloadUrl == "https://www.moddb.com/downloads/start/100"); + Assert.Contains(files, file => file.Name == "Example Addon" && file.DownloadUrl == "https://www.moddb.com/addons/start/200"); + playwright.Verify( + service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.Is>(urls => + urls.Contains(pageUrl) && urls.Contains(pageUrl + "/downloads") && urls.Contains(pageUrl + "/addons")), + It.IsAny()), + Times.Once); + playwright.Verify(service => service.FetchAndParseAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + /// + /// Verifies the file-only acquisition path resolves a FileDetail download without fetching the + /// parent mod's downloads/addons/videos/images/reviews/articles sections (the seven-page sweep + /// that previously fired on every card download). + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseFileDetailAsync_FetchesOnlyFileDetailPageAndSkipsSectionSweepAsync() + { + // Arrange: the FileDetail page already carries a real (non-guest) icon, so the parent-mod + // icon fallback fetch is skipped too — exactly one fetch total. + const string pageUrl = "https://www.moddb.com/mods/genspeed/downloads/genspeed-v25"; + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentAsync( + ModDBConstants.BrowserProfileName, + pageUrl, + It.IsAny())) + .ReturnsAsync(await CreateDocumentAsync(""" + +

GenSpeed v2.5

+
+
+
Filename
GenSpeed-v2.5.zip
+
Size
65.04mb (68,197,650 bytes)
+ +
+ + """)); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseFileDetailAsync(pageUrl); + + // Assert: exactly one DownloadableFile, no section sweep, icon from the FileDetail page. + var file = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("GenSpeed v2.5", file.Name); + Assert.Equal("GenSpeed-v2.5.zip", file.Filename); + Assert.Equal("https://www.moddb.com/downloads/start/311183", file.DownloadUrl); + Assert.Equal(68_197_650, file.SizeBytes); + Assert.Equal("https://static.moddb.com/mods/genspeed/icon.png", parsed.Context.IconUrl); + + playwright.Verify( + service => service.FetchAndParsePersistentAsync( + ModDBConstants.BrowserProfileName, + It.Is(url => url != pageUrl), + It.IsAny()), + Times.Never); + } + + /// + /// Verifies that ParseFileDetailAsync performs only a single page fetch for file details without + /// secondary parent mod fetches or section sweeps. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseFileDetailAsync_WithGuestIcon_FetchesOnlyFileDetailPageAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/genspeed/downloads/genspeed-v25"; + var fetchedUrls = new List(); + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentAsync( + ModDBConstants.BrowserProfileName, + It.IsAny(), + It.IsAny())) + .Returns((string _, string url, CancellationToken _) => + { + fetchedUrls.Add(url); + return Task.FromResult(CreateDocumentAsync(""" + +

GenSpeed v2.5

+
+
+
Filename
GenSpeed-v2.5.zip
+ +
+ + """).GetAwaiter().GetResult()); + }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseFileDetailAsync(pageUrl); + + // Assert: exactly one fetch (FileDetail), never parent mod or section pages. + Assert.Equal(new[] { pageUrl }, fetchedUrls); + Assert.Contains(parsed.Sections.OfType(), f => f.Filename == "GenSpeed-v2.5.zip"); + } + + /// + /// Verifies that comment parsing creates nested reply threads with correct author attribution + /// and cleans out ModDB action text like 'Reply Good karma Bad karma+1 vote'. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_NestedComments_ParsesThreadHierarchyAndCleansActionTextAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/example-mod/comments"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
+
+
Scorpionwins Jul 31 2026
+
+ How to activate additional weapons? +
Reply Good karma Bad karma+1 vote
+
+
+
+
BagaturKhan Jul 31 2026
+
+ If you are talking about stolen tech, train your infiltrator. +
Reply Good karma Bad karma+1 vote
+
+
+
+
+
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var topLevelComments = parsed.Sections.OfType().ToList(); + var parentComment = Assert.Single(topLevelComments); + Assert.Equal("Scorpionwins", parentComment.Author); + Assert.Equal("How to activate additional weapons?", parentComment.Content); + Assert.Equal(0, parentComment.IndentLevel); + + var reply = Assert.Single(parentComment.Replies!); + Assert.Equal("BagaturKhan", reply.Author); + Assert.Equal("If you are talking about stolen tech, train your infiltrator.", reply.Content); + Assert.Equal(1, reply.IndentLevel); + } + + /// + /// Verifies reply markup nested inside .commentbody does not inflate the parent content + /// into a huge whitespace block (the layout bug seen in the Community tab). + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_NestedCommentsInsideCommentBody_DoesNotPolluteParentContentAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/example-mod/comments"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
+
+
Scorpionwins Jul 31 2026
+
+ + + How to activate additional weapons? +
Reply Good karma Bad karma+1 vote
+
+
+
BagaturKhan Jul 31 2026
+
+ Train your infiltrator. +
Reply
+
+
+
+
+
+
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var parentComment = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("How to activate additional weapons?", parentComment.Content); + Assert.DoesNotContain("BagaturKhan", parentComment.Content); + Assert.DoesNotContain("infiltrator", parentComment.Content, StringComparison.OrdinalIgnoreCase); + + var reply = Assert.Single(parentComment.Replies!); + Assert.Equal("BagaturKhan", reply.Author); + Assert.Equal("Train your infiltrator.", reply.Content); + } + + /// + /// Verifies rating widgets without author/body are not surfaced as empty Community review cards. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_BareRatingWidget_IsNotTreatedAsReviewAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/example-mod/reviews"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
9.0people found this helpful
+
+ Alice +
Solid patch for ROTR.
+ 8.5 +
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var review = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("Alice", review.Author); + Assert.Equal("Solid patch for ROTR.", review.Content); + } + + /// + /// The live ModDB composer (#commentform plus guest/email rows and injected CSS) must + /// not appear as Community comments. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_CommentComposer_IsNotTreatedAsCommentsAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/downloads/cc-generals-undone"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +

C&C Generals Undone file

+

C&C Generals Undone

+
+
Filename
GeneralsUndone_v1.0.zip
+ +
+
+
+
+

Your comment will be anonymous unless you join the community. +

+
+
+
+
+
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + Assert.Empty(parsed.Sections.OfType()); + } + + /// + /// File-page chrome (game icon, developer avatar, download title art) must not appear in Media. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_FileDetailChromeImages_AreNotGalleryMediaAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/downloads/cc-generals-undone"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +

C&C Generals Undone

+
+
Filename
GeneralsUndone_v1.0.zip
+ +
+ + C&C Generals Undone + + + + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + Assert.Empty(parsed.Sections.OfType()); + } + + /// + /// The images tab should yield unique gallery shots, not share icons or duplicate featured thumbs. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_ImagesPage_ExtractsUniqueGalleryShotsAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/images"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + +
+ + View media + + + View media + +
+ ICBM POWTruck +
+ Share on Facebook +
+ + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + var images = parsed.Sections.OfType().ToList(); + Assert.Equal(2, images.Count); + Assert.Contains(images, image => image.Title.Contains("ICBM", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(images, image => image.Title.Contains("Spectre", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(images, image => image.Title.Contains("Share", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(images, image => image.ThumbnailUrl?.StartsWith("data:", StringComparison.OrdinalIgnoreCase) == true); + Assert.All(images, image => Assert.DoesNotContain("crop_", image.ThumbnailUrl ?? string.Empty)); + Assert.All(images, image => Assert.DoesNotContain("/cache/", image.ThumbnailUrl ?? string.Empty)); + } + + /// + /// Image titles with CamelCase or raw filenames should be formatted with clean spaces. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_ImageTitles_FormatsCamelCaseAndFilenamesAsync() + { + const string pageUrl = "https://www.moddb.com/mods/test-mod/images"; + var playwright = CreatePlaywrightMock(); + var doc = await CreateDocumentAsync(""" + + + + """); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + var images = parsed.Sections.OfType().ToList(); + Assert.Equal(2, images.Count); + Assert.Equal("Life Of BRRRRTTT", images[0].Title); + Assert.Equal("BASSBASSBASSASS", images[1].Title); + Assert.Equal("https://media.moddb.com/images/mods/1/73/72174/LifeOfBRRRRTTT.png", images[0].ThumbnailUrl); + Assert.Equal("https://media.moddb.com/images/mods/1/73/72174/LifeOfBRRRRTTT.png", images[0].FullSizeUrl); + } + + /// + /// FileDetail filename plus the parent downloads listing of the same start URL must collapse + /// to one release, keeping the human listing name. + /// + /// A task representing the asynchronous test. + [Fact] + public async Task ParseAsync_FileDetailAndParentDownloads_DedupesSameBinaryAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/downloads/cc-generals-undone"; + const string parentUrl = "https://www.moddb.com/mods/cc-generals-undone"; + var documents = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [pageUrl] = await CreateDocumentAsync(""" + +

C&C Generals Undone file

+

C&C Generals Undone

+ register + + Games : C&C: Generals Zero Hour : Mods : C&C Generals Undone : Files +

This is the first version of Undone, and I know it's still very much in development.

+
+
Filename
GeneralsUndone_v1.0.zip
+ +
+ + """), + [parentUrl] = await CreateDocumentAsync("

C&C Generals Undone

"), + [parentUrl + "/downloads"] = await CreateDocumentAsync(""" +
+

C&C Generals Undone

+ 289.6 MB + Download +
+
+

Generals Undone v1.01 Patch

+ 1 MB + Download +
+ """), + [parentUrl + "/addons"] = await CreateDocumentAsync(""), + [parentUrl + "/videos"] = await CreateDocumentAsync(""), + [parentUrl + "/images"] = await CreateDocumentAsync(""), + [parentUrl + "/reviews"] = await CreateDocumentAsync(""), + [parentUrl + "/articles"] = await CreateDocumentAsync(""), + }; + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .Returns((string _, IReadOnlyList urls, CancellationToken _) => + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var url in urls) + { + if (documents.TryGetValue(url, out var doc)) + { + result[url] = doc; + } + } + + return Task.FromResult>(result); + }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + var files = parsed.Sections.OfType().ToList(); + Assert.Equal(2, files.Count); + Assert.Contains(files, file => file.Name == "C&C Generals Undone" && file.DownloadUrl == "https://www.moddb.com/downloads/start/313719"); + Assert.Contains(files, file => file.Name == "Generals Undone v1.01 Patch"); + Assert.DoesNotContain(files, file => file.Name == "GeneralsUndone_v1.0.zip"); + Assert.Equal("C&C Generals Undone", parsed.Context.Title); + Assert.Equal("WhiteSkull#9044", parsed.Context.Developer); + Assert.Contains("first version of Undone", parsed.Context.Description, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Games :", parsed.Context.Description, StringComparison.Ordinal); + } + + /// + /// Verifies that ParseFileDetailAsync correctly parses metadata when ModDB uses alternative label names + /// such as "File Name", "File Size", "Uploaded By", "MD5 Checksum", and "Total Downloads". + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseFileDetailAsync_WithAlternativeLabels_ParsesMd5ChecksumTotalDownloadsAndUploaderAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/downloads/generals-undone-v101-patch"; + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentAsync( + ModDBConstants.BrowserProfileName, + pageUrl, + It.IsAny())) + .ReturnsAsync(await CreateDocumentAsync(""" + +
+
File Name
GeneralsUndone_v1.01.csf
+
Category
Patch
+
Uploaded By
WhiteSkull#9044
+
File Size
289.6mb (303,663,235 bytes)
+
MD5 Checksum
6e5b1fd58fc7a58cf21af86933116942
+
Total Downloads
185
+ +
+ + """)); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseFileDetailAsync(pageUrl); + + var file = Assert.Single(parsed.Sections.OfType()); + Assert.Equal("GeneralsUndone_v1.01.csf", file.Filename); + Assert.Equal("Patch", file.Category); + Assert.Equal("WhiteSkull#9044", file.Uploader); + Assert.Equal(303_663_235, file.SizeBytes); + Assert.Equal("6e5b1fd58fc7a58cf21af86933116942", file.Md5Hash); + Assert.Equal(185, file.DownloadCount); + Assert.Equal("https://www.moddb.com/downloads/start/313720", file.DownloadUrl); + } + + /// + /// Verifies that ModDB download listing rows with subheading metadata (size in subheading, button class) + /// extract size, category, uploader, details URL, and download URL correctly. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_ModernModDBDownloadsListing_ExtractsSubheadingSizeAndLinksAsync() + { + const string pageUrl = "https://www.moddb.com/mods/cc-generals-undone/downloads/cc-generals-undone"; + const string parentUrl = "https://www.moddb.com/mods/cc-generals-undone"; + var documents = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [pageUrl] = await CreateDocumentAsync(""" + +

C&C Generals Undone file

+

C&C Generals Undone

+ +

First release of Undone.

+
+
Filename
GeneralsUndone_v1.0.zip
+
Category
Full Version
+
Size
289.6mb (303,663,235 bytes)
+
MD5 Hash
6e5b3fcf30fc7a58ef21af869551bb942
+ +
+ + """), + [parentUrl] = await CreateDocumentAsync("

C&C Generals Undone

"), + [parentUrl + "/downloads"] = await CreateDocumentAsync(""" +
+
+

C&C Generals Undone

+ - Full Version, 289.6mb +
+
+ Download +
+
+
+
+

Generals Undone v1.01 Patch

+ - Patch, 1 MB +
+
+ Download +
+
+ """), + [parentUrl + "/addons"] = await CreateDocumentAsync(""), + [parentUrl + "/videos"] = await CreateDocumentAsync(""), + [parentUrl + "/images"] = await CreateDocumentAsync(""), + [parentUrl + "/reviews"] = await CreateDocumentAsync(""), + [parentUrl + "/articles"] = await CreateDocumentAsync(""), + }; + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .Returns((string _, IReadOnlyList urls, CancellationToken _) => + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var url in urls) + { + if (documents.TryGetValue(url, out var doc)) + { + result[url] = doc; + } + } + + return Task.FromResult>(result); + }); + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + var parsed = await parser.ParseAsync(pageUrl); + + var files = parsed.Sections.OfType().ToList(); + Assert.Equal(2, files.Count); + + var mainRelease = Assert.Single(files, f => f.Name == "C&C Generals Undone"); + Assert.Equal("GeneralsUndone_v1.0.zip", mainRelease.Filename); + Assert.Equal("https://www.moddb.com/downloads/start/313719", mainRelease.DownloadUrl); + Assert.Equal("https://www.moddb.com/mods/cc-generals-undone/downloads/cc-generals-undone", mainRelease.DetailsUrl); + Assert.Equal("Full Version", mainRelease.Category); + Assert.Equal(303_663_235, mainRelease.SizeBytes); + Assert.Equal("6e5b3fcf30fc7a58ef21af869551bb942", mainRelease.Md5Hash); + + var patchRelease = Assert.Single(files, f => f.Name == "Generals Undone v1.01 Patch"); + Assert.Equal("https://www.moddb.com/mods/cc-generals-undone/downloads/generals-undone-v101-patch", patchRelease.DetailsUrl); + Assert.Equal("Patch", patchRelease.Category); + Assert.Equal(1048576, patchRelease.SizeBytes); + Assert.Equal("1 MB", patchRelease.SizeDisplay); + } + + /// + /// Verifies that embedded YouTube iframes on mod pages have their title, thumbnail, platform, + /// and normalized embed URL properly extracted. + /// + /// A task that represents the asynchronous test. + [Fact] + public async Task ParseAsync_WithYouTubeIframe_ExtractsTitlePlatformThumbnailAndEmbedUrlAsync() + { + // Arrange + const string pageUrl = "https://www.moddb.com/mods/korean-war-2"; + var doc = await CreateDocumentAsync(""" + +

Korean War 2

+
+ +
+
+

Gameplay Teaser

+ +
+ + """); + var playwright = CreatePlaywrightMock(); + playwright + .Setup(service => service.FetchAndParsePersistentManyAsync( + ModDBConstants.BrowserProfileName, + It.IsAny>(), + It.IsAny())) + .ReturnsAsync(new Dictionary(StringComparer.OrdinalIgnoreCase) { [pageUrl] = doc }); + + var parser = new ModDBPageParser(playwright.Object, new Mock>().Object); + + // Act + var parsed = await parser.ParseAsync(pageUrl); + + // Assert + var videos = parsed.Sections.OfType