feat(content): add ModDB HTML page parser and Playwright headless service

GenHub/GenHub.Core/Constants/ContentConstants.cs

@@ -94,4 +94,31 @@ public static class ContentConstants
     /// Maximum allowed size for the content catalog in bytes (10 MB).
     /// </summary>
     public const long MaxCatalogSizeBytes = 10 * ConversionConstants.BytesPerMegabyte;
+
+    /// <summary>
+    /// Shared resolver/display metadata key for map player counts.
+    /// Builtin and catalog publishers should set this so download cards can render a consistent badge.
+    /// </summary>
+    public const string PlayerCountMetadataKey = "playerCount";
+
+    /// <summary>
+    /// Shared resolver/display metadata key for content categories (AOA, Compstomp, ModDB category, etc.).
+    /// </summary>
+    public const string CategoryMetadataKey = "category";
+
+    /// <summary>
+    /// Display metadata key for a comma-separated list of included/required content names
+    /// (e.g. catalog ContentBundle dependencies resolved to friendly titles).
+    /// </summary>
+    public const string IncludesSummaryMetadataKey = "includesSummary";
+
+    /// <summary>
+    /// Number of recent releases and addons to eagerly preload extended details for.
+    /// </summary>
+    public const int PreloadRecentItemsLimit = 5;
+
+    /// <summary>
+    /// Maximum concurrent background requests when preloading recent item details.
+    /// </summary>
+    public const int PreloadConcurrencyLimit = 3;
 }

GenHub/GenHub.Core/Constants/DirectoryNames.cs

@@ -59,4 +59,14 @@ public static class DirectoryNames
     /// Directory for storing tool workspaces.
     /// </summary>
     public const string ToolWorkspaces = "ToolWorkspaces";
+
+    /// <summary>
+    /// Directory for persistent Playwright browser profiles (cookies/storage for bot-protected sites).
+    /// </summary>
+    public const string BrowserProfiles = "BrowserProfiles";
+
+    /// <summary>
+    /// Directory for the app-owned Playwright Chromium runtime (not the system Chrome/Edge install).
+    /// </summary>
+    public const string BrowserRuntime = "BrowserRuntime";
 }

GenHub/GenHub.Core/Constants/ModDBConstants.cs

@@ -72,6 +72,12 @@ public static class ModDBConstants
     /// <summary>ModDB website URL.</summary>
     public const string PublisherWebsite = BaseUrl;
 
+    /// <summary>
+    /// On-disk Playwright browser profile name used to persist the Cloudflare clearance cookie so
+    /// the user only solves the bot challenge once per session (and across restarts until expiry).
+    /// </summary>
+    public const string BrowserProfileName = "moddb";
+
     /// <summary>Short description for publisher card display.</summary>
     public const string ShortDescription = "Community mods, maps, and content from ModDB";
 
@@ -184,6 +190,29 @@ public static class ModDBConstants
     /// <summary>Value for filter parameter when enabled.</summary>
     public const string FilterEnabledValue = "t";
 
+    // ===== Sort Values =====
+
+    /// <summary>Sort: Date descending (newest first).</summary>
+    public const string SortDateDesc = "date-desc";
+
+    /// <summary>Sort: Date ascending (oldest first).</summary>
+    public const string SortDateAsc = "date-asc";
+
+    /// <summary>Sort: Visits / Popularity descending.</summary>
+    public const string SortVisitDesc = "visit-desc";
+
+    /// <summary>Sort: Rating descending.</summary>
+    public const string SortRatingDesc = "rating-desc";
+
+    /// <summary>Sort: Name ascending (A-Z).</summary>
+    public const string SortNameAsc = "name-asc";
+
+    /// <summary>Sort: Name descending (Z-A).</summary>
+    public const string SortNameDesc = "name-desc";
+
+    /// <summary>Default sort value for ModDB searches and listings (newest first).</summary>
+    public const string DefaultSort = SortDateDesc;
+
     // ===== Category Values =====
 
     // Downloads Section - Releases
@@ -356,6 +385,34 @@ public static class ModDBConstants
     /// <summary>Metadata key for original category.</summary>
     public const string OriginalCategoryMetadataKey = "moddbCategory";
 
+    /// <summary>Metadata key for identifying if content is a mod.</summary>
+    public const string IsModMetadataKey = "IsMod";
+
+    /// <summary>Metadata key for parent mod URL.</summary>
+    public const string ParentModUrlMetadataKey = "ParentModUrl";
+
+    // ===== Playwright / Scraping Constants =====
+
+    /// <summary>Default timeout for page navigation (ms).</summary>
+    public const int DefaultGotoTimeout = 30000;
+
+    /// <summary>
+    /// Default timeout for waiting for a selector (ms). ModDB sits behind Cloudflare; the headed
+    /// browser persistent profile usually receives the clearance cookie after verification, but 15 s gives a safe margin
+    /// for manual challenge solves before the scraper parses whatever it has.
+    /// </summary>
+    public const int DefaultSelectorTimeout = 15000;
+
+    /// <summary>
+    /// How long (ms) the listing scrape waits for the user to solve a Cloudflare challenge in the
+    /// visible browser before giving up. Long enough for a manual "I am not a robot" click; the
+    /// page stays open after the deadline so the user can finish and retry.
+    /// </summary>
+    public const int VerificationWaitTimeoutMs = 120000;
+
+    /// <summary>Selector for content items in listing pages (Fallback).</summary>
+    public const string DefaultListItemSelector = "div.row.rowcontent, div.table tr";
+
     // ===== Error Messages =====
 
     /// <summary>Error message for invalid URL.</summary>

GenHub/GenHub.Core/Constants/ModDBParserConstants.cs

@@ -17,6 +17,9 @@ public static class ModDBParserConstants
     /// <summary>Selector for developer/publisher links.</summary>
     public const string DeveloperSelector = "a[href*='/members/'], a[href*='/company/']";
 
+    /// <summary>Selector for the profile/mod info box that carries the real developer name.</summary>
+    public const string DeveloperProfileSelector = "#modsinfo a[href*='/members/'], #modsinfo a[href*='/company/'], .sidecolumn a[href*='/members/'], .sidecolumn a[href*='/company/']";
+
     /// <summary>Selector for release date.</summary>
     public const string ReleaseDateSelector = "time[datetime], .date, .released";
 
@@ -59,7 +62,7 @@ public static class ModDBParserConstants
     public const string FileMetadataValueSelector = "td:last-child";
 
     /// <summary>Selector for the main download button on file pages.</summary>
-    public const string MainDownloadButtonSelector = "a.download, a.downloadarea, .downloadbutton a, a[href*='/downloads/start/']";
+    public const string MainDownloadButtonSelector = "a.download, a.downloadarea, .downloadbutton a, a[href*='/downloads/start/'], a[href*='/addons/start/']";
 
     /// <summary>Selector for download size on the button.</summary>
     public const string DownloadSizeSelector = ".download .size, .downloadbutton .size";
@@ -87,39 +90,45 @@ public static class ModDBParserConstants
     // ===== Description/Summary Selectors =====
 
     /// <summary>Selector for full description content.</summary>
-    public const string FullDescriptionSelector = "#articlebrowse, .summary .content, .description .content, .modtext";
+    public const string FullDescriptionSelector = "#downloaddescription, #downloadsummary, #articlebrowse .articlebody, .articlebody, #modsummary, .modtext, #profile .description, #description, #articlebrowse, .summary .content, .description .content";
+
+    /// <summary>Selector for the file-page body copy (not the breadcrumb .summary trail).</summary>
+    public const string FileDescriptionSelector = "#downloaddescription, #downloadsummary";
 
-    /// <summary>Selector for truncated summary.</summary>
-    public const string SummarySelector = ".summary p, .description p";
+    /// <summary>Selector for summary or description container.</summary>
+    public const string SummarySelector = ".description, .rubric, p[itemprop='description']";
 
     // ===== Legacy File Selectors =====
 
     /// <summary>Selector for files table.</summary>
     public const string FilesTableSelector = "table.filelist, .table.files, #files";
 
     /// <summary>Selector for individual file rows.</summary>
-    public const string FileRowSelector = "tr.file, .row.file, .file";
+    public const string FileRowSelector = "tr.file, .row.file, .file, .row.rowcontent";
 
     /// <summary>Selector for file name.</summary>
-    public const string FileNameSelector = "h5, h4, .name, .title";
+    public const string FileNameSelector = "h4 a, h5 a, h3 a, .heading a, .title a, a.title, .name a, h5, h4, .name, .title";
 
     /// <summary>Selector for file version.</summary>
     public const string FileVersionSelector = ".version, .ver";
 
     /// <summary>Selector for file size.</summary>
-    public const string FileSizeSelector = ".size, .filesize";
+    public const string FileSizeSelector = ".size, .filesize, .filesizes, span.size";
+
+    /// <summary>Selector for file subheading or metadata row.</summary>
+    public const string FileSubheadingSelector = ".subheading, span.subheading, .meta, .details, .info, p.summary, .summary";
 
     /// <summary>Selector for file upload date.</summary>
-    public const string FileDateSelector = "time[datetime], .date, .uploaded";
+    public const string FileDateSelector = "time[datetime], .date, .uploaded, time";
 
     /// <summary>Selector for file category.</summary>
-    public const string FileCategorySelector = ".category, .type";
+    public const string FileCategorySelector = ".category, .type, span.category";
 
     /// <summary>Selector for file uploader.</summary>
-    public const string FileUploaderSelector = ".uploader, .author, a[href*='/members/']";
+    public const string FileUploaderSelector = ".uploader, .author, a[href*='/members/'], a[href*='/company/']";
 
     /// <summary>Selector for file download link (robust).</summary>
-    public const string FileDownloadSelector = "a.button.download, a[href*='/downloads/start/'], .download a";
+    public const string FileDownloadSelector = "a.button, a.buttonlarge, a.download, a.btn, a[href*='/downloads/start/'], a[href*='/addons/start/'], a[href*='/downloads/'], a[href*='/addons/'], .download a, .actions a";
 
     /// <summary>Selector for file MD5 hash.</summary>
     public const string FileMd5Selector = ".md5, .hash";
@@ -130,22 +139,41 @@ public static class ModDBParserConstants
     // ===== Videos Section Selectors =====
 
     /// <summary>Selector for embedded video iframes.</summary>
-    public const string VideoSelector = "iframe[src*='youtube'], iframe[src*='vimeo'], iframe[src*='youtu.be']";
+    public const string VideoSelector = "iframe[src*='youtube'], iframe[src*='youtube-nocookie'], iframe[src*='youtu.be'], iframe[src*='vimeo'], iframe[src*='dailymotion'], iframe[src*='moddb.com/media/iframe'], iframe[src*='moddb.com/media/embed'], iframe[src*='moddb.com/videos/iframe'], iframe[src*='moddb.com/videos/embed']";
+
+    /// <summary>Selector for video gallery containers and items.</summary>
+    public const string VideoGallerySelector = "#videobox, #videosbrowse, #mediabrowse, .mediarow, .mediabox";
+
+    /// <summary>Selector for video links.</summary>
+    public const string VideoLinkSelector = "a[href*='/videos/'], a[href*='youtube.com/watch'], a[href*='youtu.be/'], a[href*='vimeo.com/']";
 
     /// <summary>Selector for video thumbnails.</summary>
-    public const string VideoThumbnailSelector = ".thumbnail img, .preview img";
+    public const string VideoThumbnailSelector = ".thumbnail img, .preview img, img";
 
     /// <summary>Selector for video titles.</summary>
-    public const string VideoTitleSelector = ".title, h3, h4";
+    public const string VideoTitleSelector = ".title, h3, h4, h5, .caption";
+
+    /// <summary>Selector for recommendation and related content sections.</summary>
+    public const string RecommendationsSelector = "#recommendations, .recommendations, #related, .related, #similar, .similar, #fansalsoviewed, .fansalsoviewed, .youmayalso, [class*='recommend'], [id*='recommend'], [class*='similar'], [id*='similar']";
 
     // ===== Images Section Selectors =====
 
     /// <summary>Selector for image gallery container.</summary>
-    public const string ImageGallerySelector = ".mediarow, .screenshot, .imagebox, .gallery";
+    public const string ImageGallerySelector = "#imagebox, #mediaimage, #imagebrowse, #mediabrowse, .mediarow";
+
+    /// <summary>
+    /// Selector for gallery images only. Deliberately excludes a blanket
+    /// <c>img[src*='media.moddb.com']</c> match, which previously pulled game icons, member
+    /// avatars, and file-page chrome into the Media tab.
+    /// </summary>
+    public const string GalleryImageSelector = "#imagebox img, #mediaimage img, #imagebrowse img, #mediabrowse img, .mediarow img, .media .holder img, #downloadsummary img, #downloaddescription img, .preview img, a[href*='/mods/'][href*='/images/'] img";
 
     /// <summary>Selector for individual images.</summary>
     public const string ImageSelector = "img";
 
+    /// <summary>Sidebar/profile containers whose images are icons and avatars, not gallery media.</summary>
+    public const string ImageSidebarSelector = "#modsinfo, #downloadsprofilemenu, #profile, .sidecolumn, aside";
+
     /// <summary>Selector for image thumbnails.</summary>
     public const string ImageThumbnailSelector = ".thumbnail img, .thumb img";
 
@@ -197,20 +225,24 @@ public static class ModDBParserConstants
 
     // ===== Comments Section Selectors =====
 
-    /// <summary>Selector for comments container.</summary>
-    public const string CommentsSelector = ".comment, .post, .comments";
+    /// <summary>Selector for comments container. Do not use #commentform — that is the composer.</summary>
+    public const string CommentsSelector = "#commentsbrowse";
 
-    /// <summary>Selector for individual comment rows.</summary>
-    public const string CommentRowSelector = ".comment, .post";
+    /// <summary>
+    /// Selector for posted comment rows. Requires the exact <c>rowcomment</c> class so the
+    /// composer rows (<c>rowcommentguest</c>, <c>rowcommentsummary</c>, <c>rowcommentemail</c>)
+    /// and <c>#commentform</c> are not treated as comments.
+    /// </summary>
+    public const string CommentRowSelector = ".row.rowcomment, .rowcomment";
 
     /// <summary>Selector for comment authors.</summary>
-    public const string CommentAuthorSelector = ".author, .username, a[href*='/members/']";
+    public const string CommentAuthorSelector = ".author, .username, .heading a, a[href*='/members/']";
 
-    /// <summary>Selector for comment content.</summary>
-    public const string CommentContentSelector = ".content, .body, .text";
+    /// <summary>Selector for comment content. Avoids bare <c>p</c> which matches login chrome and CSS blobs.</summary>
+    public const string CommentContentSelector = ":scope > .commentbody, .commentbody, p.comment";
 
     /// <summary>Selector for comment dates.</summary>
-    public const string CommentDateSelector = "time[datetime], .date";
+    public const string CommentDateSelector = "time[datetime], time, .date, .datetime, span.subheading";
 
     /// <summary>Selector for comment karma/votes.</summary>
     public const string CommentKarmaSelector = ".karma, .votes, .goodkarma, .badkarma";
@@ -245,4 +277,100 @@ public static class ModDBParserConstants
 
     /// <summary>Pattern for games URLs.</summary>
     public const string GamesUrlPattern = "/games/";
+
+    // ===== Mod Detail Page Selectors =====
+
+    /// <summary>Selector for the downloads section on mod pages.</summary>
+    public const string DownloadsSectionSelector = "#downloads, .downloads, .files";
+
+    /// <summary>Selector for the addons section on mod pages.</summary>
+    public const string AddonsSectionSelector = "#addons, .addons";
+
+    /// <summary>Selector for the tabs/navigation on mod pages.</summary>
+    public const string TabsSelector = ".tabs, .navigation, nav";
+
+    /// <summary>Selector for individual tab links.</summary>
+    public const string TabLinkSelector = "a[href*='/downloads'], a[href*='/addons']";
+
+    // ===== Metadata Keys (Internal/Normalized) =====
+
+    /// <summary>Metadata key for filename.</summary>
+    public const string MetadataFilename = "filename";
+
+    /// <summary>Alternative metadata key for filename.</summary>
+    public const string MetadataFileNameAlt = "file name";
+
+    /// <summary>Alternative metadata key for file.</summary>
+    public const string MetadataFileAlt = "file";
+
+    /// <summary>Metadata key for size.</summary>
+    public const string MetadataSize = "size";
+
+    /// <summary>Alternative metadata key for size.</summary>
+    public const string MetadataFileSizeAlt = "file size";
+
+    /// <summary>Metadata key for uploader.</summary>
+    public const string MetadataUploader = "uploader";
+
+    /// <summary>Alternative metadata key for uploaded by.</summary>
+    public const string MetadataUploadedBy = "uploaded by";
+
+    /// <summary>Alternative metadata key for author.</summary>
+    public const string MetadataAuthor = "author";
+
+    /// <summary>Metadata key for category.</summary>
+    public const string MetadataCategory = "category";
+
+    /// <summary>Alternative metadata key for file category.</summary>
+    public const string MetadataFileCategory = "file category";
+
+    /// <summary>Alternative metadata key for type.</summary>
+    public const string MetadataType = "type";
+
+    /// <summary>Metadata key for MD5 hash.</summary>
+    public const string MetadataMd5Hash = "md5 hash";
+
+    /// <summary>Metadata key for MD5 hash (alternative).</summary>
+    public const string MetadataMd5HashAlt = "md5hash";
+
+    /// <summary>Alternative metadata key for MD5 checksum.</summary>
+    public const string MetadataMd5Checksum = "md5 checksum";
+
+    /// <summary>Alternative metadata key for MD5.</summary>
+    public const string MetadataMd5 = "md5";
+
+    /// <summary>Alternative metadata key for hash.</summary>
+    public const string MetadataHash = "hash";
+
+    /// <summary>Alternative metadata key for checksum.</summary>
+    public const string MetadataChecksum = "checksum";
+
+    /// <summary>Metadata key for total downloads.</summary>
+    public const string MetadataTotalDownloads = "total downloads";
+
+    /// <summary>Alternative metadata key for download count.</summary>
+    public const string MetadataDownloadCount = "download count";
+
+    /// <summary>Metadata key for added date.</summary>
+    public const string MetadataAdded = "added";
+
+    /// <summary>Metadata key for updated date.</summary>
+    public const string MetadataUpdated = "updated";
+
+    // ===== Additional Selectors =====
+
+    /// <summary>Selector for fallback titles (h1, h2, etc).</summary>
+    public const string FallbackTitleSelector = "h2 a, h1 a, h2, h1";
+
+    /// <summary>Selector for file detail page title heading outside the global headerbox.</summary>
+    public const string FilePageTitleSelector = ".midcolumn h2, .columncenter h2, #downloadsfiles h2, #downloadsinfo h2, #downloads h2, .heading h2, .title h2, h2.title, .midcolumn h3, .heading h3";
+
+    /// <summary>Selector for file detail page preview images.</summary>
+    public const string FilePreviewImagesSelector = "#downloadmedia img, #downloadsmedia img, #preview img, #media img, .mediagallery img, .imagebox img, #imagebox img, #downloaddescription img, #downloadsummary img, a[href*='/images/'] img, .previewholder img, .media .holder img";
+
+    /// <summary>Selector for file description container elements.</summary>
+    public const string FileDescriptionContainerSelector = "#downloaddescription, #downloadsummary, #description, .description, .articlebody, #profiletotal";
+
+    /// <summary>Regex pattern for extracting parent mod path.</summary>
+    public const string ParentModPathRegex = @"(/mods/[^/]+)/(?:downloads|addons)/";
 }