diff --git a/docs/.vitepress/config.js b/docs/.vitepress/config.js index a1c96e894..2f324a372 100644 --- a/docs/.vitepress/config.js +++ b/docs/.vitepress/config.js @@ -36,7 +36,8 @@ export default withMermaid( { text: 'Introduction', link: '/' }, { text: 'Developer Onboarding', link: '/onboarding' }, { text: 'Architecture Overview', link: '/architecture' }, - { text: 'Velopack Integration', link: '/velopack-integration' } + { text: 'Velopack Integration', link: '/velopack-integration' }, + { text: 'Contributor Guidelines', link: '/dev/contribution-guidelines' } ] }, { @@ -45,6 +46,7 @@ export default withMermaid( { text: 'Overview', link: '/features/index' }, { text: 'App Update & Installer', link: '/velopack-integration' }, { text: 'Content System', link: '/features/content' }, + { text: 'Downloads Browser', link: '/features/downloads' }, { text: 'Content Reconciliation', link: '/features/reconciliation' }, { text: 'Manifest Service', link: '/features/manifest' }, { text: 'Storage & CAS', link: '/features/storage' }, @@ -54,7 +56,6 @@ export default withMermaid( { text: 'GameProfiles System', link: '/features/gameprofiles' }, { text: 'Game Installations', link: '/features/game-installations/' }, { text: 'User Data Management', link: '/features/userdata' }, - { text: 'Downloads UI', link: '/features/downloads-ui' }, { text: 'Notifications', link: '/features/notifications' }, { text: 'Desktop Shortcuts', link: '/features/desktop-shortcuts' }, { text: 'Steam Proxy Launcher', link: '/features/steam-proxy-launcher' }, @@ -99,6 +100,7 @@ export default withMermaid( { text: 'Content Acquisition', link: '/FlowCharts/Acquisition-Flow' }, { text: 'Workspace Assembly', link: '/FlowCharts/Assembly-Flow' }, { text: 'Manifest Creation', link: '/FlowCharts/Manifest-Creation-Flow' }, + { text: 'Downloads User Flow', link: '/FlowCharts/Downloads-Flow' }, { text: 'Complete User Flow', link: '/FlowCharts/Complete-User-Flow' }, { text: 'CAS Storage Flow', link: '/FlowCharts/CAS-Storage-Flow' }, { text: 'Dependency Resolution', link: '/FlowCharts/Dependency-Resolution-Flow' }, diff --git a/docs/FlowCharts/Downloads-Flow.md b/docs/FlowCharts/Downloads-Flow.md new file mode 100644 index 000000000..cd6d6b5f9 --- /dev/null +++ b/docs/FlowCharts/Downloads-Flow.md @@ -0,0 +1,792 @@ +--- +title: Downloads Flow +description: Complete user flow for downloading and installing content in GenHub +--- + +## Flowchart: Downloads User Flow + +This flowchart details the complete user journey from browsing publishers to downloading and installing content, including state management, profile selection, and caching. + +## Table of Contents + +1. [User Browsing Flow](#user-browsing-flow) +2. [Content State Management](#content-state-management) +3. [Publisher Selection](#publisher-selection) +4. [Content Acquisition Flow (Updated)](#content-acquisition-flow-updated) +5. [Profile Selection Flow](#profile-selection-flow) +6. [ModDB Integration](#moddb-integration) +7. [Content Caching Layer](#content-caching-layer) +8. [Key Components](#key-components) +9. [Error Handling](#error-handling) + +## User Browsing Flow + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart TD + subgraph User["๐Ÿ‘ค User Actions"] + A["Open Downloads Tab"] + B["Select Publisher
(ModDB, CNC Labs, etc.)"] + C["Browse/Search Content"] + D["Click Content Card"] + E["View Details"] + F["Click Download"] + end + + subgraph ViewModel["๐Ÿ“ฑ DownloadsBrowserViewModel"] + V1["LoadPublishersAsync()"] + V2["SetSelectedPublisher()"] + V3["DiscoverContentAsync()"] + V4["OpenContentDetail()"] + V5["DownloadContentCommand"] + end + + subgraph Pipeline["๐Ÿ”ง Content Pipeline"] + P1["IContentDiscoverer"] + P2["ContentDiscoveryResult"] + P3["IContentResolver"] + P4["IContentManifestFactory"] + end + + subgraph Storage["๐Ÿ’พ Storage"] + S1["CAS Service"] + S2["Manifest Pool"] + S3["Profile Integration"] + end + + A --> V1 + V1 --> B + B --> V2 + V2 --> V3 + V3 --> P1 + P1 --> P2 + P2 --> C + C --> D + D --> V4 + V4 --> E + E --> F + F --> V5 + V5 --> P3 + P3 --> P4 + P4 --> S1 + P4 --> S2 + S2 --> S3 + + classDef user fill:#3182ce,stroke:#2c5282,stroke-width:2px,color:#ffffff + classDef viewmodel fill:#805ad5,stroke:#6b46c1,stroke-width:2px,color:#ffffff + classDef pipeline fill:#38a169,stroke:#2f855a,stroke-width:2px,color:#ffffff + classDef storage fill:#e53e3e,stroke:#c53030,stroke-width:2px,color:#ffffff + + class A,B,C,D,E,F user + class V1,V2,V3,V4,V5 viewmodel + class P1,P2,P3,P4 pipeline + class S1,S2,S3 storage +``` + +## Content State Management + +The `ContentStateService` determines the current state of content for UI display, enabling the Downloads browser to show appropriate buttons (Download, Update, Add to Profile) based on content availability. + +### State Flow Diagram + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +stateDiagram-v2 + [*] --> NotDownloaded: Content discovered + NotDownloaded --> Downloaded: Download complete + Downloaded --> UpdateAvailable: Newer version found + UpdateAvailable --> Downloaded: Update downloaded + Downloaded --> [*]: Content removed + NotDownloaded --> [*]: Content skipped + + NotDownloaded: Show "Download" button + Downloaded: Show "Add to Profile" button + UpdateAvailable: Show "Update" button +``` + +### ContentStateService + +**Location**: `GenHub/Features/Downloads/Services/ContentStateService.cs` + +The service uses the 5-segment manifest ID format to detect content versions: + +```text +Format: schemaVersion.userVersion.publisher.contentType.contentName +Example: 1.20240315.moddb.mod.releasename +``` + +**Detection Logic**: + +1. **Exact Match Check**: Generates prospective manifest ID using `ManifestIdGenerator.GeneratePublisherContentId(publisher, contentType, name, releaseDate)` +2. **Update Detection**: Searches for manifests with same publisher, contentType, and contentName but older userVersion (date) +3. **State Determination**: + - `Downloaded`: Exact match found in manifest pool + - `UpdateAvailable`: Older version found + - `NotDownloaded`: No versions found + +**Usage Example**: + +```csharp +var state = await contentStateService.GetStateAsync(searchResult); +switch (state) +{ + case ContentState.NotDownloaded: + // Show Download button + break; + case ContentState.UpdateAvailable: + // Show Update button + break; + case ContentState.Downloaded: + // Show "Add to Profile" button + break; +} +``` + +### Content State Sequence Diagram + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +sequenceDiagram + participant VM as ContentGridItemViewModel + participant CSS as ContentStateService + participant MIG as ManifestIdGenerator + participant Pool as ManifestPool + + VM->>CSS: GetStateAsync(searchResult) + CSS->>MIG: GeneratePublisherContentId(publisher, type, name, date) + MIG-->>CSS: "1.20240315.moddb.mod.mycontent" + CSS->>Pool: IsManifestAcquiredAsync(prospectiveId) + + alt Exact Match Found + Pool-->>CSS: true + CSS-->>VM: ContentState.Downloaded + else No Exact Match + Pool-->>CSS: false + CSS->>Pool: GetAllManifestsAsync() + Pool-->>CSS: List + CSS->>CSS: FindOlderVersionsAsync() + + alt Older Version Found + CSS-->>VM: ContentState.UpdateAvailable + else No Versions Found + CSS-->>VM: ContentState.NotDownloaded + end + end +``` + +## Publisher Selection + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart LR + subgraph Sidebar["Publisher Sidebar"] + P1["๐ŸŽฎ ModDB"] + P2["๐Ÿ—บ๏ธ CNC Labs"] + P3["๐Ÿ—บ๏ธ AOD Maps"] + P4["๐Ÿ”ง Community Outpost"] + P5["๐Ÿ™ GitHub"] + P6["๐ŸŒ Generals Online"] + end + + subgraph Filter["Filter Panel"] + F1["Content Type"] + F2["Game (Generals/ZH)"] + F3["Search Term"] + F4["Sort Order"] + end + + subgraph Grid["Content Grid"] + G1["ContentCardView 1"] + G2["ContentCardView 2"] + G3["ContentCardView n..."] + end + + P1 & P2 & P3 & P4 & P5 & P6 --> Filter + Filter --> Grid +``` + +## Content Acquisition Flow (Updated) + +This sequence diagram shows the complete flow from download to profile integration, including state detection and profile selection. + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +sequenceDiagram + actor User + participant UI as ContentCardView + participant VM as ContentGridItemViewModel + participant BVM as DownloadsBrowserViewModel + participant CSS as ContentStateService + participant R as Resolver + participant MIG as ManifestIdGenerator + participant MF as ManifestFactory + participant CAS as CAS Service + participant Pool as ManifestPool + participant PS as ProfileSelectionViewModel + participant PCS as ProfileContentService + + User->>UI: Click "Download" / "Update" + UI->>VM: DownloadCommand / UpdateCommand + VM->>BVM: DownloadContentAsync(item) + + Note over BVM: Get resolver for publisher + BVM->>R: ResolveAsync(searchResult) + + alt ModDB Content + R->>R: Parse page (Playwright + AngleSharp) + R->>R: Extract files with FileSectionType + end + + R->>MIG: GeneratePublisherContentId() + Note over MIG: Format: 1.yyyyMMdd.publisher.type.name + MIG-->>R: Manifest ID + + R->>MF: CreateManifestAsync(details) + MF-->>BVM: ContentManifest + + Note over BVM: Download files to temp + BVM->>CAS: DownloadFileAsync(url, tempPath) + + alt Archive File + BVM->>BVM: Extract all files + loop Each file + BVM->>CAS: StoreContentAsync(file, hash) + end + else Single File + BVM->>CAS: StoreContentAsync(file, hash) + end + + Note over BVM: Store manifest in pool + BVM->>Pool: AddManifestAsync(manifest, tempDir) + Pool-->>BVM: Success + + Note over BVM: Update item state + BVM->>VM: CurrentState = Downloaded + VM->>UI: Show "Add to Profile" button + + Note over User: Content ready for profiles + User->>UI: Click "Add to Profile" + UI->>BVM: AddContentToProfileAsync(item) + BVM->>PS: LoadProfilesAsync(targetGame, manifestId) + + Note over PS: Filter by game type compatibility + PS->>PS: Separate compatible vs incompatible + PS-->>User: Show profile dialog + + User->>PS: Select profile + PS->>PCS: AddContentToProfileAsync(profileId, manifestId) + PCS-->>PS: Success + PS-->>User: Close dialog + notification +``` + +## Profile Selection Flow + +The `ProfileSelectionViewModel` provides smart filtering for game profiles, showing compatible profiles first and incompatible profiles with warnings. This ensures content is added to the correct game type profile. + +### Profile Selection Diagram + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart TD + subgraph Dialog["ProfileSelectionView"] + direction TB + Header["Select Profile for: {ContentName}"] + + subgraph Compatible["โœ… Compatible Profiles"] + C1["Profile 1 (Zero Hour)"] + C2["Profile 2 (Zero Hour)"] + end + + subgraph Incompatible["โš ๏ธ Incompatible Profiles"] + I1["Profile 3 (Generals)
Warning: Content is for Zero Hour"] + I2["Profile 4 (Generals)
Warning: Content is for Zero Hour"] + end + + Buttons["Create New Profile | Cancel"] + end + + User["User clicks profile"] --> SelectProfile[SelectProfileCommand] + SelectProfile --> PCS[ProfileContentService] + PCS --> Profile[Add content to profile] + Profile --> Notify[Show success notification] +``` + +### Smart Filtering Logic + +**Location**: `GenHub/Features/Downloads/ViewModels/ProfileSelectionViewModel.cs` + +The profile selection uses the following compatibility rules: + +| Content Type | Compatible Profile | Incompatible Profile | +| :--- | :--- | :--- | +| ZeroHour Mod | Zero Hour profiles | Generals profiles | +| Generals Mod | Generals profiles | Zero Hour profiles | + +**Key Methods**: + +- `LoadProfilesAsync(targetGame, contentManifestId, contentName)` - Loads and filters profiles +- `IsCompatible(profile, targetGame)` - Checks if profile's game type matches content +- `CreateNewProfileAsync()` - Creates a new profile with the content pre-enabled + +**Profile Summary Display**: + +```text +"2 compatible, 1 incompatible" - Mixed compatibility +"3 compatible profiles" - All compatible +"1 incompatible profile" - All incompatible +"No profiles available" - No profiles exist +``` + +### Profile Selection Sequence Diagram + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +sequenceDiagram + participant User + participant CDVM as ContentDetailViewModel + participant PSVM as ProfileSelectionViewModel + participant PM as ProfileManager + participant PCS as ProfileContentService + participant Profile as GameProfile + + User->>CDVM: Click "Add to Profile" + CDVM->>PSVM: Create(targetGame, manifestId, contentName) + PSVM->>PM: GetAllProfilesAsync() + PM-->>PSVM: List + + loop For each profile + PSVM->>PSVM: IsCompatible(profile, targetGame) + alt Game Type Matches + PSVM->>PSVM: Add to CompatibleProfiles + else Game Type Mismatch + PSVM->>PSVM: Add to OtherProfiles
with warning + end + end + + PSVM-->>User: Show dialog with filtered profiles + User->>PSVM: Select profile + PSVM->>PCS: AddContentToProfileAsync(profileId, manifestId) + PCS->>Profile: Add content + PCS-->>PSVM: Success + PSVM-->>User: Close dialog + notify +``` + +## ModDB Integration + +ModDB content discovery uses a two-stage approach: Playwright for JavaScript-rendered content, followed by AngleSharp for structured HTML parsing. The parser distinguishes between main releases (Downloads section) and addons. + +### ModDB Parsing Flow + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart TD + Start["ModDB URL"] --> Playwright["Playwright Fetch
(handles JavaScript)"] + Playwright --> HTML["Raw HTML"] + HTML --> AngleSharp["AngleSharp Parser
(structured extraction)"] + + AngleSharp --> Detect{Page Type?} + + Detect -->|Mod Detail| Detail["Detail Page"] + Detect -->|File Detail| FileDetail["File Detail Page"] + Detect -->|List| List["List Page
(addons/images)"] + + Detail --> FetchBoth["Fetch Both Sections"] + FetchBoth --> Downloads["/downloads section
(FileSectionType.Downloads)"] + FetchBoth --> Addons["/addons section
(FileSectionType.Addons)"] + + Downloads --> Files["Extract Files"] + Addons --> Files + FileDetail --> Files + List --> Files + + Files --> Parse["Parse File Metadata"] + Parse --> SectionTag["Tag with FileSectionType"] + SectionTag --> Result["ParsedWebPage"] +``` + +### FileSectionType Enum + +**Location**: `GenHub/Core/Models/Parsers/FileSectionType.cs` + +```csharp +public enum FileSectionType +{ + /// Files from the main releases/downloads section + Downloads, + + /// Files from the addons section + Addons, +} +``` + +### Addon-Only Mod Handling + +For mods that only have addons (no main downloads): + +1. **Detection**: Parser detects mod detail pages without a `/downloads` section +2. **Addons Section**: Fetches `/addons` subsection and parses with `FileSectionType.Addons` +3. **Manifest Creation**: Each addon gets its own manifest with `ContentType.Addon` +4. **Content Type**: Addons are tagged separately from main mod releases + +### ModDB Resolver Flow + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +sequenceDiagram + participant DC as DownloadsBrowserViewModel + participant MR as ModDBResolver + participant MP as ModDBPageParser + participant MF as ModDBManifestFactory + participant MIG as ManifestIdGenerator + + DC->>MR: ResolveAsync(searchResult) + MR->>MP: ParseAsync(sourceUrl) + + alt Mod Detail Page + MP->>MP: Fetch /downloads + MP->>MP: Fetch /addons + MP-->>MR: ParsedWebPage with both sections + else Standard Page + MP-->>MR: ParsedWebPage + end + + MR->>MR: Extract files from parsed page + + alt Has Downloads Section Files + MR->>MR: Use primary file from Downloads + else Only Addons + MR->>MR: Use primary file from Addons + end + + MR->>MR: ConvertFileToMapDetails(file) + Note over MR: ContentType = Addon if
FileSectionType.Addons + + MR->>MF: CreateManifestAsync(mapDetails, sourceUrl) + MF->>MIG: GeneratePublisherContentId() + Note over MIG: Uses release date as version
Format: 1.yyyyMMdd.publisher.type.name + MIG-->>MF: Manifest ID + MF-->>MR: ContentManifest + MR-->>DC: ContentManifest with section metadata tags +``` + +## Content Caching Layer + +The `ContentCacheService` provides an in-memory cache for parsed web page content with a configurable TTL (Time To Live). This reduces redundant fetching and parsing of the same pages. + +### Cache Architecture + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart LR + subgraph Cache["ContentCacheService"] + CacheStore["ConcurrentDictionary"] + TTL["Default TTL: 1 Hour"] + end + + subgraph Operations["Cache Operations"] + Get["GetAsync(key)"] + Set["SetAsync(key, data, ttl?)"] + Has["HasValidCache(key)"] + Invalidate["Invalidate(key)"] + Clear["ClearAll()"] + end + + Get --> CacheStore + Set --> CacheStore + Has --> CacheStore + Invalidate --> CacheStore + Clear --> CacheStore + + CacheEntry["CacheEntry
- ParsedWebPage Data
- ExpiresAt DateTime"] + + CacheStore --> CacheEntry +``` + +### Cache Service Details + +**Location**: `GenHub/Features/Content/Services/ContentCacheService.cs` + +| Method | Purpose | Returns | +| :--- | :--- | :--- | +| `GetAsync(cacheKey)` | Retrieve cached content | `ParsedWebPage?` or `null` if expired/missing | +| `SetAsync(cacheKey, data, ttl?)` | Store content in cache | `Task` (completed) | +| `HasValidCache(cacheKey)` | Check if valid cache exists | `bool` | +| `Invalidate(cacheKey)` | Remove specific entry | `void` | +| `ClearAll()` | Clear all cache entries | `void` | + +**Cache Entry Structure**: + +```csharp +private record CacheEntry( + ParsedWebPage Data, // The cached parsed page + DateTime ExpiresAt // When the cache expires +); +``` + +**Default TTL**: 1 hour (`TimeSpan.FromHours(1)`) + +### Lazy Loading for Tabs + +The `ContentDetailViewModel` implements lazy loading for detail view tabs to improve performance: + +```mermaid +%%{init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#e2e8f0', + 'primaryTextColor': '#1a202c', + 'primaryBorderColor': '#4a5568', + 'lineColor': '#2d3748', + 'background': '#ffffff' + } +}}%% + +flowchart TD + User["User opens detail view"] --> Basic["Load Basic Content"] + Basic --> Icon["Load Icon"] + Icon --> Idle["Idle State"] + + Idle --> ImagesTab["User clicks Images tab"] + Idle --> VideosTab["User clicks Videos tab"] + Idle --> ReleasesTab["User clicks Releases tab"] + Idle --> AddonsTab["User clicks Addons tab"] + + ImagesTab --> LoadImages["LoadImagesAsync()"] + VideosTab --> LoadVideos["LoadVideosAsync()"] + ReleasesTab --> LoadReleases["LoadReleasesAsync()"] + AddonsTab --> LoadAddons["LoadAddonsAsync()"] + + LoadImages --> ImagesDone["Images loaded (flag set)"] + LoadVideos --> VideosDone["Videos loaded (flag set)"] + LoadReleases --> ReleasesDone["Releases populated"] + LoadAddons --> AddonsDone["Addons populated"] +``` + +**Lazy Load Flags**: + +- `_imagesLoaded` - Prevents re-loading images tab +- `_videosLoaded` - Prevents re-loading videos tab +- `_releasesLoaded` - Prevents re-loading releases tab +- `_addonsLoaded` - Prevents re-loading addons tab +- `_basicContentLoaded` - Basic page info loaded on open + +## Key Components + +### DownloadsBrowserViewModel + +**Location**: `GenHub/Features/Downloads/ViewModels/DownloadsBrowserViewModel.cs` + +| Property/Command | Type | Purpose | +| :--- | :--- | :--- | +| `Publishers` | `ObservableCollection` | Available content sources | +| `SelectedPublisher` | `PublisherItemViewModel` | Currently selected publisher | +| `ContentItems` | `ObservableCollection` | Discovered content | +| `FilterViewModel` | `IFilterPanelViewModel` | Publisher-specific filters | +| `DownloadContentCommand` | `IAsyncRelayCommand` | Initiates download | +| `AddContentToProfileCommand` | `IAsyncRelayCommand` | Adds content to profile | + +### ContentGridItemViewModel + +**Location**: `GenHub/Features/Downloads/ViewModels/ContentGridItemViewModel.cs` + +Represents a single content item in the grid with: + +- Title, description, preview image +- Publisher info and tags +- Download URL and content type +- Installation status tracking via `CurrentState` property + +**State-Dependent UI Properties**: + +| Property | Condition | Purpose | +| :--- | :--- | :--- | +| `ShowDownloadButton` | `CurrentState == NotDownloaded` | Shows download button | +| `ShowUpdateButton` | `CurrentState == UpdateAvailable` | Shows update button | +| `ShowAddToProfileButton` | `CurrentState == Downloaded` | Shows "Add to Profile" button | +| `CanDownload` | `!IsDownloaded && !IsDownloading` | Enables download action | + +### ContentDetailViewModel + +**Location**: `GenHub/Features/Downloads/ViewModels/ContentDetailViewModel.cs` + +Provides detailed content view with lazy-loaded tabs: + +- **Overview Tab**: Basic content info (loaded immediately) +- **Images Tab**: Gallery images (loaded on first access) +- **Videos Tab**: Embedded videos (loaded on first access) +- **Releases Tab**: Main downloads section files (loaded on first access) +- **Addons Tab**: Addon section files (loaded on first access) + +**Lazy Loading Implementation**: + +```csharp +private bool _imagesLoaded; +private bool _videosLoaded; +private bool _releasesLoaded; +private bool _addonsLoaded; +private bool _basicContentLoaded; + +[RelayCommand] +private async Task LoadImagesAsync() +{ + if (_imagesLoaded || IsLoadingImages) return; + // ... load images + _imagesLoaded = true; +} +``` + +### Filter ViewModels + +Each publisher has a specialized filter ViewModel: + +| Publisher | Filter ViewModel | Special Filters | +| :--- | :--- | :--- | +| ModDB | `ModDBFilterViewModel` | Category, release date | +| CNC Labs | `CNCLabsFilterViewModel` | Map size, player count | +| AOD Maps | `AODMapsFilterViewModel` | Map type | +| Community Outpost | `CommunityOutpostFilterViewModel` | Tool vs patch | +| GitHub | `GitHubFilterViewModel` | Repository, release type | + +### ContentStateService Reference + +**Location**: `GenHub/Features/Downloads/Services/ContentStateService.cs` + +| Method | Purpose | +| :--- | :--- | +| `GetStateAsync(item)` | Gets current state (NotDownloaded, UpdateAvailable, Downloaded) | +| `GetLocalManifestIdAsync(item)` | Returns local manifest ID if downloaded | + +### ProfileSelectionViewModel + +**Location**: `GenHub/Features/Downloads/ViewModels/ProfileSelectionViewModel.cs` + +| Property | Type | Purpose | +| :--- | :--- | :--- | +| `CompatibleProfiles` | `ObservableCollection` | Matching game type profiles | +| `OtherProfiles` | `ObservableCollection` | Non-matching profiles with warnings | +| `ProfileSummary` | `string` | Human-readable profile counts | +| `SelectProfileCommand` | `IAsyncRelayCommand` | Adds content to selected profile | +| `CreateNewProfileCommand` | `IAsyncRelayCommand` | Creates new profile with content | + +## Error Handling + +```mermaid +flowchart TD + D["Download Attempt"] --> N{Network OK?} + N -->|No| E1["Show network error
+ retry option"] + N -->|Yes| A{Auth Required?} + A -->|Yes| E2["Prompt for auth
(ModDB WAF)"] + A -->|No| DL["Download File"] + DL --> V{Valid File?} + V -->|No| E3["Show validation error"] + V -->|Yes| EX{Extract OK?} + EX -->|No| E4["Show extraction error
fallback to single file"] + EX -->|Yes| S["Store in CAS"] + S --> M["Create Manifest"] +``` + +## Related Documentation + +- [Content Pipeline](../features/content/content-pipeline.md) - Detailed pipeline architecture +- [Discovery Flow](./Discovery-Flow.md) - Discovery process +- [Acquisition Flow](./Acquisition-Flow.md) - Content acquisition diff --git a/docs/FlowCharts/index.md b/docs/FlowCharts/index.md index 13c011102..42aa2e741 100644 --- a/docs/FlowCharts/index.md +++ b/docs/FlowCharts/index.md @@ -12,6 +12,7 @@ This section contains detailed flowcharts that illustrate how GenHub's various s - **[Content Discovery Flow](./Discovery-Flow.md)** - How GenHub discovers content from publishers and sources - **[Content Resolution Flow](./Resolution-Flow.md)** - Converting discovered content into installable manifests - **[Content Acquisition Flow](./Acquisition-Flow.md)** - Downloading and preparing content packages +- **[Downloads User Flow](./Downloads-Flow.md)** - Complete user journey from browsing to installation - **[Workspace Assembly Flow](./Assembly-Flow.md)** - Building isolated game workspaces - **[Manifest Creation Flow](./Manifest-Creation-Flow.md)** - Creating ContentManifest files programmatically - **[Game Detection Flow](./Detection-Flow.md)** - Detecting and validating game installations diff --git a/docs/dev/constants.md b/docs/dev/constants.md index 0c05a990e..a66c06581 100644 --- a/docs/dev/constants.md +++ b/docs/dev/constants.md @@ -54,10 +54,10 @@ Application-wide constants for GenHub. | `PullRequestNumber` | Dynamic | PR number if PR build | | `BuildChannel` | Dynamic | Build channel (Dev, PR, CI, Release) | | `IsCiBuild` | bool | Whether this is a CI/CD build | -| `FullDisplayVersion` | string | Full display version with hash | -| `GitHubRepositoryUrl` | `"https://github.com/community-outpost/GenHub"` | GitHub repository URL | -| `GitHubRepositoryOwner` | `"community-outpost"` | GitHub repository owner | -| `GitHubRepositoryName` | `"GenHub"` | GitHub repository name | +| `FullDisplayVersion` | string | Full display version with hash | +| `GitHubRepositoryUrl` | `"https://github.com/community-outpost/GenHub"` | GitHub repository URL | +| `GitHubRepositoryOwner` | `"community-outpost"` | GitHub repository owner | +| `GitHubRepositoryName` | `"GenHub"` | GitHub repository name | | `DefaultTheme` | `Theme.Dark` | Default UI theme | | `DefaultThemeName` | `"Dark"` | Default theme name as string | | `TokenFileName` | `".ghtoken"` | Default GitHub token file name | @@ -137,6 +137,18 @@ Configuration key constants for `appsettings.json` and environment variables. Constants related to workspace management and configuration. - `DefaultWorkspaceStrategy`: The default workspace strategy to use when none is specified (`WorkspaceStrategy.HardLink`) +## CommandLineConstants Class + +Constants for command line arguments and URI schemes. + +| Constant | Value | Description | +| --------------------------- | --------------------- | ---------------------------------------------------------- | +| `LaunchProfileArg` | `"--launch-profile"` | Command-line argument used to request launching a profile | +| `LaunchProfileInlinePrefix` | `"--launch-profile="` | Prefix for inline profile launching | +| `UriScheme` | `"genhub://"` | URI scheme used for protocol handling | +| `SubscribeCommand` | `"subscribe"` | Command for subscribing to a catalog via URI | +| `SubscribeUriPrefix` | `"genhub://subscribe"`| Full prefix for subscription URI | +| `SubscribeUrlParam` | `"?url="` | Query parameter name for the catalog URL | --- @@ -192,11 +204,11 @@ File and directory name constants to prevent typos and ensure consistency. | Constant | Value | Description | | ----------------------- | ------------------- | --------------------------------- | -| `ManifestsDirectory` | `"Manifests"` | Directory for manifest files | -| `ManifestFilePattern` | `"*.manifest.json"` | File pattern for manifest files | -| `ManifestFileExtension` | `".manifest.json"` | File extension for manifest files | -| `UserDataManifestExtension` | `".userdata.json"` | File extension for user data manifest files | -| `BackupExtension` | `".ghbak"` | File extension for backup files | +| `ManifestsDirectory` | `"Manifests"` | Directory for manifest files | +| `ManifestFilePattern` | `"*.manifest.json"` | File pattern for manifest files | +| `ManifestFileExtension` | `".manifest.json"` | File extension for manifest files | +| `UserDataManifestExtension` | `".userdata.json"` | File extension for user data manifest files | +| `BackupExtension` | `".ghbak"` | File extension for backup files | ### JSON Files @@ -831,17 +843,27 @@ Storage and CAS (Content-Addressable Storage) related constants. --- -### Status Colors +## UiConstants Class -- `StatusSuccessColor`: Color used to indicate success or positive status (`"#4CAF50"`) -- `StatusErrorColor`: Color used to indicate error or negative status (`"#F44336"`) +User interface sizing and theming constants. -### ValidationLimits +### Window and Layout Sizing - `DefaultWindowWidth`: 1200 - `DefaultWindowHeight`: 800 +- `DefaultProfileSettingsWidth`: 750 +- `DefaultProfileSettingsHeight`: 700 +- `DefaultProfileSettingsSidebarWidth`: 190 +- `MinProfileSettingsSidebarWidth`: 68 +- `MaxProfileSettingsSidebarWidth`: 300 ---- +### Status Colors + +- `StatusSuccessColor`: Color used to indicate success or positive status (`"#4CAF50"`) +- `StatusErrorColor`: Color used to indicate error or negative status (`"#F44336"`) +- `StatusDownloadedColor`: Color used for downloaded status indicator (`"#4CAF50"`) +- `StatusNotDownloadedColor`: Color used for not downloaded status indicator (`"#B388FF"`) +- `StatusUpdateAvailableColor`: Color used for update available status indicator (`"#FFB74D"`) ## ValidationLimits Class @@ -1393,10 +1415,19 @@ Constants for various community content publishers and manifest generation. Constants related to the Community Outpost (GenPatcher) catalog and metadata. -- `CatalogFilename`: Default filename for the GenPatcher catalog (`"GenPatcher.dat"`) -- `VersionKey`: Metadata key for version information (`"Version"`) -- `DescriptionKey`: Metadata key for description information (`"Description"`) -- `DownloadUrlKey`: Metadata key for download URLs (`"DownloadUrl"`) +- `CatalogFormat`: Catalog format identifier (`"genpatcher-dat"`) +- `UnknownVersion`: Default version string when unknown (`"unknown"`) +- `DefaultBaseUrl`: Default base URL for making relative URLs absolute (`"https://legi.cc/patch"`) +- `DefaultFilesBaseUrl`: Default base URL for downloading GenPatcher content .dat packages (`"https://legi.cc/gp2/f"`) +- `ContentCodeKey`: Metadata key for content code (`"contentCode"`) +- `CatalogVersionKey`: Metadata key for catalog version (`"catalogVersion"`) +- `FileSizeKey`: Metadata key for file size (`"fileSize"`) +- `CategoryKey`: Metadata key for content category (`"category"`) +- `InstallTargetKey`: Metadata key for install target (`"installTarget"`) +- `MirrorUrlsKey`: Metadata key for mirror URLs (`"mirrorUrls"`) +- `MirrorsKey`: Metadata key for mirror names display string (`"mirrors"`) +- `PatchPageUrlEndpoint`: Endpoint key for patch page URL (`"patchPageUrl"`) +- `DefaultMetadataVersion`: Default version for content metadata (`"1.0"`) ### GeneralsOnlineConstants Class @@ -1512,6 +1543,139 @@ Constants specifically for the Map Manager feature. | `ToolName` | `"Map Manager"` | Display name for Map Manager | | `ToolDescription` | `"Manage, import, and share custom maps. Create MapPacks for easy profile switching."` | Description of the tool | +## Content Publisher Constants + +Constants for various community content publishers and manifest generation. + +### CommunityOutpostCatalogConstants Class + +Constants related to the Community Outpost (GenPatcher) catalog and metadata. + +- `CatalogFormat`: Catalog format identifier (`"genpatcher-dat"`) +- `UnknownVersion`: Default version string when unknown (`"unknown"`) +- `DefaultBaseUrl`: Default base URL for making relative URLs absolute (`"https://legi.cc/patch"`) +- `DefaultFilesBaseUrl`: Default base URL for downloading GenPatcher content .dat packages (`"https://legi.cc/gp2/f"`) +- `ContentCodeKey`: Metadata key for content code (`"contentCode"`) +- `CatalogVersionKey`: Metadata key for catalog version (`"catalogVersion"`) +- `FileSizeKey`: Metadata key for file size (`"fileSize"`) +- `CategoryKey`: Metadata key for content category (`"category"`) +- `InstallTargetKey`: Metadata key for install target (`"installTarget"`) +- `MirrorUrlsKey`: Metadata key for mirror URLs (`"mirrorUrls"`) +- `MirrorsKey`: Metadata key for mirror names display string (`"mirrors"`) +- `PatchPageUrlEndpoint`: Endpoint key for patch page URL (`"patchPageUrl"`) +- `DefaultMetadataVersion`: Default version for content metadata (`"1.0"`) + +### GeneralsOnlineConstants Class + +Constants for Generals Online content discovery and manifest creation. + +- `PublisherPrefix`: Publisher prefix string (`"generalsonline"`) +- `PublisherId`: Publisher identifier (`"generals-online"`) +- `PublisherDisplayName`: Display name for the publisher (`"Generals Online"`) +- `QfeMarkerPrefix`: Prefix used for QFE (Quick Fix Engineering) versions (`"qfe-"`) +- `MapPackTags`: Default tags for MapPack manifests (`["mappack", "generalsonline"]`) +- `UnknownVersion`: Default version string when unknown (`"unknown"`) +- `CoverSource`: Default path for cover images (`"/Assets/Covers/zerohour-cover.png"`) + +### CNCLabsConstants Class + +Constants for CNC Labs (CNC Maps) content discovery and manifest creation. + +- `PublisherPrefix`: Publisher prefix string (`"cnclabs"`) +- `PublisherId`: Publisher identifier (`"cnc-labs"`) +- `PublisherName`: Display name for the publisher (`"CNC Labs"`) +- `PublisherWebsite`: Main website URL (`"https://www.cnclabs.com"`) +- `DefaultTags`: Default tags for CNC Labs manifests (`["cnclabs"]`) +- `DefaultDownloadFilename`: Default filename for downloads when parsing fails (`"download.zip"`) + +### ModDBConstants Class + +Constants for ModDB content discovery and manifest creation. + +- `PublisherPrefix`: Publisher prefix string (`"moddb"`) +- `PublisherDisplayName`: Display name for the publisher (`"ModDB"`) +- `PublisherWebsite`: Main website URL (`"https://www.moddb.com"`) +- `ReleaseDateFormat`: Date format used in ModDB metadata (`"MMMM dd, yyyy"`) +- `PublisherNameFormat`: Format string for including the author with the publisher name (`"ModDB ({0})"`) +- `DefaultDownloadFilename`: Default filename for downloads when parsing fails (`"download.zip"`) + +### SuperHackersConstants Class + +Constants for The Super Hackers content discovery and manifest creation. + +- `PublisherPrefix`: Publisher prefix string (`"thesuperhackers"`) +- `PublisherDisplayName`: Display name for the publisher (`"The Super Hackers"`) +- `VersionDelimiter`: Character used to separate components in version strings (`':'`) +--- + +## AODMapsConstants Class + +Constants for AODMaps (Art of Defense Maps) provider. + +### Publisher & Source Information + +| Constant | Value | Description | +| ------------------------ | ------------ | ---------------------------------------- | +| `PublisherType` | `"aodmaps"` | Publisher type identifier | +| `DiscovererSourceName` | `"AODMaps"` | Source name for discoverer | +| `DiscovererDescription` | `"Art of Defense Maps"` | Display description | +| `ResolverId` | `"AODMaps"` | Resolver identifier | + +### URLs & Page Patterns + +| Constant | Value | +| ------------------------ | --------------------------------------------------- | +| `BaseUrl` | `"https://aodmaps.com"` | +| `PlayerPagePattern` | `"https://aodmaps.com/Players/{0}_players{1}.html"` | +| `AoaMapsUrl` | `"https://aodmaps.com/AOA/aoamaps.html"` | +| `RaceMapsUrl` | `"https://aodmaps.com/race/racemaps.html"` | +| `AirMapsUrl` | `"https://aodmaps.com/air/airmaps.html"` | +| `ContraAodUrl` | `"https://aodmaps.com/ContraAOD/ContraAOD.html"` | + +### CSS Selectors + +| Constant | Selector/Value | +| -------------------------------- | ------------------------------- | +| `GallerySelector` | `"#gallery ul.nospace.clear"` | +| `DetailsPageDescriptionSelector` | `"#description"` | +| `NameSelector` | `".resource-header h1"` | + +--- + +## CommunityOutpostConstants Class + +Constants for the Community Outpost content provider. + +| Constant | Value | +| ---------------------- | ----------------------------------------------------- | +| `PublisherId` | `"community-outpost"` | +| `PublisherType` | `"communityoutpost"` | +| `PublisherName` | `"Community Outpost"` | +| `LogoSource` | `"avares://GenHub/Assets/Logos/communityoutpost-logo.png"` | +| `ProviderDescription` | `"Official patches, tools, and addons from GenPatcher"` | + +--- + +## ModDBParserConstants Class + +CSS selectors for the `ModDBPageParser`. + +### Page Type Detection + +| Constant | Selector | +| ------------------------ | -------------------- | +| `ArticlesBrowseSelector` | `"#articlesbrowse"` | +| `DownloadsInfoSelector` | `"#downloadsinfo"` | +| `TableSelector` | `".table"` | + +### Global Context + +| Constant | Selector | +| --------------------- | --------------------------------------------- | +| `HeaderBoxSelector` | `".headerbox"` | +| `TitleSelector` | `"h1, h2, .title"` | +| `DeveloperSelector` | `"a[href*='/members/'], a[href*='/company/']"` | + --- ## Related Documentation diff --git a/docs/dev/contribution-guidelines.md b/docs/dev/contribution-guidelines.md new file mode 100644 index 000000000..cba01be1d --- /dev/null +++ b/docs/dev/contribution-guidelines.md @@ -0,0 +1,45 @@ +# Contributor Guidelines + +This document outlines the standards and patterns for contributing to the GenHub documentation system. + +## Documentation Structure + +GenHub uses **VitePress** for documentation. The structure is organized as follows: + +- `/docs`: Root directory for all documentation. +- `/docs/dev`: Technical documentation for developers (constants, models, converters). +- `/docs/features`: Detailed descriptions of application features. +- `/docs/FlowCharts`: Mermaid-based flowcharts representing system logic. +- `/docs/.vitepress/config.js`: Central configuration for the sidebar and navigation. + +## Standardized Patterns + +### Constants Documentation + +When adding new constants to the codebase: + +1. Update the corresponding C# class in `GenHub.Core/Constants`. +2. Update `docs/dev/constants.md` by adding a new `## ClassName Class` section. +3. Use markdown tables for listing constants and their values/descriptions. + +### Model Documentation + +When adding or modifying data models: + +1. Update `docs/dev/models.md`. +2. Include C# record/class snippets for clarity. +3. Explain the *Purpose* of the model if it's not immediately obvious. + +### Mermaid Flowcharts + +Flowcharts use the `vitepress-plugin-mermaid`. + +- Themes are customized in `.vitepress/config.js`. +- Use `graph TD` for top-down logic. +- Maintain consistent `classDef` styles for Orchestrators, Providers, and Components. + +## Maintenance Guidelines + +1. **Scope of Changes**: Always ensure that documentation updates match the scope of code changes (e.g., if a new provider is added, document it in both `features/` and `dev/` sections). +2. **Cross-Referencing**: Link to other parts of the documentation using relative links (e.g., `[Content Pipeline](../features/content.md)`). +3. **Diagram Updates**: If logic flows change (e.g., adding a new step to content resolution), update the corresponding mermaid diagram in `docs/FlowCharts/`. diff --git a/docs/dev/manifest-id-system.md b/docs/dev/manifest-id-system.md index 00119e839..638d0fdee 100644 --- a/docs/dev/manifest-id-system.md +++ b/docs/dev/manifest-id-system.md @@ -68,6 +68,64 @@ This normalization ensures the manifest ID schema remains valid (dots separate s **Publisher Attribution**: Community publisher name (e.g., "genhub", "generalsonline", "cnclabs") +### ModDB Format + +**Format**: `{schemaVersion}.{dateVersion}.{publisher}.{contentType}.{contentName}` + +**Examples**: + +- ModDB Addon (release date 2025-01-20): `1.20250120.moddb.addon.supercolors-newcolors` +- ModDB Mod (release date 2024-12-15): `1.20241215.moddb.mod.contra-007` +- ModDB Map Pack (release date 2025-01-10): `1.20250110.moddb.mappack.desert-storm-collection` + +**Key Points**: + +- **Date-based versioning**: Uses the release date (YYYYMMDD format) as the version component +- **No semantic version parsing**: ModDB content titles are not parsed for semantic versions (v1.0, etc.) +- **Deterministic**: Same content + same release date = same manifest ID +- **Publisher identifier**: Always uses "moddb" as the publisher +- **Date source**: Extracted from the ModDB page's release date metadata during content discovery + +**Version Fallback Priority**: + +When generating manifest IDs for content, the version is determined by the following priority order: + +1. **Semantic version** (when explicitly provided by the publisher or detected in release tags like "v2.3") +2. **Release date** (default for ModDB, CNCLabs, and AODMaps) +3. **Upload date** (when release date is unavailable) +4. **Discovery date** (fallback when no other date information is available) + +This fallback hierarchy ensures that content always has a valid version component for the manifest ID, with preference given to publisher-provided semantic versions when available. + +## ModDB Manifest IDs + +ModDB content uses a deterministic ID format based on release dates. + +### Format + +```text +{schemaVersion}.{dateVersion}.{publisher}.{contentType}.{contentName} +``` + +### Example + +```text +1.20250120.moddb.addon.supercolors-newcolors +``` + +### Components + +- **schemaVersion**: Always `1` +- **dateVersion**: Release date in `YYYYMMDD` format +- **publisher**: Always `moddb` +- **contentType**: Content type (mod, addon, map, etc.) +- **contentName**: Normalized content name + +### Key Points + +- Release date is extracted from ModDB's "Added" field +- No semantic version parsing from titles (conservative approach) +- Deterministic: same content + same date = same ID ## API Reference @@ -201,6 +259,20 @@ if (clientResult.Success) { ManifestId id = clientResult.Data; // 1.0.generalsonline.gameclient.generalsonline_30hz } + +// Generate ID for ModDB content with date-based version +var moddbResult = _manifestIdService.GeneratePublisherContentId("moddb", ContentType.Addon, "supercolors-newcolors", 20250120); +if (moddbResult.Success) +{ + ManifestId id = moddbResult.Data; // 1.20250120.moddb.addon.supercolors-newcolors +} + +// Generate ID for ModDB modpack with date version +var moddbModResult = _manifestIdService.GeneratePublisherContentId("moddb", ContentType.Mod, "contra-007", 20241215); +if (moddbModResult.Success) +{ + ManifestId id = moddbModResult.Data; // 1.20241215.moddb.mod.contra-007 +} ``` ### Validation @@ -230,7 +302,7 @@ The `NormalizeVersionString()` method processes version values as follows: ### Examples | Input Version | Normalized Output | Resulting Manifest ID | -|--------------|-------------------|----------------------| +| :--- | :--- | :--- | | `0` | `"0"` | `1.0.steam.gameinstallation.generals` | | `1` | `"1"` | `1.1.steam.gameinstallation.generals` | | `"1.08"` | `"108"` | `1.108.steam.gameinstallation.generals` | @@ -298,6 +370,109 @@ NormalizeVersionString("v1.08"); // โŒ Contains letters NormalizeVersionString("1..08"); // โŒ Results in "108" but has invalid format ``` +## ContentState Integration + +The Manifest ID system is deeply integrated with the ContentState tracking system to detect content updates and manage installation states. + +### State Detection through Manifest IDs + +ContentState uses manifest IDs as the primary key for tracking content across different publishers. The system supports: + +- **Installed state**: Content that has been downloaded and installed +- **UpdateAvailable state**: A newer version of existing content is available +- **Available state**: Content that can be installed but is not currently installed + +### Prefix Matching for Update Detection + +The system uses prefix matching to detect updates for content with date-based versioning (like ModDB): + +```csharp +// Example: Detecting updates for ModDB content +// Installed: 1.20250110.moddb.addon.supercolors-newcolors +// Available: 1.20250120.moddb.addon.supercolors-newcolors + +// The system compares: +// - Schema version (1) - must match +// - Publisher (moddb) - must match +// - Content type (addon) - must match +// - Content name (supercolors-newcolors) - must match +// - Version (20250110 vs 20250120) - used to determine if newer + +// Since the base ID (excluding version) matches and the available version +// is newer (higher date), the state is set to UpdateAvailable +``` + +### ID Comparison Logic + +The manifest ID comparison for update detection follows this logic: + +1. **Extract base ID**: Remove the version component to get the content signature + - From `1.20250110.moddb.addon.supercolors-newcolors` + - Base: `moddb.addon.supercolors-newcolors` + +2. **Compare signatures**: Check if installed and available content have the same base + - If base IDs match โ†’ same content, compare versions + - If base IDs differ โ†’ different content entirely + +3. **Version comparison**: For matching base IDs, determine if update available + - **Date-based versions** (YYYYMMDD): Higher numeric value = newer version + - **Semantic versions** (normalized): Standard semantic version comparison + - **Integer versions**: Higher integer value = newer version + +### Practical Example + +```csharp +// Scenario: ModDB content update detection + +// 1. User installs "Super Colors" addon on January 10, 2025 +var installedId = "1.20250110.moddb.addon.supercolors-newcolors"; +var manifest = new ContentManifest +{ + Id = installedId, + State = ContentState.Installed, + // ... other properties +}; + +// 2. System discovers updated version released on January 20, 2025 +var availableId = "1.20250120.moddb.addon.supercolors-newcolors"; +var discoveredManifest = new ContentManifest +{ + Id = availableId, + State = ContentState.Available, + // ... other properties +}; + +// 3. ContentStateService detects update: +// - Base IDs match: "moddb.addon.supercolors-newcolors" +// - Version comparison: 20250120 > 20250110 +// - Result: State set to ContentState.UpdateAvailable + +// 4. UI shows "Update Available" indicator on the content card +// User can click to download and install the newer version +``` + +### Publisher-Specific Behavior + +Different publishers interact with the ContentState system differently: + +**ModDB (Date-based versioning)**: + +- Each new release date creates a new manifest ID +- Updates detected when same content has newer release date +- Historical versions tracked separately (different IDs) + +**GitHub (Semantic versioning)**: + +- Explicit version tags (v1.0, v2.0) used in manifest ID +- Updates follow semantic version rules +- Pre-release handling supported (beta, alpha tags) + +**Creator Publishing (User-specified versions)**: + +- Publisher defines version in catalog JSON +- Semantic or date-based at publisher discretion +- System respects publisher's version scheme + ## Validation Rules ### All Content (5-Segment Format) @@ -308,6 +483,7 @@ NormalizeVersionString("1..08"); // โŒ Results in "108" but has invalid format - **ContentType**: Must be valid content type (gameinstallation, gameclient, mod, patch, addon, mappack, languagepack, moddingtool, etc.) - **ContentName**: Alphanumeric with dashes (e.g., "generals", "custom-mod") - **Total Segments**: Exactly 5 segments required + ## Error Handling Uses **ResultBase pattern** for robust error handling: diff --git a/docs/dev/models.md b/docs/dev/models.md index 70cfdbf0e..4052e24af 100644 --- a/docs/dev/models.md +++ b/docs/dev/models.md @@ -97,7 +97,7 @@ Comprehensive manifest for content distribution in GenHub ecosystem. ```csharp public class ContentManifest { - public string ManifestVersion { get; set; } + public string SchemaVersion { get; set; } public ManifestId Id { get; set; } public string Name { get; set; } public string Version { get; set; } @@ -683,3 +683,71 @@ public class ValidationIssue ``` This ensures thread safety and prevents accidental modification of model state. + +--- + +## Universal Parser Models + +Models used by the `IWebPageParser` system to extract rich content from provider websites. + +### ParsedWebPage + +The root container for all data extracted from a single web page. + +```csharp +public record ParsedWebPage( + string Url, + GlobalContext Context, + List Sections, + PageType PageType); +``` + +### GlobalContext + +Standard metadata extracted from the page header or sidebar. + +```csharp +public record GlobalContext( + string Title, + string Developer, + DateTime? ReleaseDate, + string? GameName = null, + string? IconUrl = null, + string? Description = null); +``` + +### Content Sections + +All extracted content is categorized into sections that inherit from `ContentSection`. + +| Model | Description | +| --------- | ----------------------------------------------------- | +| `Article` | News posts, articles, or blog entries | +| `File` | Downloadable files with metadata (size, hash, etc.) | +| `Video` | Embedded videos from YouTube, Vimeo, etc. | +| `Image` | Gallery images or screenshots | +| `Review` | User reviews with ratings and content | +| `Comment` | User discussion comments with karma/creator info | + +#### ContentSection (Base) + +```csharp +public abstract record ContentSection( + SectionType Type, + string Title); +``` + +### Enums + +#### PageType + +Defines the structural role of the page. + +- `List`: A gallery or listing of multiple items. +- `Summary`: A news feed or overview page. +- `Detail`: A deep-dive page for a specific mod or addon. +- `FileDetail`: A targeted page for a specific file download. + +#### SectionType + +Identifies the type of a `ContentSection` (Article, Video, Image, File, Review, Comment). diff --git a/docs/features/content/content-pipeline.md b/docs/features/content/content-pipeline.md new file mode 100644 index 000000000..627286334 --- /dev/null +++ b/docs/features/content/content-pipeline.md @@ -0,0 +1,427 @@ +--- +title: Content Pipeline Architecture +description: Detailed documentation of the GenHub three-tier content pipeline for discovering, resolving, and acquiring content +--- + +# Content Pipeline Architecture + +The GenHub content system uses a **three-tier pipeline architecture** that transforms external content sources into installable content with full manifest and CAS (Content-Addressable Storage) integration. + +## Pipeline Overview + +```mermaid +flowchart TB + subgraph "Tier 1: Orchestration" + CO["ContentOrchestrator"] + end + + subgraph "Tier 2: Providers" + BP["BaseContentProvider"] + MDP["ModDBContentProvider"] + CLP["CNCLabsContentProvider"] + GOP["GeneralsOnlineProvider"] + end + + subgraph "Tier 3: Components" + D["Discoverers"] + P["Parsers"] + R["Resolvers"] + DEL["Deliverers"] + MF["ManifestFactories"] + end + + subgraph "Storage" + CAS["CAS Storage"] + MP["Manifest Pool"] + end + + CO --> BP + BP --> D + D --> P + P --> R + R --> DEL + DEL --> MF + MF --> CAS + MF --> MP +``` + +## Tier 1: ContentOrchestrator + +**Location**: `GenHub.Core/Interfaces/Content/IContentOrchestrator.cs` + +The orchestrator is the system-wide coordinator for all content operations. + +### Responsibilities + +| Operation | Method | Description | +|-----------|--------|-------------| +| **Search** | `SearchAsync()` | Broadcasts query to all providers, aggregates results | +| **Acquire** | `AcquireContentAsync()` | Downloads, extracts, stores, and registers content | +| **Cache** | `IDynamicContentCache` | System-wide caching for performance | + +### Search Flow + +```csharp +// User initiates search in DownloadsBrowserView +var results = await _orchestrator.SearchAsync(new ContentSearchQuery +{ + SearchTerm = "Rise of the Reds", + ContentType = ContentType.Mod, + TargetGame = GameType.ZeroHour +}); +``` + +--- + +## Tier 2: Content Providers + +**Base**: `GenHub/Features/Content/Services/ContentProviders/BaseContentProvider.cs` + +Providers are source-specific facades that orchestrate the internal pipeline. + +### Provider Pattern + +```csharp +public abstract class BaseContentProvider : IContentProvider +{ + protected abstract IContentDiscoverer Discoverer { get; } + protected abstract IContentResolver Resolver { get; } + protected abstract IContentDeliverer Deliverer { get; } + + // Common pipeline orchestration + public virtual async Task>> SearchAsync( + ContentSearchQuery query, CancellationToken cancellationToken = default) + { + var providerDefinition = GetProviderDefinition(); + return await Discoverer.DiscoverAsync(providerDefinition, query, cancellationToken); + } +} +``` + +### Registered Providers + +| Provider | Discoverer | Parser | Notes | +|----------|------------|--------|-------| +| **ModDB** | `ModDBDiscoverer` | `ModDBPageParser` (AngleSharp) | Uses Playwright for WAF bypass | +| **CNC Labs** | `CNCLabsMapDiscoverer` | AngleSharp HTML | Direct HTTP scraping | +| **AOD Maps** | `AODMapsDiscoverer` | `AODMapsPageParser` | Pagination support | +| **Community Outpost** | `CommunityOutpostDiscoverer` | `GenPatcherDatCatalogParser` | `.dat` catalog format | +| **GitHub** | `GitHubDiscoverer` | GitHub API JSON | Release assets | +| **Generals Online** | `GeneralsOnlineDiscoverer` | GitHub API | Multi-variant releases | +| **File System** | `FileSystemDiscoverer` | Direct scan | Local manifests | + +--- + +## Tier 3: Pipeline Components + +### Discoverers (`IContentDiscoverer`) + +**Location**: `GenHub.Core/Interfaces/Content/IContentDiscoverer.cs` + +Discoverers fetch catalog data from external sources and delegate to parsers. + +```csharp +public interface IContentDiscoverer : IContentSource +{ + Task> DiscoverAsync( + ContentSearchQuery query, + CancellationToken cancellationToken = default); + + // Overload with provider definition for data-driven configuration + Task> DiscoverAsync( + ProviderDefinition? provider, + ContentSearchQuery query, + CancellationToken cancellationToken = default); +} +``` + +**Key principle**: Discoverers handle network concerns (timeouts, retries, WAF bypass) but do NOT parse data themselvesโ€”that's the parser's job. + +### Parsers (`ICatalogParser`, `IWebPageParser`) + +**Locations**: + +- `GenHub.Core/Interfaces/Providers/ICatalogParser.cs` +- `GenHub.Core/Interfaces/Parsers/IWebPageParser.cs` + +Parsers transform raw data (HTML, JSON, `.dat` files) into `ContentSearchResult` objects. + +| Parser | Format | Source | +|--------|--------|--------| +| `GenPatcherDatCatalogParser` | `.dat` pipe-delimited | Community Outpost | +| `ModDBPageParser` | HTML | ModDB | +| `AODMapsPageParser` | HTML | AOD Maps | +| AngleSharp | Generic HTML | CNC Labs | + +### Resolvers (`IContentResolver`) + +**Location**: `GenHub.Core/Interfaces/Content/IContentResolver.cs` + +Resolvers transform lightweight search results into complete `ContentManifest` blueprints. + +```csharp +public interface IContentResolver : IContentSource +{ + Task> ResolveAsync( + ContentSearchResult discoveredItem, + CancellationToken cancellationToken = default); +} +``` + +**Resolution tasks**: + +1. Fetch detail page for full metadata (description, screenshots) +2. Extract download URL +3. Determine target game and content type +4. Build manifest structure + +### Deliverers (`IContentDeliverer`) + +**Location**: `GenHub.Core/Interfaces/Content/IContentDeliverer.cs` + +Deliverers download content files and prepare them for storage. + +```csharp +public interface IContentDeliverer : IContentSource +{ + bool CanDeliver(ContentManifest manifest); + + Task> DeliverContentAsync( + ContentManifest manifest, + string targetDirectory, + CancellationToken cancellationToken = default); +} +``` + +#### CAS-Resident Content (Skip Delivery) + +Several publisher manifest factories (`AODMapsManifestFactory`, `CNCLabsManifestFactory`, +`ModDBManifestFactory`) download the file into CAS **during resolution** and register it as a +`ContentAddressable` file with a hash but no `DownloadUrl`. The HTTP deliverer cannot handle such +files (`CanDeliver` requires an http download URL), so the matching content providers short-circuit +preparation: when **every** file is already `ContentAddressable` with a hash, the provider returns the +manifest as-is and the orchestrator's delivery stage is skipped. `ContentValidator` and +`ContentStorageService` both resolve such files against CAS by hash (not the staging folder), so the +file is found end-to-end. A pool-agnostic hash lookup is used as a fallback so files stored under one +content-type pool are still found when the manifest reports another. + +### Manifest Factories (`IContentManifestFactory`) + +**Location**: `GenHub.Core/Interfaces/Manifest/IContentManifestFactory.cs` + +Factories create proper `ContentManifest` objects after downloading, handling publisher-specific logic. + +| Factory | Publisher | Features | +|---------|-----------|----------| +| `ModDBManifestFactory` | ModDB | ID format: `1.YYYYMMDD.moddb.{type}.{name}` | +| `CNCLabsManifestFactory` | CNC Labs | Map-specific metadata | +| `AODMapsManifestFactory` | AOD Maps | Referer header handling | +| `GitHubManifestFactory` | GitHub | Release asset handling | +| `SuperHackersManifestFactory` | The Super Hackers | Multi-game releases (Generals + ZH) | + +--- + +## Archive Handling + +The `ContentManifestBuilder.AddDownloadedFileAsync()` method automatically handles archives: + +```mermaid +flowchart TD + DL["Download to Temp"] --> CHECK{"Is Archive?
(by file signature)"} + CHECK -->|"Yes (ZIP/RAR/7z)"| EXTRACT["Extract All Files"] + EXTRACT --> HASH["Hash Each File"] + HASH --> CAS["Store in CAS"] + CAS --> MANIFEST["Add to Manifest
as ContentAddressable"] + + CHECK -->|"No"| HASHSINGLE["Hash Single File"] + HASHSINGLE --> CASSINGLE["Store in CAS"] + CASSINGLE --> MANIFESTSINGLE["Add Single Entry"] +``` + +**Supported formats**: ZIP, RAR, 7z, TAR, GZ (via SharpCompress library) + +**Detection**: By file signature (magic bytes), NOT file extension + +--- + +## ContentManifest Builder + +**Location**: `GenHub/Features/Manifest/ContentManifestBuilder.cs` + +The fluent builder API for manifest creation: + +```csharp +var manifest = manifestBuilder + .WithBasicInfo(publisherId, contentName, manifestVersion) + .WithContentType(ContentType.Mod, GameType.ZeroHour) + .WithPublisher( + name: "ModDB - Author Name", + website: "https://moddb.com", + publisherType: "moddb") + .WithMetadata( + description: details.Description, + tags: ["mod", "zerohour"], + iconUrl: details.PreviewImage) + .Build(); + +// Add downloaded file (handles archive extraction automatically) +await manifest.AddDownloadedFileAsync( + relativePath: "content.zip", + downloadUrl: "https://example.com/download", + refererUrl: detailPageUrl, // For sites requiring referer + userAgent: customUserAgent); // Triggers Playwright if set +``` + +### Key Methods + +| Method | Purpose | +|--------|---------| +| `AddDownloadedFileAsync()` | Downloads, extracts archives, stores in CAS | +| `AddFilesFromDirectoryAsync()` | Scans directory, hashes files, adds to manifest | +| `AddLocalFileAsync()` | Adds existing local file | +| `AddContentAddressableFileAsync()` | Adds CAS reference by hash | +| `AddDependency()` | Adds content dependency | + +--- + +## Manifest ID System + +**Documentation**: [manifest-id-system.md](../../dev/manifest-id-system.md) + +IDs follow a deterministic format: + +``` +{version}.{userVersion}.{publisherId}.{contentType}.{contentName} +``` + +**Examples**: + +- `1.20190826.moddb.mod.hanpatchv32` - ModDB mod +- `1.0.zerohour.gameinstallation` - Base game + +--- + +## Downloads View Integration + +### User Flow + +```mermaid +sequenceDiagram + actor User + participant SB as PublisherSidebar + participant VM as DownloadsBrowserViewModel + participant D as Discoverer + participant UI as ContentGrid + + User->>SB: Select "ModDB" + SB->>VM: SetPublisher(ModDB) + VM->>D: DiscoverAsync(query) + D-->>VM: ContentDiscoveryResult + VM->>UI: Update ContentItems + User->>UI: Click content card + UI->>VM: OpenDetail(item) +``` + +### Acquisition Flow + +```mermaid +sequenceDiagram + actor User + participant Detail as ContentDetailView + participant CO as ContentOrchestrator + participant Resolver as ContentResolver + participant Factory as ManifestFactory + participant CAS as CAS Service + participant Pool as ManifestPool + + User->>Detail: Click "Download" + Detail->>CO: AcquireContentAsync(item) + CO->>Resolver: ResolveAsync(searchResult) + Resolver-->>CO: Full details + download URL + CO->>Factory: CreateManifestAsync(details) + Factory->>Factory: AddDownloadedFileAsync() + Factory->>CAS: StoreContentAsync(files) + Factory-->>CO: ContentManifest + CO->>Pool: AddManifest(manifest) + CO-->>Detail: Success +``` + +--- + +## Per-Publisher Implementation Checklist + +To add support for a new publisher: + +### 1. Create Constants + +```csharp +// GenHub.Core/Constants/MyPublisherConstants.cs +public static class MyPublisherConstants +{ + public const string PublisherPrefix = "mypub"; + public const string PublisherName = "My Publisher"; + public const string PublisherWebsite = "https://mypub.example.com"; +} +``` + +### 2. Create Discoverer + +```csharp +public class MyPublisherDiscoverer : IContentDiscoverer +{ + public async Task> DiscoverAsync( + ContentSearchQuery query, CancellationToken ct) + { + // 1. Fetch catalog from source + // 2. Parse into ContentSearchResult objects + // 3. Apply query filters + return OperationResult.CreateSuccess( + new ContentDiscoveryResult { Items = results }); + } +} +``` + +### 3. Create Resolver (if needed) + +```csharp +public class MyPublisherResolver : IContentResolver +{ + public async Task> ResolveAsync( + ContentSearchResult item, CancellationToken ct) + { + // Fetch detail page, build full manifest + } +} +``` + +### 4. Create Manifest Factory + +```csharp +public class MyPublisherManifestFactory : IContentManifestFactory +{ + public bool CanHandle(ContentManifest manifest) => + manifest.Publisher.Name.Contains("My Publisher"); + + public async Task CreateManifestAsync(...) + { + // Build manifest with AddDownloadedFileAsync() + } +} +``` + +### 5. Register in DI + +```csharp +// ContentPipelineModule.cs +services.AddTransient(); +services.AddTransient(); +``` + +--- + +## Related Documentation + +- [Publisher Configuration](./publisher-configuration.md) - Data-driven publisher settings +- [Discovery Flow](../../FlowCharts/Discovery-Flow.md) - Visual discovery workflow +- [Manifest ID System](../../dev/manifest-id-system.md) - ID generation rules +- [Publisher Infrastructure](./publisher-infrastructure.md) - Clean architecture for content publishers diff --git a/docs/features/content/index.md b/docs/features/content/index.md index d07e0e7eb..a8e0dc8aa 100644 --- a/docs/features/content/index.md +++ b/docs/features/content/index.md @@ -3,21 +3,25 @@ title: Content System description: Documentation for GenHub content management features --- -# Content Features +## Content Features The GenHub content system provides a flexible, extensible architecture for discovering, acquiring, and managing game content from various sources. ## Core Documentation +- [Content Pipeline Architecture](./content-pipeline.md) - Three-tier pipeline for discovering, resolving, and acquiring content - [Publisher Configuration](./publisher-configuration.md) - Data-driven publisher configuration for flexible content pipeline customization - [Publisher Infrastructure](./publisher-infrastructure.md) - Extensible architecture for publisher-specific content handling +- [Content Dependencies](./content-dependencies.md) - Dependency system for mods and content packages +- [Universal Parser](./universal-parser.md) - Unified parsing system for web content +- [Downloads Flow](../../FlowCharts/Downloads-Flow.md) - User journey from browsing to installation ## Architecture The content system follows a layered architecture with clear separation of concerns: 1. **Content Orchestrator**: Coordinates all content operations -2. **Content Providers**: Publisher-specific facades (GitHub, CNCLabs, ModDB) +2. **Content Providers**: Publisher-specific facades (GitHub, CNCLabs, AODMaps, ModDB, Community Outpost) 3. **Pipeline Components**: - **Discoverers**: Find available content - **Resolvers**: Transform lightweight results into full manifests @@ -31,8 +35,11 @@ The content system follows a layered architecture with clear separation of conce - GitHub releases - CNCLabs maps +- AODMaps (Art of Defense Maps) +- Community Outpost (GenPatcher) +- ModDB (mods, addons, patches, maps, skins, videos, modding tools) - Local file system -- Future: ModDB, Steam Workshop +- Future: Steam Workshop ### Publisher-Agnostic Architecture @@ -97,13 +104,15 @@ The Publisher Manifest Factory pattern enables extensible content handling: 1. **IPublisherManifestFactory**: Interface for factory implementations 2. **SuperHackersManifestFactory**: Handles multi-game releases -3. **PublisherManifestFactoryResolver**: Selects appropriate factory +3. **ModDBManifestFactory**: Handles ModDB content with date-based versioning +4. **PublisherManifestFactoryResolver**: Selects appropriate factory ### Factory Selection Factories self-identify via `CanHandle(manifest)`: - SuperHackers GameClient โ†’ SuperHackersManifestFactory +- ModDB content (mods, addons, patches, maps, etc.) โ†’ ModDBManifestFactory - Custom publishers โ†’ Custom factories (when implemented) ### Benefits @@ -123,6 +132,7 @@ Content is stored in the **Content Pool**: - Deterministic ManifestId generation - Hash-based validation - Duplicate detection +- Content caching for parsed web pages (ContentCacheService) ## Integration Points @@ -161,9 +171,105 @@ To add support for a new publisher: See [Publisher Infrastructure](./publisher-infrastructure.md) for detailed implementation guidance. +## Implemented Content Providers + +### ModDB Provider + +The ModDB content provider enables discovery and acquisition of game content from ModDB.com: + +**Capabilities:** + +- **Content Types**: Mods, patches, maps, addons, skins, videos, modding tools, language packs +- **Discovery**: Playwright-based browser automation to bypass WAF/bot protections +- **Parsing**: AngleSharp-based HTML parser for extracting rich content metadata +- **Multi-Section Support**: Searches both Downloads and Addons sections +- **Rich Metadata**: Extracts files, videos, images, articles, reviews, and comments + +**Architecture:** + +- **ModDBDiscoverer**: Uses Playwright to fetch listing pages with browser automation +- **ModDBPageParser**: Universal web page parser supporting multiple page types (detail, list, file detail) +- **ModDBResolver**: Transforms discovered items into content manifests using parsed data +- **ModDBManifestFactory**: Generates manifest IDs with release-date versioning (format: `1.YYYYMMDD.moddb.{contentType}.{contentName}`) + +**Key Features:** + +- WAF bypass using headless Chromium browser +- Separate manifest creation for each file based on FileSectionType +- Release date extraction for accurate version tracking +- Support for ModDB's pagination and filtering (category, license, timeframe) +- Content caching to avoid repeated web fetches + +### Provider Comparison + +| Provider | Content Types | Discovery Method | Versioning | Factory | +| :--- | :--- | :--- | :--- | :--- | +| GitHub | Mods, GameClients, Patches | API | Semantic (tags) | GitHubManifestFactory | +| CNCLabs | Maps | Web scraping | Date-based | CNCLabsManifestFactory | +| AODMaps | Maps | Web scraping | Date-based | AODMapsManifestFactory | +| ModDB | Mods, Addons, Patches, Maps, Skins, Videos, Tools | Playwright + AngleSharp | Date-based (YYYYMMDD) | ModDBManifestFactory | +| Community Outpost | Patches | API | Semantic | CommunityOutpostFactory | + +## Downloads UI Integration + +The content pipeline directly feeds the Downloads browser, enabling users to discover, install, and manage game content. + +### Content State Service + +The **ContentStateService** determines the current state of content for UI display: + +- **NotDownloaded**: Content has not been downloaded. Show "Download" button +- **UpdateAvailable**: Content exists locally but a newer version is available. Show "Update" button +- **Downloaded**: Content is downloaded and up-to-date. Show "Add to Profile" dropdown + +**State Detection:** + +- Generates prospective manifest IDs using `ManifestIdGenerator` +- Checks manifest pool for exact matches +- Searches for older versions by comparing publisher, content type, and content name +- Uses release date (yyyyMMdd) for version comparison + +### Manifest ID Generation + +The **ManifestIdGenerator** creates deterministic, human-readable manifest IDs following a 5-segment format: + +```text +schemaVersion.userVersion.publisher.contentType.contentName +``` + +**Examples:** + +- `1.20240315.moddb.mod.contra` (ModDB content with date versioning) +- `1.0.themodders.gameclient.generals` (Publisher content with semantic versioning) +- `1.108.ea.gameinstallation.zerohour` (Game installation) + +**Benefits:** + +- Consistent parsing and validation across the system +- Hierarchical organization for efficient querying +- Unique identification across publishers and content types +- Schema versioning support for future format evolution +- Human-readable format for debugging and logging + +### Profile Selection Integration + +The Downloads UI integrates with the game profile system: + +- Users can add downloaded content to specific game profiles +- Profile selection dropdown shown for downloaded content +- Content references stored in profile configuration via ManifestId +- Automatic dependency resolution during profile setup + +### Content Acquisition Flow + +1. **Discovery**: Users browse content from various providers (GitHub, ModDB, CNCLabs, etc.) +2. **Resolution**: Selecting content triggers resolution to full manifest +3. **State Check**: ContentStateService determines current state (NotDownloaded/UpdateAvailable/Downloaded) +4. **Acquisition**: Download/update triggers content pipeline execution +5. **Profile Assignment**: Users assign content to game profiles for deployment + ## Future Enhancements -- [ ] ModDB content provider - [ ] Steam Workshop integration - [ ] Automatic content updates - [ ] Content dependency resolution diff --git a/docs/features/content/publisher-configuration.md b/docs/features/content/publisher-configuration.md index 6db1c91c0..759618fbd 100644 --- a/docs/features/content/publisher-configuration.md +++ b/docs/features/content/publisher-configuration.md @@ -428,7 +428,7 @@ public interface IPublisherDefinitionLoader Static publishers have a fixed publisher identity. All content discovered from the source is attributed to a single known publisher. -**Examples**: Community Outpost, Generals Online, TheSuperHackers +**Examples**: Community Outpost, AODMaps, Generals Online, TheSuperHackers ```json { @@ -454,6 +454,24 @@ Dynamic publishers support multiple publishers where content authors become indi } ``` +#### AODMaps Configuration + +AODMaps uses a static publisher configuration to map its custom catalog format: + +```json +{ + "publisherId": "aodmaps", + "publisherType": "aodmaps", + "displayName": "Art of Defense Maps", + "providerType": "Static", + "catalogFormat": "html-scraping", + "endpoints": { + "catalogUrl": "https://aodmaps.com", + "websiteUrl": "https://aodmaps.com" + } +} +``` + ## Benefits | Feature | Description | diff --git a/docs/features/content/universal-parser.md b/docs/features/content/universal-parser.md new file mode 100644 index 000000000..f01d75c8d --- /dev/null +++ b/docs/features/content/universal-parser.md @@ -0,0 +1,441 @@ +# Universal Web Page Parser Architecture + +## Overview + +The Universal Web Page Parser is a provider-agnostic architecture for extracting rich content from web pages. It enables content providers like ModDB, AODMaps, and others to parse web pages and extract structured data including articles, videos, images, files, reviews, and comments. + +## Architecture + +### Core Components + +#### 1. Data Models (`GenHub.Core/Models/Parsers/`) + +All parser data models are defined in the `GenHub.Core` project for maximum reusability. + +##### `PageType` Enum + +Defines the different types of pages that can be parsed: + +- `Unknown` - Page type could not be determined +- `List` - Gallery or list view (e.g., addons, images) +- `Summary` - News feed or summary page +- `Detail` - Full detail page with all content sections +- `FileDetail` - Specific file download page + +##### `SectionType` Enum + +Defines the different types of content sections: + +- `Article` - News articles or blog posts +- `Video` - Embedded videos (YouTube, Vimeo, etc.) +- `Image` - Images and screenshots +- `File` - Downloadable files +- `Review` - User reviews with ratings +- `Comment` - User comments + +##### `GlobalContext` Record + +Contains global information about the page: + +```csharp +public record GlobalContext( + string Title, + string Developer, + DateTime? ReleaseDate, + string? GameName = null, + string? IconUrl = null, + string? Description = null +); +``` + +##### `ContentSection` (Abstract Base Class) + +Base class for all content sections: + +```csharp +public abstract record ContentSection( + SectionType Type, + string Title +); +``` + +##### Specific Content Type Records + +**Article** - News articles or blog posts: + +```csharp +public record Article( + string Title, + string? Author = null, + DateTime? PublishDate = null, + string? Content = null, + string? Url = null +) : ContentSection(SectionType.Article, Title); +``` + +**Video** - Embedded videos: + +```csharp +public record Video( + string Title, + string? ThumbnailUrl = null, + string? EmbedUrl = null, + string? Platform = null +) : ContentSection(SectionType.Video, Title); +``` + +**Image** - Images and screenshots: + +```csharp +public record Image( + string Title, + string? ThumbnailUrl = null, + string? FullSizeUrl = null, + string? Description = null +) : ContentSection(SectionType.Image, Title); +``` + +**File** - Downloadable files: + +```csharp +public record File( + string Name, + string? Version = null, + long? SizeBytes = null, + string? SizeDisplay = null, + DateTime? UploadDate = null, + string? Category = null, + string? Uploader = null, + string? DownloadUrl = null, + string? Md5Hash = null, + int? CommentCount = null +) : ContentSection(SectionType.File, Title); +``` + +**Review** - User reviews with ratings: + +```csharp +public record Review( + string? Author = null, + float? Rating = null, + string? Content = null, + DateTime? Date = null, + int? HelpfulVotes = null +) : ContentSection(SectionType.Review, "Review"); +``` + +**Comment** - User comments: + +```csharp +public record Comment( + string? Author = null, + string? Content = null, + DateTime? Date = null, + int? Karma = null, + bool? IsCreator = null +) : ContentSection(SectionType.Comment, "Comment"); +``` + +##### `ParsedWebPage` Record + +The complete result of parsing a web page: + +```csharp +public record ParsedWebPage( + string Url, + GlobalContext Context, + List Sections, + PageType PageType +); +``` + +#### 2. Interfaces + +##### `IWebPageParser` (`GenHub.Core/Interfaces/Parsers/IWebPageParser.cs`) + +Universal parser interface that all provider-specific parsers implement: + +```csharp +public interface IWebPageParser +{ + /// + /// Gets the unique identifier for this parser. + /// + string ParserId { get; } + + /// + /// Determines whether this parser can handle the given URL. + /// + /// The URL to check. + /// True if this parser can handle the URL; otherwise, false. + bool CanParse(string url); + + /// + /// Parses the web page at the given URL. + /// + /// The URL to parse. + /// Cancellation token. + /// The parsed web page data. + Task ParseAsync(string url, CancellationToken cancellationToken = default); + + /// + /// Parses the provided HTML content. + /// + /// The URL the HTML was retrieved from. + /// The HTML content to parse. + /// Cancellation token. + /// The parsed web page data. + Task ParseAsync(string url, string html, CancellationToken cancellationToken = default); +} +``` + +##### `IPlaywrightService` (`GenHub.Core/Interfaces/Tools/IPlaywrightService.cs`) + +Service for managing Playwright browser instances: + +```csharp +public interface IPlaywrightService +{ + /// + /// Creates a new Playwright page with optional context options. + /// + Task CreatePageAsync(BrowserNewContextOptions? options = null, CancellationToken cancellationToken = default); + + /// + /// Fetches HTML content from the given URL. + /// + Task FetchHtmlAsync(string url, CancellationToken cancellationToken = default); + + /// + /// Fetches HTML and parses it into an AngleSharp document. + /// + Task FetchAndParseAsync(string url, CancellationToken cancellationToken = default); +} +``` + +#### 3. Implementation + +##### `PlaywrightService` (`GenHub/GenHub/Features/Content/Services/Tools/PlaywrightService.cs`) + +Singleton service that manages a shared Playwright browser instance: + +- Uses a semaphore to ensure thread-safe initialization +- Creates a single browser instance shared across all requests +- Provides realistic user agent to bypass WAF/Bot protections +- Implements `IAsyncDisposable` for proper cleanup + +##### `ModDBPageParser` (`GenHub/GenHub/Features/Content/Services/Parsers/ModDBPageParser.cs`) + +Implementation of `IWebPageParser` for ModDB pages: + +**Features:** + +- Parses three distinct page types: List, Summary, Detail, FileDetail +- Extracts global context from `.headerbox` elements +- Extracts all content sections (articles, videos, images, files, reviews, comments) +- Uses comprehensive CSS selectors defined in `ModDBParserConstants` + +**Page Type Detection:** + +- **List**: URLs ending in `/addons`, `/images`, or containing `.table .row.rowcontent` +- **Summary**: Pages with `#articlesbrowse` element +- **Detail**: Pages with `.headerbox` but no specific list/summary indicators +- **FileDetail**: Pages with `#downloadsinfo` element + +**Content Extraction:** + +- **Files**: Extracts from `.table .row.file` or `tr.file` elements +- **Videos**: Extracts from `iframe` elements with YouTube/Vimeo URLs +- **Images**: Extracts from `.mediarow`, `.screenshot`, or `.imagebox` elements +- **Articles**: Extracts from `.article`, `.newsitem`, or `.post` elements +- **Reviews**: Extracts from `.review` elements with rating information +- **Comments**: Extracts from `.comment` elements with karma/creator badges + +##### `ModDBParserConstants` (`GenHub/GenHub.Core/Constants/ModDBParserConstants.cs`) + +Contains all CSS selectors for ModDB page parsing: + +- Global Context selectors +- Page Type Detection selectors +- Content section selectors (Files, Videos, Images, Articles, Reviews, Comments) +- Pagination selectors +- URL pattern constants + +#### 4. Integration + +##### `ModDBResolver` (`GenHub/GenHub/Features/Content/Services/ContentResolvers/ModDBResolver.cs`) + +Updated to use the universal parser: + +```csharp +public class ModDBResolver( + HttpClient httpClient, + ModDBManifestFactory manifestFactory, + IWebPageParser webPageParser, // Injected via DI + ILogger logger) : IContentResolver +{ + public async Task> ResolveAsync( + ContentSearchResult discoveredItem, + CancellationToken cancellationToken = default) + { + // Parse the web page + var parsedPage = await _webPageParser.ParseAsync( + discoveredItem.SourceUrl, + cancellationToken); + + // Store parsed page in search result for UI display + discoveredItem.SetData(parsedPage); + + // Extract primary download URL + var primaryDownloadUrl = ExtractPrimaryDownloadUrl(parsedPage); + + // Convert to MapDetails for manifest factory + var mapDetails = ConvertToMapDetails(parsedPage, discoveredItem, primaryDownloadUrl); + + // Create manifest + var manifest = await _manifestFactory.CreateManifestAsync( + mapDetails, + discoveredItem.SourceUrl); + + return OperationResult.CreateSuccess(manifest); + } +} +``` + +##### `ContentDetailViewModel` (`GenHub/GenHub/Features/Downloads/ViewModels/ContentDetailViewModel.cs`) + +Updated to display rich content from parsed pages: + +```csharp +public partial class ContentDetailViewModel : ObservableObject +{ + [ObservableProperty] + private ParsedWebPage? _parsedPage; + + public ObservableCollection
Articles => + ParsedPage?.Sections.OfType
().ToObservableCollection() ?? new(); + + public ObservableCollection