From c4b7d32fe931e30abaed4abf5ea688f6d816eac3 Mon Sep 17 00:00:00 2001 From: Amadeusz Sadowski Date: Sun, 15 Mar 2026 02:58:47 +0100 Subject: [PATCH 1/2] Modernize repository: fix workflows, update scripts, improve docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix update-registry.yml branch trigger (master → main) - Remove dead gh-pages code and Update-GhPagesData.ps1 - Replace Amadevus/pwsh-script@v2 with native pwsh run steps - Rename checkout path master → main in update-index.yml - Update GitHub API auth from 'token' to 'Bearer' prefix - Remove deprecated preview API header in Validate-Registry.ps1 - Fix DisableLastModifiedCaching param type ([string] → [switch]) - Parameterize hardcoded BSData org name in Update-Registry.ps1 - Remove dead FormatFileRef function from Validate-Registry.ps1 - Replace Write-Action* calls with native workflow commands - Update powershell-yaml from 0.4.2 to 0.4.7 - Rewrite .gitignore (351-line VS template → 25-line minimal) - Update Twitter URL to X in settings.yml - Improve README with architecture overview and structure - Add CONTRIBUTING.md, PR template, issue template - Add BsdataGallery.psd1 module manifest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/new-repository.yml | 22 ++ .github/PULL_REQUEST_TEMPLATE.md | 17 + .github/actions/install-yaml/action.yml | 2 +- .github/scripts/Update-GhPagesData.ps1 | 121 ------ .github/scripts/Update-Registry.ps1 | 15 +- .github/scripts/Validate-Registry.ps1 | 24 +- .../lib/BsdataGallery/BsdataGallery.psd1 | 16 + .../lib/BsdataGallery/BsdataGallery.psm1 | 6 +- .github/workflows/ci.yml | 31 +- .github/workflows/update-index.yml | 110 ++---- .github/workflows/update-registry.yml | 49 +-- .gitignore | 364 +----------------- CONTRIBUTING.md | 66 ++++ README.md | 39 +- registry/settings.yml | 2 +- 15 files changed, 272 insertions(+), 612 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/new-repository.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/scripts/Update-GhPagesData.ps1 create mode 100644 .github/scripts/lib/BsdataGallery/BsdataGallery.psd1 create mode 100644 CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE/new-repository.yml b/.github/ISSUE_TEMPLATE/new-repository.yml new file mode 100644 index 00000000..54eeec36 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-repository.yml @@ -0,0 +1,22 @@ +name: New Repository Request +description: Request adding a new BattleScribe data repository to the gallery +title: "Add repository: " +labels: ["new-repository"] +body: + - type: input + id: repo-url + attributes: + label: Repository URL + description: Full GitHub URL of the repository to add + placeholder: "https://github.com/owner/repo-name" + validations: + required: true + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: Repository is publicly accessible + required: true + - label: Repository has the `battlescribe-data` topic + - label: Repository has a published release with a `.catpkg.json` asset diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..01268e36 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## Description + + + +## Type of Change + +- [ ] New repository registration +- [ ] Registry entry update/fix +- [ ] Workflow/script change +- [ ] Documentation update + +## Checklist + +- [ ] Entry file is named `.catpkg.yml` +- [ ] Entry contains valid `location.github` field +- [ ] Repository is publicly accessible +- [ ] Repository has the `battlescribe-data` topic (recommended) diff --git a/.github/actions/install-yaml/action.yml b/.github/actions/install-yaml/action.yml index cc25089b..4f106816 100644 --- a/.github/actions/install-yaml/action.yml +++ b/.github/actions/install-yaml/action.yml @@ -5,5 +5,5 @@ runs: steps: - run: | Import-Module powershell-yaml -ErrorAction:Ignore - || Install-Module powershell-yaml -RequiredVersion 0.4.2 -Force + || Install-Module powershell-yaml -RequiredVersion 0.4.7 -Force shell: pwsh diff --git a/.github/scripts/Update-GhPagesData.ps1 b/.github/scripts/Update-GhPagesData.ps1 deleted file mode 100644 index 9c6aa511..00000000 --- a/.github/scripts/Update-GhPagesData.ps1 +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env pwsh - -[CmdletBinding()] -param ( - # Path to the index cache directory (catpkg.yml files) - [Parameter(Mandatory)] - [string] $IndexPath, - - # Path to where the data is save to (catpkg.json and bsr files) - [Parameter(Mandatory)] - [string] $DataPath, - - # Base URL where the data is served from - [Parameter(Mandatory)] - [string] $DataBaseUrl, - - # Path to catpkg-gallery.json which will be saved with modifications to Data path - [Parameter(Mandatory)] - [string] $GalleryJsonPath, - - # Paths of index cache files which were updated and should be updated in Data path as well - [Parameter()] - [string[]] $ChangedIndexPath, - - # Authorization token for GitHub API endpoints - [Parameter(Mandatory)] - [string] $Token -) - -#Requires -Version 7 -#Requires -Module powershell-yaml - -filter Update-DataEntry { - [CmdletBinding()] - param ( - [Parameter(ValueFromPipeline)] - [System.IO.FileInfo] $InputObject - ) - $cacheFile = $InputObject - $name = $cacheFile.Name.Replace(".catpkg.yml", "").ToLowerInvariant() - $cacheYml = Get-Content $cacheFile -Raw | ConvertFrom-Yaml -Ordered - $repository = $cacheYml.location.github - Write-Verbose "$repository - processing..." - if (!$cacheYml.cache.catpkg.properties) { - Write-Verbose "$repository skipped - no catpkg cache data." - return - } - $props = $cacheYml.cache.catpkg.properties - # calculate hash of cache yaml file - $cacheYmlSha = (Get-FileHash "$cacheFile" -Algorithm SHA256).Hash - # get or create directory for gh-pages data for the repo latest data - $repoBasePath = New-Item "$DataPath/$name/latest" -ItemType Directory -Force - $cacheShaFilename = "cache.sha.txt" - $catpkgFilename = "$name.catpkg.json" - $bsrFilename = "$name.bsr" - $catpkg = Get-Item "$repoBasePath/$catpkgFilename" -ErrorAction:SilentlyContinue - if ($catpkg) { - # read saved hash of last used cache yml file - $savedSha = Get-Item "$repoBasePath/$cacheShaFilename" -ErrorAction:SilentlyContinue - if ($cacheYmlSha -eq $savedSha) { - Write-Verbose "$repository skipped - data index up-to-date with cache" - return - } - } - # update required - # save cache hash - Set-Content "$repoBasePath/$cacheShaFilename" $cacheYmlSha -NoNewline -Force - # save catpkg.json - try { - $null = Invoke-WebRequest $props.repositoryUrl -OutFile "$repoBasePath/$catpkgFilename" - } catch { - # needed not to print 404 webpage in logs - throw $_.Exception.Message - } - $catpkg = Get-Content "$repoBasePath/$catpkgFilename" -Raw | ConvertFrom-Json - # save repo.bsr - try { - $null = Invoke-WebRequest $catpkg.repositoryBsrUrl -OutFile "$repoBasePath/$bsrFilename" - } catch { - # needed not to print 404 webpage in logs - throw $_.Exception.Message - } - # patch up catpkg properties to direct to gallery dataindex urls - . { - # force name - $catpkg.name = ($catpkg.name -eq $name) ? $catpkg.name : $name - # update supported properties - $catpkg.repositoryUrl = ([uri]"$DataBaseUrl/$name/latest/$catpkgFilename").AbsoluteUri - $catpkg.repositoryBsrUrl = ([uri]"$DataBaseUrl/$name/latest/$bsrFilename").AbsoluteUri - # remove unsupported properties - $catpkg.PSObject.Properties.Remove('indexUrl') - $catpkg.PSObject.Properties.Remove('repositoryGzipUrl') - # save back to file - $catpkg | ConvertTo-Json -Depth 10 | Set-Content "$repoBasePath/$catpkgFilename" - } - Write-Verbose "$repository - data updated." -} - -$caches = Get-ChildItem $IndexPath *.catpkg.yml - -Write-Verbose "Catpkg data - processing..." -$caches | Update-DataEntry -Write-Verbose "Catpkg data - updated." - -# now let's patch and save catpkg-gallery.json -Write-Verbose "Gallery data - processing..." -$gallery = Get-Content $GalleryJsonPath -Raw | ConvertFrom-Json -$galleryFilename = "catpkg-gallery.json" -$gallery.repositorySourceUrl = ([uri]"$DataBaseUrl/$galleryFilename").AbsoluteUri -$archived = @($gallery.repositories | Where-Object { $_.archived -eq $true } | Select-Object -ExpandProperty githubUrl) -$gallery.repositories = @( - Get-ChildItem $DataPath *.catpkg.json -Recurse - | Sort-Object Name - | ForEach-Object { - $catpkg = Get-Content $_ -Raw | ConvertFrom-Json | Select-Object -ExcludeProperty '$schema', 'repositoryFiles' - $catpkg.archived = $archived -contains $catpkg.githubUrl - Write-Output $catpkg - } -) -$gallery | ConvertTo-Json -Depth 10 | Set-Content "$DataPath/$galleryFilename" -Write-Verbose "Gallery data - updated." diff --git a/.github/scripts/Update-Registry.ps1 b/.github/scripts/Update-Registry.ps1 index a3f43bd1..dd804811 100644 --- a/.github/scripts/Update-Registry.ps1 +++ b/.github/scripts/Update-Registry.ps1 @@ -4,15 +4,16 @@ param ( [string] $RegistryPath, [Parameter()] - [string] $Token + [string] $Token, + + [Parameter()] + [string] $ParentOrgName = 'BSData' ) #Requires -Version 7 #Requires -Module powershell-yaml -$parentOrgName = 'BSData' - -$query = "topic:battlescribe-data+org:$parentOrgName" +$query = "topic:battlescribe-data+org:$ParentOrgName" $apiRepoSearchArgs = @{ Method = 'GET' Uri = "https://api.github.com/search/repositories?q=$query" @@ -22,7 +23,7 @@ $apiRepoSearchArgs = @{ } } if ($Token) { - $apiRepoSearchArgs.Headers['Authorization'] = "token $token" + $apiRepoSearchArgs.Headers['Authorization'] = "Bearer $Token" } $apiRepoSearchResult = Invoke-RestMethod @apiRepoSearchArgs $repositories = @($apiRepoSearchResult.items | Sort-Object full_name) @@ -57,7 +58,7 @@ $reposNoLongerExisting = $regEntries | Where-Object { # we've got it in API response, so it surely exists return $false } - if ($reponame -match "^$parentOrgName/" -and $reponame -notin $orgRepoNames) { + if ($reponame -match "^$ParentOrgName/" -and $reponame -notin $orgRepoNames) { # we've not got it in API response and it's from requested org, # so it doesn't meet search criteria (e.g. no 'battlescribe-data' topic) $_.del_reason = "not found in org repos query results" @@ -68,7 +69,7 @@ $reposNoLongerExisting = $regEntries | Where-Object { Uri = "https://api.github.com/repos/$reponame" StatusCodeVariable = 'status' SkipHttpErrorCheck = $true - Headers = ($Token ? @{ Authorization = "token $Token" } : @{}) + Headers = ($Token ? @{ Authorization = "Bearer $Token" } : @{}) } $null = Invoke-RestMethod @apiRepoGetArgs $notFound = $status -eq 404 diff --git a/.github/scripts/Validate-Registry.ps1 b/.github/scripts/Validate-Registry.ps1 index 30a57b2d..8c5a61d0 100644 --- a/.github/scripts/Validate-Registry.ps1 +++ b/.github/scripts/Validate-Registry.ps1 @@ -29,27 +29,16 @@ if ($env:GITHUB_ACTIONS -ne 'true') { function LogError($Message, $File) { Write-Error "$File`: $Message" } - function FormatFileRef($props, $msg) { - $file = $props['file'] - $line = $props['line'] - $col = $props['col'] - $value = '' - if ($file) { - $position = $line ? $col ? "($line,$col)" : "($line)" : ''; - $value += "${file}${position}: " - } - return $value + $message - } } else { function LogDebug($Message) { - Write-ActionDebug $Message + Write-Host "::debug::$Message" } function LogWarning($Message, $File) { - Write-ActionWarning $Message $File + Write-Host "::warning file=$File::$Message" } function LogError($Message, $File) { - Write-ActionError $Message $File + Write-Host "::error file=$File::$Message" } } @@ -82,9 +71,8 @@ Get-ChildItem $entriesDir *.catpkg.yml | Sort-Object Name | ForEach-Object { StatusCodeVariable = 'status' SkipHttpErrorCheck = $true Headers = @{ - # preview api for 'topics' - Accept = 'application/vnd.github.mercy-preview+json' - } + ($Token ? @{ Authorization = "token $Token" } : @{} ) + Accept = 'application/vnd.github.v3+json' + } + ($Token ? @{ Authorization = "Bearer $Token" } : @{} ) } $apiRepo = Invoke-RestMethod @apiRepoArgs if ($status -ne 200) { @@ -106,7 +94,7 @@ Get-ChildItem $entriesDir *.catpkg.yml | Sort-Object Name | ForEach-Object { Uri = "$apiUrl/contents/.github/workflows" StatusCodeVariable = 'status' SkipHttpErrorCheck = $true - Headers = ($Token ? @{ Authorization = "token $Token" } : @{} ) + Headers = ($Token ? @{ Authorization = "Bearer $Token" } : @{} ) } $workflows = Invoke-RestMethod @getWorkflowsArgs $addWorkflowsSuggestion = @" diff --git a/.github/scripts/lib/BsdataGallery/BsdataGallery.psd1 b/.github/scripts/lib/BsdataGallery/BsdataGallery.psd1 new file mode 100644 index 00000000..7e5604b0 --- /dev/null +++ b/.github/scripts/lib/BsdataGallery/BsdataGallery.psd1 @@ -0,0 +1,16 @@ +@{ + RootModule = 'BsdataGallery.psm1' + ModuleVersion = '1.0.0' + GUID = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890' + Author = 'BSData' + Description = 'PowerShell module for BSData Gallery index operations' + PowerShellVersion = '7.0' + RequiredModules = @('powershell-yaml') + FunctionsToExport = @( + 'Update-BsdataGalleryIndex' + 'Get-BsdataGalleryCatpkg' + ) + CmdletsToExport = @() + VariablesToExport = @() + AliasesToExport = @() +} diff --git a/.github/scripts/lib/BsdataGallery/BsdataGallery.psm1 b/.github/scripts/lib/BsdataGallery/BsdataGallery.psm1 index b9b9eba2..368d6e79 100644 --- a/.github/scripts/lib/BsdataGallery/BsdataGallery.psm1 +++ b/.github/scripts/lib/BsdataGallery/BsdataGallery.psm1 @@ -86,7 +86,7 @@ function Get-GHApiUpdatedResult { Headers = & { $requestHeaders = @{ } if ($Token) { - $requestHeaders['Authorization'] = "token $Token" + $requestHeaders['Authorization'] = "Bearer $Token" } if ($LastModified) { $requestHeaders['If-Modified-Since'] = $LastModified @@ -172,7 +172,7 @@ function Get-UpdatedCache { [string]$Token, [Parameter()] - [string]$DisableLastModifiedCaching + [switch]$DisableLastModifiedCaching ) # prepare result object @@ -289,7 +289,7 @@ function Update-BsdataGalleryIndex { [string]$Token, [Parameter()] - [string]$DisableLastModifiedCaching + [switch]$DisableLastModifiedCaching ) # read registry entries diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a69895e..83c2c340 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,18 +12,19 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/install-yaml - name: Validate registry - uses: Amadevus/pwsh-script@v2 - with: - script: | - $ErrorActionPreference = 'Continue' - $entries = @() - if ($github.base_ref) { - # we are in a pull request, let's only check changed entries - $baseRefBranch = 'refs/heads/base' - git fetch --depth=1 origin "+$($github.base_ref):$baseRefBranch" | Out-Host - # get a list of files changed in PR - $entries = git diff-tree --no-commit-id --name-only -r HEAD $baseRefBranch - Write-Host "Changed files:" - $entries | Out-Host - } - ./.github/scripts/Validate-Registry.ps1 -Entries $entries -Token $github.token + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $ErrorActionPreference = 'Continue' + $entries = @() + if ($env:GITHUB_BASE_REF) { + # we are in a pull request, let's only check changed entries + $baseRefBranch = 'refs/heads/base' + git fetch --depth=1 origin "+$($env:GITHUB_BASE_REF):$baseRefBranch" | Out-Host + # get a list of files changed in PR + $entries = git diff-tree --no-commit-id --name-only -r HEAD $baseRefBranch + Write-Host "Changed files:" + $entries | Out-Host + } + ./.github/scripts/Validate-Registry.ps1 -Entries $entries -Token $env:GITHUB_TOKEN diff --git a/.github/workflows/update-index.yml b/.github/workflows/update-index.yml index 84982654..56e815fb 100644 --- a/.github/workflows/update-index.yml +++ b/.github/workflows/update-index.yml @@ -13,7 +13,6 @@ on: env: GALLERY_JSON: bsdata.catpkg-gallery.json RELEASE_TAG: index-v1 - ENABLE_GHPAGES_UPDATE: false jobs: update-index: runs-on: ubuntu-latest @@ -22,7 +21,7 @@ jobs: uses: actions/checkout@v4 with: ref: main - path: master + path: main - name: Checkout index uses: actions/checkout@v4 @@ -30,23 +29,24 @@ jobs: ref: index path: index - - uses: ./master/.github/actions/install-yaml + - uses: ./main/.github/actions/install-yaml - name: Compile index - uses: Amadevus/pwsh-script@v2 id: compile - with: - script: | - $ErrorActionPreference = 'Stop' - $args = @{ - IndexPath = Resolve-Path ./index - RegistryPath = Resolve-Path ./master/registry - GalleryJsonPath = $env:GALLERY_JSON - Token = $github.token - } - ./master/.github/scripts/Update-IndexCache.ps1 @args -Verbose - # save master's SHA to associate branches - "master-sha: $env:GITHUB_SHA" > index/master-sha.yml + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $ErrorActionPreference = 'Stop' + $args = @{ + IndexPath = Resolve-Path ./index + RegistryPath = Resolve-Path ./main/registry + GalleryJsonPath = $env:GALLERY_JSON + Token = $env:GITHUB_TOKEN + } + ./main/.github/scripts/Update-IndexCache.ps1 @args -Verbose + # save main's SHA to associate branches + "main-sha: $env:GITHUB_SHA" > index/main-sha.yml - name: Upload gallery-json as workflow artifact if: success() || failure() @@ -57,29 +57,29 @@ jobs: continue-on-error: true - name: Format index commit message - uses: Amadevus/pwsh-script@v2 id: format_index_msg - with: - script: | - Set-Location ./index - # get list of changed files - $pkgids = @((git status --porcelain | % { $_.Substring(3) } | Split-Path -Leaf) -replace '\.catpkg\.yml$','') - if ($pkgids.Length -eq 0) { - Write-Host 'No changes in the index' -ForegroundColor Green - return - } - # create commit message - $summary = if ($pkgids.Length -eq 1) { - $pkgids - } else { - "{0} (+{1} more)" -f $pkgids[0],($pkgids.Length - 1) - } - $message = @" - Changed: $summary + shell: pwsh + run: | + Set-Location ./index + # get list of changed files + $pkgids = @((git status --porcelain | % { $_.Substring(3) } | Split-Path -Leaf) -replace '\.catpkg\.yml$','') + if ($pkgids.Length -eq 0) { + Write-Host 'No changes in the index' -ForegroundColor Green + "result=" >> $env:GITHUB_OUTPUT + return + } + # create commit message + $summary = if ($pkgids.Length -eq 1) { + $pkgids + } else { + "{0} (+{1} more)" -f $pkgids[0],($pkgids.Length - 1) + } + $message = @" + Changed: $summary - $($pkgids -join "`n") - "@ - return $message + $($pkgids -join "`n") + "@ + "result=$message" >> $env:GITHUB_OUTPUT - name: Push index changes uses: stefanzweifel/git-auto-commit-action@v5 @@ -124,39 +124,3 @@ jobs: }) console.log('Done'); return upload; - - - name: Checkout gh-pages - if: steps.push-index.outputs.changes_detected == 'true' && env.ENABLE_GHPAGES_UPDATE == 'true' - uses: actions/checkout@v4 - with: - ref: gh-pages - path: gh-pages - - - name: Update dataindex - if: steps.push-index.outputs.changes_detected == 'true' && env.ENABLE_GHPAGES_UPDATE == 'true' - uses: Amadevus/pwsh-script@v2 - id: update - env: - COMPILE_RESULT: ${{ needs.update-index.outputs.compile_result }} - with: - script: | - $ErrorActionPreference = 'Stop' - $compileResult = $env:COMPILE_RESULT | ConvertFrom-Json - $owner, $repo = $github.repository -split '/' - $subpath = "dataindex" - $args = @{ - IndexPath = Resolve-Path ./index - DataPath = Resolve-Path (New-Item "./gh-pages/$subpath" -ItemType Directory -Force) - DataBaseUrl = "https://$owner.github.io/$repo/$subpath" - GalleryJsonPath = $env:GALLERY_JSON - ChangedIndexPath = $compileResult.changed_index_paths - Token = $github.token - } - ./master/.github/scripts/Update-GhPagesData.ps1 @args -Verbose - - - name: Push gh-pages changes - if: steps.push-index.outputs.changes_detected == 'true' && env.ENABLE_GHPAGES_UPDATE == 'true' - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: ":robot: update data index" - repository: gh-pages diff --git a/.github/workflows/update-registry.yml b/.github/workflows/update-registry.yml index e3ee25bc..79ce6bc7 100644 --- a/.github/workflows/update-registry.yml +++ b/.github/workflows/update-registry.yml @@ -4,7 +4,7 @@ on: - cron: "0 9 * * *" # at 9:00 push: branches: - - master + - main paths: - ".github/workflows/update-registry.yml" - ".github/scripts/**" @@ -13,47 +13,50 @@ jobs: update: runs-on: ubuntu-latest steps: - - name: Checkout master + - name: Checkout main uses: actions/checkout@v4 - uses: ./.github/actions/install-yaml - name: Update registry entries - uses: Amadevus/pwsh-script@v2 id: update - with: - script: | - $ErrorActionPreference = 'Stop' - $args = @{ - RegistryPath = Resolve-Path ./registry - Token = $github.token - } - ./.github/scripts/Update-Registry.ps1 @args -Verbose + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $ErrorActionPreference = 'Stop' + $args = @{ + RegistryPath = Resolve-Path ./registry + Token = $env:GITHUB_TOKEN + } + $result = ./.github/scripts/Update-Registry.ps1 @args -Verbose + "result=$($result | ConvertTo-Json -Compress -Depth 5)" >> $env:GITHUB_OUTPUT - name: Format PR text - uses: Amadevus/pwsh-script@v2 id: formatting + shell: pwsh env: UPDATE_JSON: ${{ steps.update.outputs.result }} - with: - script: | - $repos = $env:UPDATE_JSON | ConvertFrom-Json - $adds = $repos.add | Where-Object { $_ } | ForEach-Object { "- add $($_.html_url)`n" } - $dels = $repos.del | Where-Object { $_ } | ForEach-Object { "- rem $($_.html_url) *($($_.reason))*`n" } - @{ - count = $repos.count - pr_body_repo_list = -join @($adds + $dels) - } + run: | + $repos = $env:UPDATE_JSON | ConvertFrom-Json + $adds = $repos.add | Where-Object { $_ } | ForEach-Object { "- add $($_.html_url)`n" } + $dels = $repos.del | Where-Object { $_ } | ForEach-Object { "- rem $($_.html_url) *($($_.reason))*`n" } + $count = $repos.count + $pr_body_repo_list = -join @($adds + $dels) + "count=$count" >> $env:GITHUB_OUTPUT + "pr_body_repo_list<> $env:GITHUB_OUTPUT + $pr_body_repo_list >> $env:GITHUB_OUTPUT + "EOF" >> $env:GITHUB_OUTPUT - name: Create PR with registry updates uses: peter-evans/create-pull-request@v8 with: - commit-message: ":robot: Update ${{ fromJSON(steps.formatting.outputs.result).count }} registry entries" + commit-message: ":robot: Update ${{ steps.formatting.outputs.count }} registry entries" branch: bot/registry-update delete-branch: true title: "🤖 Update registry entries" body: | Updated repositories: - ${{ fromJSON(steps.formatting.outputs.result).pr_body_repo_list }} + ${{ steps.formatting.outputs.pr_body_repo_list }} [Workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) diff --git a/.gitignore b/.gitignore index dfcfd56f..35711896 100644 --- a/.gitignore +++ b/.gitignore @@ -1,350 +1,24 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* +# OS files +Thumbs.db +ehthumbs.db +Desktop.ini +.DS_Store + +# Editor/IDE files +.vscode/ +.idea/ +*.swp +*.swo *~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak -# SQL Server files -*.mdf -*.ldf -*.ndf +# PowerShell temp files +*.pssc +*.psrc -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat +# Node modules node_modules/ -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ +# Temp and log files +*.log +*.tmp +*.bak diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..52dd640c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,66 @@ +# Contributing to BSData Gallery + +Thank you for your interest in contributing to the BSData Gallery! + +## Adding a New Repository + +The most common contribution is adding a new BattleScribe data repository to the gallery. + +### Prerequisites + +Your repository must: + +- Be publicly accessible on GitHub +- Have the `battlescribe-data` topic added +- Have a published release with a `.catpkg.json` asset +- Follow the setup guide at [BSData catalogue development wiki](https://github.com/BSData/catalogue-development/wiki/Help:-Hosting-repositories) + +### Steps + +1. **Fork** this repository +2. **Create** a new file in `registry/entries/` named `.catpkg.yml` +3. **Add** the following content: + + ```yaml + location: + github: / + ``` + + For example, for `https://github.com/BSData/wh40k-10e`: + + ```yaml + location: + github: BSData/wh40k-10e + ``` + +4. **Open a pull request** — CI will automatically validate your entry + +### Naming Convention + +- The filename should match your repository name in lowercase +- Use the `.catpkg.yml` extension +- Example: `my-game-system.catpkg.yml` + +## Repositories in the BSData Organization + +Repositories under the [BSData](https://github.com/BSData) organization with the `battlescribe-data` topic are automatically discovered and added by a daily workflow. You typically don't need to manually add these. + +## Reporting Issues + +If you notice problems with the gallery (missing repositories, broken data, etc.), please [open an issue](https://github.com/BSData/gallery/issues/new). + +## Development + +The gallery uses PowerShell scripts and GitHub Actions: + +- **Validation**: `.github/scripts/Validate-Registry.ps1` +- **Registry updates**: `.github/scripts/Update-Registry.ps1` +- **Index compilation**: `.github/scripts/Update-IndexCache.ps1` + `lib/BsdataGallery` module + +### Running Locally + +Scripts require PowerShell 7+ and the `powershell-yaml` module: + +```pwsh +Install-Module powershell-yaml -Force +``` diff --git a/README.md b/README.md index 91d76923..bb603127 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,18 @@ # gallery + Gallery to browse all BattleScribe data sources (repositories), track and index them automatically. The gallery can be visually browsed at [gallery.bsdata.net](https://gallery.bsdata.net/). +## How It Works + +This repository serves as a **registry** of BattleScribe data repositories. GitHub Actions workflows: + +1. **Validate** new registry entries on pull requests +2. **Auto-discover** repositories in the [BSData](https://github.com/BSData) organization with the `battlescribe-data` topic +3. **Compile an index** of all repositories' latest releases into a gallery JSON file +4. **Publish** the gallery JSON as a [GitHub release asset](https://github.com/BSData/gallery/releases/latest/download/bsdata.catpkg-gallery.json) + ## Usage ### BattleScribe @@ -23,17 +33,36 @@ and paste it into an **Add repository source** field. Depending on your device: ![instruction to add repo source on Android](docs/images/android-add-repo-source.png) -## Adding new repository +## Adding a New Repository To add a new data repository to this distribution system: -- first, setup your repository as instructed in https://github.com/BSData/catalogue-development/wiki/Help:-Hosting-repositories -- add a new file within registry/entries path: https://github.com/BSData/gallery/tree/master/registry/entries -- open PR +1. Set up your repository as instructed in the [hosting guide](https://github.com/BSData/catalogue-development/wiki/Help:-Hosting-repositories) +2. Add a new file in [`registry/entries/`](https://github.com/BSData/gallery/tree/main/registry/entries) +3. Open a pull request -e.g. for a repository like https://github.com/BSData/skw9k create a file `skw9k.catpkg.yml` with the following content: +For example, for a repository like `https://github.com/BSData/skw9k`, create a file `skw9k.catpkg.yml` with: ```yaml location: github: BSData/skw9k ``` + +See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed instructions. + +## Repository Structure + +``` +registry/ + settings.yml # Gallery metadata and URLs + entries/ # One .catpkg.yml per tracked repository +.github/ + workflows/ # CI, index update, registry update, ChatOps + scripts/ # PowerShell scripts for index compilation + lib/BsdataGallery/ # PowerShell module for gallery operations + actions/ # Composite actions (install-yaml) +``` + +## License + +[MIT](LICENSE) diff --git a/registry/settings.yml b/registry/settings.yml index 74d8f05f..84ddac88 100644 --- a/registry/settings.yml +++ b/registry/settings.yml @@ -9,7 +9,7 @@ gallery: githubUrl: https://github.com/BSData/gallery # url of gallery owner's site discordUrl: https://discord.gg/UrrPB3T # url of gallery owner's discord feedUrl: http://battlescribedata.appspot.com/repos/feeds/all.atom # url of Atom feed combining all package feeds - twitterUrl: https://twitter.com/BSData # url of gallery owner's twitter account + twitterUrl: https://x.com/BSData # url of gallery owner's twitter account facebookUrl: https://facebook.com/BattleScribeData # url of gallery owner's facebook page registrations: path: entries # folder where entries are saved (in yml files) From d2f955f20bfcba23e655a82f8b90481a7b14072a Mon Sep 17 00:00:00 2001 From: Amadeusz Sadowski Date: Sun, 15 Mar 2026 03:13:37 +0100 Subject: [PATCH 2/2] Add architecture docs with Mermaid data flow diagram Adds docs/architecture.md covering all 4 workflows, data flow, branches, and key files. Includes Mermaid flowchart showing the full pipeline from repo discovery through to BattleScribe clients. Links from README. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 4 ++ docs/architecture.md | 141 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 docs/architecture.md diff --git a/README.md b/README.md index bb603127..a0444bff 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ location: See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed instructions. +## Architecture + +See [docs/architecture.md](docs/architecture.md) for a detailed description of how the workflows operate, including a data flow diagram. + ## Repository Structure ``` diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..2e6eac92 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,141 @@ +# Architecture + +This document describes how the BSData Gallery workflows operate and how data flows through the system. + +## Overview + +The gallery is a **registry + index** system. Contributors register BattleScribe data repositories by adding YAML entries. Automated workflows then discover, validate, index, and publish a gallery JSON file that BattleScribe clients consume. + +## Data Flow + +```mermaid +flowchart TD + subgraph Sources + A[BSData org repos on GitHub] + B[Community repos outside BSData] + end + + subgraph Registry Update + C[/"update-registry workflow\n(daily + push to main)"/] + C -->|searches GitHub API for\nbattlescribe-data topic| A + C -->|adds/removes entries| D[registry/entries/*.catpkg.yml] + C -->|opens PR| E[Pull Request] + E -->|merge| D + end + + subgraph Manual Contributions + B -->|contributor creates entry| F[Pull Request] + F -->|CI validates| G{CI passes?} + G -->|yes, merge| D + G -->|no| H[Fix & re-push] + end + + subgraph Index Compilation + I[/"update-index workflow\n(every 2 hours + push to main)"/] + D -->|reads entries| I + I -->|fetches latest release\nfor each repo| J[GitHub Releases API] + I -->|downloads catpkg.json\nassets| J + I -->|writes cache| K[(index branch)] + I -->|compiles| L[bsdata.catpkg-gallery.json] + end + + subgraph Distribution + L -->|uploaded as\nrelease asset| M[GitHub Release: index-v1] + M -->|consumed by| N[BattleScribe clients] + end + + style A fill:#4a9eff,color:#fff + style B fill:#4a9eff,color:#fff + style D fill:#f5a623,color:#fff + style K fill:#f5a623,color:#fff + style L fill:#7ed321,color:#fff + style M fill:#7ed321,color:#fff + style N fill:#9013fe,color:#fff +``` + +## Workflows + +### CI (`ci.yml`) + +**Purpose**: Validates registry entries on PRs and pushes. + +**Triggers**: +- Every push (except to `index` branch) +- Every pull request + +**Steps**: +1. On PRs, determines which files changed vs the base branch +2. Runs `Validate-Registry.ps1` on changed entries (or all entries on push) + +**Validation checks**: +- Entry has a `location.github` property +- Repository name is formatted as `owner/repo` +- Repository exists and is accessible (not private/deleted) +- Repository has the `battlescribe-data` topic (warning if missing) +- Repository has `publish-catpkg.yml` workflow (warning if missing) + +### Update Registry (`update-registry.yml`) + +**Purpose**: Auto-discovers and registers BSData org repositories. + +**Triggers**: +- Daily at 09:00 UTC +- Push to `main` when workflow/script files change + +**Steps**: +1. Searches GitHub API for repositories in the `BSData` org with the `battlescribe-data` topic +2. Adds `.catpkg.yml` entries for newly discovered repos +3. Removes entries for repos that no longer exist or lost the topic +4. Opens a PR with the changes (via `peter-evans/create-pull-request`) + +### Update Index (`update-index.yml`) + +**Purpose**: Compiles the gallery JSON index from all registry entries. + +**Triggers**: +- Every 2 hours (cron) +- Push to `main` when registry or script files change + +**Steps**: +1. Checks out `main` branch (registry + scripts) and `index` branch (cached data) +2. For each registry entry, queries GitHub API for the latest release + - Uses `If-Modified-Since` headers for efficient caching + - Skips repos that haven't changed since last check +3. Downloads each repo's `.catpkg.json` release asset +4. Updates the index cache (YAML files on the `index` branch) +5. Compiles all index entries into `bsdata.catpkg-gallery.json` +6. Pushes updated cache to the `index` branch +7. Uploads the gallery JSON as a release asset on the `index-v1` tag + +### ChatOps (`chatops.yml`) + +**Purpose**: Enables slash commands in issues and PRs. + +**Triggers**: +- Any issue comment starting with `/` + +**Steps**: +1. Checks out the `BSData/chatops` repo for command configuration +2. Dispatches the command via `peter-evans/slash-command-dispatch` + +Example: `/template-workflows-pr BSData/some-repo` can set up CI workflows for a repository. + +## Branches + +| Branch | Purpose | +|--------|---------| +| `main` | Registry entries, scripts, workflows, documentation | +| `index` | Cached index data (one `.catpkg.yml` per repo with release info) | +| `gh-pages` | *(unused)* Previously hosted data for the gallery website | + +## Key Files + +| Path | Description | +|------|-------------| +| `registry/settings.yml` | Gallery metadata, URLs, and configuration | +| `registry/entries/*.catpkg.yml` | One file per registered repository | +| `.github/scripts/Validate-Registry.ps1` | Validates entries against GitHub API | +| `.github/scripts/Update-Registry.ps1` | Auto-discovers repos in BSData org | +| `.github/scripts/Update-IndexCache.ps1` | Orchestrates index compilation | +| `.github/scripts/lib/BsdataGallery/` | PowerShell module with core logic | +| `.github/actions/install-yaml/` | Composite action to install `powershell-yaml` |