Implement daily file processing with automatic fallback to weekly files

.github/workflows/daily-partial-update.yml

@@ -0,0 +1,35 @@
+name: Daily Partial Data Update
+
+on:
+  schedule:
+    # Run partial updates Wednesday through Sunday at midnight UTC
+    # https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows
+    - cron: "0 0 * * 3-7"
+  workflow_dispatch:
+
+jobs:
+  partial-update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download latest hamcall binary
+        run: |
+          # Get the latest release
+          LATEST_RELEASE=$(curl -s "https://api.github.com/repos/${{ github.repository }}/releases" | jq -r 'map(select(.draft == false)) | sort_by(.published_at) | last | .tag_name')
+          echo "Latest release: $LATEST_RELEASE"
+
+          # Download the binary
+          curl -L -o hamcall "https://github.com/${{ github.repository }}/releases/download/$LATEST_RELEASE/hamcall"
+          chmod +x hamcall
+
+      - name: Run hamcall in partial mode
+        id: run_hamcall_partial
+        run: ./hamcall -dl -m=b2
+        env:
+          B2_KEYID: ${{ secrets.B2_KEYID }}
+          B2_APPKEY: ${{ secrets.B2_APPKEY }}
+          GEO_URL: ${{ secrets.GEO_URL }}
+          ULS_MODE: partial
+
+      - name: Send workflow notification
+        run: |
+          curl -X POST --data-urlencode "payload={\"username\": \"Github Actions\", \"icon_url\": \"https://camo.githubusercontent.com/eeb25c8fe7d597494949032a0a1b2e79830bd8c69d5cc495efdb62fac04007c0/68747470733a2f2f63646e2e73766172756e2e6465762f67682f616374696f6e732e706e67\", \"attachments\": [{\"author_name\": \"hamcall.dev\", \"title\": \"daily partial update result: ${{ job.status }}\", \"title_link\": \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"text\": \"Partial update completed - Updated ${{steps.run_hamcall_partial.outputs.updated-records}} records in ${{steps.run_hamcall_partial.outputs.run-time}}\"}]}" ${{ secrets.DISCORD_WEBHOOK }}

.github/workflows/run-process.yml

@@ -1,13 +1,14 @@
-name: Weekly Data Update
+name: Weekly Full Data Update
 
 on:
   schedule:
+    # Run full update every Monday at midnight UTC
     # https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows
     - cron: "0 0 * * 1"
   workflow_dispatch:
 
 jobs:
-  update:
+  full-update:
     runs-on: ubuntu-latest
     steps:
       - name: Download latest hamcall binary
@@ -30,4 +31,4 @@ jobs:
 
       - name: Send workflow notification
         run: |
-          curl -X POST --data-urlencode "payload={\"username\": \"Github Actions\", \"icon_url\": \"https://camo.githubusercontent.com/eeb25c8fe7d597494949032a0a1b2e79830bd8c69d5cc495efdb62fac04007c0/68747470733a2f2f63646e2e73766172756e2e6465762f67682f616374696f6e732e706e67\", \"attachments\": [{\"author_name\": \"hamcall.dev\", \"title\": \"build result: ${{ job.status }}\", \"title_link\": \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"text\": \"Updated ${{steps.run_hamcall.outputs.updated-records}} records in ${{steps.run_hamcall.outputs.run-time}}\"}]}" ${{ secrets.DISCORD_WEBHOOK }}
+          curl -X POST --data-urlencode "payload={\"username\": \"Github Actions\", \"icon_url\": \"https://camo.githubusercontent.com/eeb25c8fe7d597494949032a0a1b2e79830bd8c69d5cc495efdb62fac04007c0/68747470733a2f2f63646e2e73766172756e2e6465762f67682f616374696f6e732e706e67\", \"attachments\": [{\"author_name\": \"hamcall.dev\", \"title\": \"weekly full update result: ${{ job.status }}\", \"title_link\": \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\", \"text\": \"Full weekly update completed - Updated ${{steps.run_hamcall.outputs.updated-records}} records in ${{steps.run_hamcall.outputs.run-time}}\"}]}" ${{ secrets.DISCORD_WEBHOOK }}

downloader/downloader_test.go

@@ -66,8 +66,8 @@ func TestFetchHttpInvalidURL(t *testing.T) {
 	os.Chdir(tempDir)
 	defer os.Chdir(originalDir)
 
-	// Test with invalid URL
-	err = FetchHttp("test.txt", "http://invalid-url-that-does-not-exist.com")
+	// Test with invalid URL - use an invalid scheme and malformed URL
+	err = FetchHttp("test.txt", "://invalid-malformed-url")
 	if err == nil {
 		t.Fatalf("Expected error for invalid URL, got nil")
 	}
@@ -244,4 +244,4 @@ func TestUnzipZipSlipProtection(t *testing.T) {
 	if !strings.Contains(err.Error(), "illegal file path") {
 		t.Fatalf("Expected 'illegal file path' error, got: %v", err)
 	}
-}
+}

main.go

@@ -99,11 +99,14 @@ func downloadFiles() {
 }
 
 func process(calls *map[string]data.HamCall) {
+	ulsMode := os.Getenv("ULS_MODE")
+	partialMode := ulsMode == "partial"
+
 	uls.Process(calls)
 	ised.Process(calls, "ised_data/amateur_delim.txt")
-	radioid.Process(calls)
-	lotw.Process(calls)
-	geo.Process(calls)
+	radioid.Process(calls, partialMode)
+	lotw.Process(calls, partialMode)
+	geo.Process(calls, partialMode)
 }
 
 func writeToB2(calls *map[string]data.HamCall, keyID, applicationKey string, uploadWorkers int, osSigExit chan bool, dryRun bool) {

source/geo/geo.go

@@ -24,7 +24,7 @@ func Download(wg *sync.WaitGroup) error {
 	return nil
 }
 
-func Process(calls *map[string]data.HamCall) {
+func Process(calls *map[string]data.HamCall, partialMode bool) {
 	start := time.Now()
 	fmt.Print("processing GEO")
 
@@ -65,16 +65,18 @@ func Process(calls *map[string]data.HamCall) {
 		call := record[0]
 		item, c := (*calls)[call]
 		if c {
+			// Update existing callsign
 			item.Location = &loc
-		} else {
+			(*calls)[call] = item
+		} else if !partialMode {
+			// Only create new callsign if not in partial mode
 			item = data.HamCall{
 				Callsign: call,
 				Location: &loc,
 			}
+			(*calls)[call] = item
 		}
-		(*calls)[call] = item
-
+		// In partial mode, skip callsigns that don't exist in ULS data
 	}
 	fmt.Printf(" ... %s\n", time.Since(start).String())
-
 }

source/geo/geo_test.go

@@ -41,7 +41,7 @@ N0DEF,Bob Wilson,Bob,Wilson,789 Pine Rd,Nowhere,FL,98765,US,25.5,-80.25,EL95ab`
 
 	// Test processing
 	calls := make(map[string]data.HamCall)
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results - should be 3 calls (header is skipped due to invalid coordinates)
 	if len(calls) != 3 {
@@ -141,7 +141,7 @@ W5TEST,John Smith,John,Smith,123 Main St,Anytown,TX,12345,US,32.5,-97.0,EM12ab`
 	}
 
 	// Test processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results - should have 1 call (header skipped due to invalid coordinates)
 	if len(calls) != 1 {
@@ -204,7 +204,7 @@ N0DEF,Bob Wilson,Bob,Wilson,789 Pine Rd,Nowhere,FL,98765,US,25.5,-80.25,EL95ab`
 
 	// Test processing
 	calls := make(map[string]data.HamCall)
-	Process(&calls)
+	Process(&calls, false)
 
 	// Should only process valid records - only N0DEF (header, W5TEST and KC5ABC have invalid coords)
 	if len(calls) != 1 {
@@ -247,7 +247,7 @@ func TestGeoProcessMissingFile(t *testing.T) {
 	calls := make(map[string]data.HamCall)
 
 	// This should not panic or error, just return without processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Should have no calls since file doesn't exist
 	if len(calls) != 0 {

source/lotw/lotw.go

@@ -23,7 +23,7 @@ func Download(wg *sync.WaitGroup) error {
 	return nil
 }
 
-func Process(calls *map[string]data.HamCall) {
+func Process(calls *map[string]data.HamCall, partialMode bool) {
 	start := time.Now()
 	fmt.Print("processing LOTW")
 
@@ -49,16 +49,19 @@ func Process(calls *map[string]data.HamCall) {
 		call := record[0]
 		item, c := (*calls)[call]
 		if c {
+			// Update existing callsign
 			item.LOTW = record[1] + record[2]
-		} else {
+			(*calls)[call] = item
+		} else if !partialMode {
+			// Only create new callsign if not in partial mode
 			item = data.HamCall{
 				Callsign: call,
 				LOTW:     record[1] + record[2],
 			}
+			(*calls)[call] = item
 		}
-		(*calls)[call] = item
+		// In partial mode, skip callsigns that don't exist in ULS data
 	}
 
 	fmt.Printf(" ... %s\n", time.Since(start).String())
-
 }

source/lotw/lotw_test.go

@@ -41,7 +41,7 @@ N0DEF,2023-02-28,2023-02-28`
 
 	// Test processing
 	calls := make(map[string]data.HamCall)
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results - should be 4 calls (3 real + 1 header)
 	if len(calls) != 4 {
@@ -126,7 +126,7 @@ W5TEST,2023-01-15,2023-01-15`
 	}
 
 	// Test processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results - should have 2 calls (1 existing + 1 header)
 	if len(calls) != 2 {
@@ -162,7 +162,7 @@ func TestLotwProcessMissingFile(t *testing.T) {
 	calls := make(map[string]data.HamCall)
 
 	// This should not panic or error, just return without processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Should have no calls since file doesn't exist
 	if len(calls) != 0 {

source/radioid/radioid.go

@@ -24,7 +24,7 @@ func Download(wg *sync.WaitGroup) error {
 	return nil
 }
 
-func Process(calls *map[string]data.HamCall) {
+func Process(calls *map[string]data.HamCall, partialMode bool) {
 	start := time.Now()
 	fmt.Print("processing radioID")
 
@@ -56,15 +56,18 @@ func Process(calls *map[string]data.HamCall) {
 		call := record[1]
 		item, c := (*calls)[call]
 		if c {
+			// Update existing callsign
 			item.DMRID = append(item.DMRID, id)
-		} else {
+			(*calls)[call] = item
+		} else if !partialMode {
+			// Only create new callsign if not in partial mode
 			item = data.HamCall{
 				Callsign: call,
 				DMRID:    []int{id},
 			}
-
+			(*calls)[call] = item
 		}
-		(*calls)[call] = item
+		// In partial mode, skip callsigns that don't exist in ULS data
 	}
 	fmt.Printf(" ... %s\n", time.Since(start).String())
 }

source/radioid/radioid_test.go

@@ -40,7 +40,7 @@ func TestRadioIdProcess(t *testing.T) {
 
 	// Test processing
 	calls := make(map[string]data.HamCall)
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results
 	if len(calls) != 3 {
@@ -126,7 +126,7 @@ func TestRadioIdProcessWithExistingCalls(t *testing.T) {
 	}
 
 	// Test processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Verify results - should have 1 call with multiple DMRIDs
 	if len(calls) != 1 {
@@ -189,7 +189,7 @@ func TestRadioIdProcessInvalidID(t *testing.T) {
 
 	// Test processing
 	calls := make(map[string]data.HamCall)
-	Process(&calls)
+	Process(&calls, false)
 
 	// Should only process the valid record
 	if len(calls) != 1 {
@@ -224,7 +224,7 @@ func TestRadioIdProcessMissingFile(t *testing.T) {
 	calls := make(map[string]data.HamCall)
 
 	// This should not panic or error, just return without processing
-	Process(&calls)
+	Process(&calls, false)
 
 	// Should have no calls since file doesn't exist
 	if len(calls) != 0 {