-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathserver.py
More file actions
3185 lines (2919 loc) · 135 KB
/
Copy pathserver.py
File metadata and controls
3185 lines (2919 loc) · 135 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""Windrose admin console HTTP server (stdlib only).
Replaces the busybox httpd + CGI shell scripts. Runs in the same image
as the game server (just a different command). Exposes a small JSON API
and serves a single HTML page.
Routing:
GET / index.html
GET /healthz (always open, no auth)
GET /metrics Prometheus metrics (only with UI_ENABLE_METRICS_ROUTE=true)
GET /api/status full status JSON (status of game, players, resources)
GET /api/invite plain-text invite code
POST /api/upload stream tarball to PVC, preserve identity+saves
GET /api/saves/download stream tarball of R5/Saved
GET /api/config current ServerDescription + worlds
PUT /api/config stage changes (write Staged.json)
POST /api/config/apply swap staged -> live; signal restart
GET /api/backups list /home/steam/backups/*
POST /api/backups create a manual snapshot now
POST /api/backups/{id}/restore swap backup -> live (destructive)
GET /api/mods list uploaded mods + staged state
POST /api/mods/upload stage a .pak/.zip/.tar mod upload
Auth:
If UI_PASSWORD is set, HTTP basic auth is required on everything
except /healthz. Username is ignored.
Destructive endpoints (upload, config apply, backup restore, server
stop, world config PUT) are allowed when EITHER:
- UI_PASSWORD is set (admin is authenticated), or
- UI_ENABLE_ADMIN_WITHOUT_PASSWORD=true (explicit LAN-only opt-in).
Without either, destructive endpoints return 403.
"""
from __future__ import annotations
import base64
import hashlib
import io
import json
import os
import re
import shutil
import signal
import socket
import subprocess
import sys
import tarfile
import tempfile
import threading
import time
import urllib.parse
import urllib.request
import zipfile
from datetime import datetime, timezone
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Callable
# --- Config -----------------------------------------------------------------
BIND = os.environ.get("UI_BIND", "0.0.0.0")
PORT = int(os.environ.get("UI_PORT", "28080"))
WINDROSE_SERVER_DIR = Path(os.environ.get("WINDROSE_SERVER_DIR", "/home/steam/windrose/WindowsServer"))
R5_DIR = WINDROSE_SERVER_DIR / "R5"
SAVE_ROOT = R5_DIR / "Saved" / "SaveProfiles" / "Default" / "RocksDB"
R5_LOG = R5_DIR / "Saved" / "Logs" / "R5.log"
CONFIG_PATH = R5_DIR / "ServerDescription.json"
STAGED_CONFIG_PATH = R5_DIR / "ServerDescription.staged.json"
BACKUP_ROOT = Path(os.environ.get("WINDROSE_BACKUP_ROOT", "/home/steam/backups"))
# Frontend assets live in a sibling ui/ directory. Deployments that
# ship server.py and ui/ together at the same level (Docker image,
# bare-Linux install, dev checkout) just work. An override env var
# supports the pattern where nginx serves the bundle from a different
# mount but the Python sidecar still handles /api/*.
STATIC_DIR = Path(os.environ.get(
"UI_STATIC_DIR", str(Path(__file__).resolve().parent / "ui"),
))
UI_PASSWORD = os.environ.get("UI_PASSWORD", "")
# Opt-in flag that lets destructive actions (upload, server stop, config
# apply, backup restore) run even when no password is set. Intended only
# for LAN-only / firewalled deployments where the operator has accepted
# the risk. With UI_PASSWORD set, destructive is always on regardless.
UI_ENABLE_ADMIN_WITHOUT_PASSWORD = os.environ.get("UI_ENABLE_ADMIN_WITHOUT_PASSWORD", "false").lower() in ("1", "true", "yes")
# Serve the static HTML/CSS/JS bundle ourselves. Set false to make this
# pod "/api/*-only" so an nginx in front (ingress or sidecar) can own
# the static assets (and possibly auth) and reverse-proxy /api/* here.
UI_SERVE_STATIC = os.environ.get("UI_SERVE_STATIC", "true").lower() not in ("0", "false", "no")
UI_ENABLE_METRICS_ROUTE = os.environ.get("UI_ENABLE_METRICS_ROUTE", "false").lower() in ("1", "true", "yes")
BACKUP_RETAIN = int(os.environ.get("WINDROSE_BACKUP_RETAIN", "10"))
BACKUP_RETAIN_DAYS = float(os.environ.get("WINDROSE_BACKUP_RETAIN_DAYS", "7"))
# Auto-backup scheduler defaults. Zero on either disables that trigger.
# Both defaults can be overridden per-install via the admin UI, which
# writes an atomic JSON override file at $R5_DIR/.backup-config.json
# (see effective_backup_config() below). Env vars only seed the initial
# values until the operator saves from the UI.
#
# Semantics:
# - idleMinutes: N min after last player disconnects → take a backup.
# The idle clock resets every time the player count becomes non-zero.
# - floorHours: if the server has been continuously active (any players
# connected) for M hours with no auto-backup, take one.
# - Manual backups do NOT reset either of these clocks (separate systems,
# like in-game auto-saves vs manual saves).
AUTO_BACKUP_IDLE_MINUTES_DEFAULT = float(os.environ.get("WINDROSE_AUTO_BACKUP_IDLE_MINUTES", "1"))
AUTO_BACKUP_FLOOR_HOURS_DEFAULT = float(os.environ.get("WINDROSE_AUTO_BACKUP_FLOOR_HOURS", "6"))
# Poll cadence for the scheduler thread. Fast enough that an idle
# trigger of 1 min fires within ~15s of the threshold; piggybacks on
# the existing event-detector cadence.
AUTO_BACKUP_POLL_SECONDS = float(os.environ.get("WINDROSE_AUTO_BACKUP_POLL_SECONDS", "15"))
AUTO_BACKUP_MARKER_NAME = ".auto"
# Backup dir names starting with this prefix are exempt from the
# retention sweep — operators can pin important snapshots by naming
# them with this prefix (`POST /api/backups {"pin": true}` does the
# rename automatically). Prevents rapid automated backups from pushing
# out a known-good recovery snapshot.
BACKUP_PIN_PREFIX = "manual-"
# Windrose itself writes a per-launch backup under
# R5/Saved/SaveProfiles/Default_Backups/<timestamp>/ (up to 30, auto-
# rotated — behavior documented in the game's 2026-04 release notes).
# We surface them alongside our own backups in the UI so operators have
# one more recovery path. Read-only from our side — we never write
# into this tree.
GAME_BACKUPS_DIR = Path(os.environ.get(
"WINDROSE_GAME_BACKUPS_DIR",
str(R5_DIR / "Saved" / "SaveProfiles" / "Default_Backups"),
))
GAME_ROCKSDB_DIR = R5_DIR / "Saved" / "SaveProfiles" / "Default" / "RocksDB"
GAME_CPU_LIMIT_STR = os.environ.get("WINDROSE_GAME_CPU_LIMIT", "")
GAME_MEM_LIMIT_STR = os.environ.get("WINDROSE_GAME_MEM_LIMIT", "")
CPU_STATE_PATH = Path("/tmp/windrose-ui-cpu.state")
MODS_METADATA_NAME = ".mods.json"
MODS_STAGED_METADATA_NAME = ".mods.staged.json"
MODS_STAGE_DIR_NAME = ".mods-staging"
MODS_BACKUP_MARKER_NAME = ".mods-included"
MOD_FILE_SUFFIXES = (".pak", ".utoc", ".ucas")
# Idle-CPU patch UI override file. Entrypoint consults this on every boot
# to decide whether to apply or revert the binary patch (see
# maybe_patch_idle_cpu in scripts/entrypoint.sh). `disabled` forces OFF
# (revert on next restart if currently patched); `enabled` forces ON
# (patch regardless of WINDROSE_PATCH_IDLE_CPU env). Absent = follow env.
IDLE_PATCH_OVERRIDE_FILE = Path(os.environ.get(
"WINDROSE_PATCH_OVERRIDE_FILE",
str(R5_DIR / ".idle-patch-override"),
))
GAME_EXE_PATH = WINDROSE_SERVER_DIR / "R5" / "Binaries" / "Win64" / "WindroseServer-Win64-Shipping.exe"
# Maintenance-mode flag. Entrypoint loop-sleeps instead of launching Proton
# when this file exists, so the container stays up but the game stays stopped.
# Operator clears the flag (UI toggle or manual rm) and the next restart
# boots normally. Matches the entrypoint's default path so overriding the
# env var on either side moves both endpoints together.
MAINTENANCE_FLAG_FILE = Path(os.environ.get(
"WINDROSE_MAINTENANCE_FLAG_FILE",
str(R5_DIR / ".maintenance-mode"),
))
# Scratch dir for /api/saves/download's copytree snapshot. Defaults to
# /tmp which is fine on most hosts, but on tmpfs-backed /tmp (common on
# RHEL / some Docker setups) a large save can OOM the host mid-stream.
# Point this at a PVC-backed path (e.g. /home/steam/tmp) on those hosts.
SAVES_DOWNLOAD_SCRATCH_DIR = os.environ.get("WINDROSE_DOWNLOAD_SCRATCH_DIR", "/tmp")
# First: the Docker / bare-Linux install target. Second: the repo-root
# source-run fallback (server.py at repo root, patch-idle-cpu.py lives
# under scripts/).
PATCH_SCRIPT_CANDIDATES = [
Path("/usr/local/bin/patch-idle-cpu.py"),
Path(__file__).resolve().parent / "scripts" / "patch-idle-cpu.py",
]
_PATCH_STATE_CACHE: dict = {"mtime": None, "md5": None, "state": None, "reason": None}
_PATCH_STATE_LOCK = threading.Lock()
# --- Webhooks ---------------------------------------------------------------
WEBHOOK_URL = os.environ.get("WINDROSE_WEBHOOK_URL", "").strip()
WEBHOOK_DISCORD_URL = os.environ.get("WINDROSE_DISCORD_WEBHOOK_URL", "").strip()
WEBHOOK_EVENTS_RAW = os.environ.get("WINDROSE_WEBHOOK_EVENTS",
"server.online,server.offline,server.crashed,"
"player.join,player.leave,"
"backup.failed,backup.restore.failed,config.apply.failed").strip()
WEBHOOK_TIMEOUT = float(os.environ.get("WINDROSE_WEBHOOK_TIMEOUT", "5"))
WEBHOOK_POLL_SECONDS = float(os.environ.get("WINDROSE_WEBHOOK_POLL_SECONDS", "15"))
WEBHOOK_EVENTS = {e.strip() for e in WEBHOOK_EVENTS_RAW.split(",") if e.strip()}
# Set by /api/server/{stop,restart} so EventDetector can distinguish an
# operator-initiated stop (fires server.offline) from an unexpected exit
# (fires server.crashed). Plain float to keep this lock-free; the worst
# race is one missed crash-classification across two consecutive 15s polls.
_LAST_OPERATOR_STOP_AT: float = 0.0
_OPERATOR_STOP_GRACE_SECONDS: float = 30.0
# --- Utility: resource quantity parsing -------------------------------------
def parse_cpu_to_mcpu(q: str) -> int:
if not q:
return 0
q = q.strip()
if q.endswith("m"):
try:
return int(q[:-1])
except ValueError:
return 0
try:
return int(float(q) * 1000)
except ValueError:
return 0
_MEM_UNITS = {
"": 1,
"K": 1_000, "Ki": 1_024,
"M": 1_000_000, "Mi": 1_048_576,
"G": 1_000_000_000, "Gi": 1_073_741_824,
"T": 1_000_000_000_000, "Ti": 1_099_511_627_776,
}
def parse_mem_to_bytes(q: str) -> int:
if not q:
return 0
q = q.strip()
m = re.match(r"^([0-9.]+)([A-Za-z]*)$", q)
if not m:
return 0
n, unit = m.groups()
try:
return int(float(n) * _MEM_UNITS.get(unit, 1))
except (ValueError, KeyError):
return 0
def read_file(p: Path) -> str | None:
try:
return p.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
# --- Game process / resource detection --------------------------------------
def find_game_pid() -> tuple[int | None, int]:
"""Return (pid, rss_bytes) of the main Windrose game process.
Multiple UE threads share the "WindroseServer-*-Shipping.exe" cmdline;
pick the one with the highest VmRSS (the main process).
"""
best_pid, best_rss = None, 0
try:
for entry in Path("/proc").iterdir():
if not entry.name.isdigit():
continue
try:
cmdline = (entry / "cmdline").read_text()
if "WindroseServer" not in cmdline:
continue
if "WindroseServer-Win64-Shipping" not in cmdline and "WindroseServer.exe" not in cmdline:
continue
status = (entry / "status").read_text()
rss_kb = 0
for line in status.splitlines():
if line.startswith("VmRSS:"):
rss_kb = int(line.split()[1])
break
if rss_kb > best_rss:
best_rss = rss_kb
best_pid = int(entry.name)
except (OSError, ValueError):
continue
except OSError:
pass
return best_pid, best_rss * 1024
def game_uptime_seconds(pid: int | None) -> int:
if not pid:
return 0
try:
stat = (Path("/proc") / str(pid) / "stat").read_text()
# Field 22 (index 21 after comm-paren) = starttime in jiffies since boot.
comm_end = stat.rfind(")")
fields = stat[comm_end + 2:].split()
start_jiffies = int(fields[19])
clk = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
boot_up = float(Path("/proc/uptime").read_text().split()[0])
start_sec = start_jiffies / clk
return max(0, int(boot_up - start_sec))
except (OSError, ValueError, IndexError):
return 0
def cpu_sample(pid: int | None) -> float:
"""CPU percentage of 1 core since last call, smoothed over previous invocation."""
if not pid:
return 0.0
try:
stat = (Path("/proc") / str(pid) / "stat").read_text()
comm_end = stat.rfind(")")
fields = stat[comm_end + 2:].split()
pid_ticks = int(fields[11]) + int(fields[12]) # utime + stime
sys_line = Path("/proc/stat").read_text().splitlines()[0]
sys_ticks = sum(int(x) for x in sys_line.split()[1:])
n_cpus = os.cpu_count() or 1
except (OSError, ValueError, IndexError):
return 0.0
prev = {}
if CPU_STATE_PATH.exists():
try:
prev = json.loads(CPU_STATE_PATH.read_text())
except (OSError, ValueError):
prev = {}
# Only compare samples from the same pid (pid changes on pod bounce).
if prev.get("pid") == pid:
d_pid = pid_ticks - prev.get("pid_ticks", 0)
d_sys = sys_ticks - prev.get("sys_ticks", 0)
pct = (d_pid / d_sys) * n_cpus * 100 if d_sys > 0 and d_pid >= 0 else 0.0
else:
pct = 0.0
try:
CPU_STATE_PATH.write_text(json.dumps({
"pid": pid, "pid_ticks": pid_ticks, "sys_ticks": sys_ticks,
}))
except OSError:
pass
return round(pct, 2)
def resource_ceiling() -> dict:
cpu_mcpu = parse_cpu_to_mcpu(GAME_CPU_LIMIT_STR)
cpu_src = "chart_value"
if cpu_mcpu <= 0:
# Try cgroup v2 cpu.max
try:
text = Path("/sys/fs/cgroup/cpu.max").read_text().split()
if len(text) >= 2 and text[0] != "max":
quota, period = int(text[0]), int(text[1])
if period > 0:
cpu_mcpu = quota * 1000 // period
cpu_src = "cgroup"
except (OSError, ValueError):
pass
if cpu_mcpu <= 0:
cpu_mcpu = (os.cpu_count() or 1) * 1000
cpu_src = "host"
mem_bytes = parse_mem_to_bytes(GAME_MEM_LIMIT_STR)
mem_src = "chart_value"
if mem_bytes <= 0:
try:
mem_max = Path("/sys/fs/cgroup/memory.max").read_text().strip()
if mem_max and mem_max != "max":
mem_bytes = int(mem_max)
mem_src = "cgroup"
except (OSError, ValueError):
pass
if mem_bytes <= 0:
try:
for line in Path("/proc/meminfo").read_text().splitlines():
if line.startswith("MemTotal:"):
mem_bytes = int(line.split()[1]) * 1024
mem_src = "host"
break
except (OSError, ValueError):
pass
return {
"cpuLimitMcpu": cpu_mcpu,
"cpuLimitSource": cpu_src,
"memLimitBytes": mem_bytes,
"memLimitSource": mem_src,
}
# --- Log parsing: players + backend region ---------------------------------
_PLAYER_LINE = re.compile(
r"Name '(?P<name>[^']*)'\. AccountId '(?P<accountId>[^']*)'\. State '(?P<state>[^']*)'.*?TimeInGame (?P<timeInGame>\+[0-9:.]+)"
)
_ACCT_ID = re.compile(r"AccountId ([0-9A-Fa-f]+)")
def parse_active_players() -> list[dict]:
"""Scan tail of R5.log for the current set of active players.
Snapshot blocks are written sporadically, so we keep a per-AccountId
dedup (last wins) and supplement with event lines that carry a
definitive state (e.g. OnClientIsReady → ReadyToPlay).
"""
if not R5_LOG.exists():
return []
try:
# Read last ~1MB which covers several snapshots without loading
# hours of log.
sz = R5_LOG.stat().st_size
with R5_LOG.open("rb") as f:
if sz > 1_500_000:
f.seek(-1_500_000, os.SEEK_END)
data = f.read().decode("utf-8", errors="replace")
except OSError:
return []
seen: dict[str, tuple[dict, str]] = {} # accountId -> (player, section)
state_override: dict[str, str] = {}
section = None
for line in data.splitlines():
if "Connected Accounts" in line:
section = "connected"; continue
if "Reserved Accounts" in line:
section = "reserved"; continue
if "Disconnected Accounts" in line:
section = "disconnected"; continue
if not line.strip():
section = None; continue
# Disconnect events — forget the player entirely.
if "MoveAccountToListOfDisconnected" in line or "Account disconnected. AccountId" in line:
m = _ACCT_ID.search(line)
if m:
aid = m.group(1)
seen.pop(aid, None)
state_override.pop(aid, None)
continue
if "OnClientIsReady" in line and "Client id ReadyToPlay" in line:
m = _ACCT_ID.search(line)
if m:
state_override[m.group(1)] = "ReadyToPlay"
continue
# Snapshot numbered line, only in Connected / Reserved.
if section in ("connected", "reserved") and re.match(r"^\s+\d+\.\s+Name", line):
m = _PLAYER_LINE.search(line)
if m:
p = m.groupdict()
seen[p["accountId"]] = (p, section)
players = []
for aid, (p, sect) in seen.items():
p = {**p, "section": sect}
if aid in state_override:
p["state"] = state_override[aid]
players.append(p)
return players
def backend_region() -> str:
if not R5_LOG.exists():
return ""
try:
# Quick grep-like scan through the tail. Use stat().st_size for the
# file size (seek(0, SEEK_END) leaves the cursor at EOF, which
# causes the subsequent read() to return empty bytes on files
# smaller than the tail window — the bug that was blanking
# backendRegion).
sz = R5_LOG.stat().st_size
with R5_LOG.open("rb") as f:
if sz > 200_000:
f.seek(sz - 200_000)
data = f.read().decode("utf-8", errors="replace")
except OSError:
return ""
last = ""
for m in re.finditer(r"r5coopapigateway-([a-z]+)-release", data):
last = m.group(1)
return last
# --- Config + world data ----------------------------------------------------
def load_json(p: Path) -> dict | None:
try:
return json.loads(p.read_text())
except (OSError, ValueError):
return None
def _world_desc_path(island_id: str) -> "Path | None":
"""Resolve the active WorldDescription.json for island_id, picking the
newest GameVersion that has one."""
if not SAVE_ROOT.exists():
return None
for ver in sorted([p.name for p in SAVE_ROOT.iterdir() if p.is_dir()], reverse=True):
wd = SAVE_ROOT / ver / "Worlds" / island_id / "WorldDescription.json"
if wd.is_file():
return wd
return None
def _world_staged_path(island_id: str) -> "Path | None":
"""Path to the per-world staged file. Returns None if we can't locate
the live world directory."""
live = _world_desc_path(island_id)
return live.with_name("WorldDescription.staged.json") if live else None
def find_worlds() -> list[dict]:
"""List worlds on disk from all GameVersions."""
worlds: list[dict] = []
if not SAVE_ROOT.exists():
return worlds
for version_dir in sorted(SAVE_ROOT.iterdir()):
wdir = version_dir / "Worlds"
if not wdir.is_dir():
continue
for island in sorted(wdir.iterdir()):
wd = island / "WorldDescription.json"
staged = island / "WorldDescription.staged.json"
w: dict = {
"gameVersion": version_dir.name,
"islandId": island.name,
"staged": staged.is_file(),
}
# The row values shown in the UI's worlds table. When a
# staged override exists we prefer its values here — the
# editor already opens from staged (so the operator sees
# their in-progress edits), and showing the live values
# in the list produced a confusing "list says X, editor
# says Y" mismatch on any row with pending changes.
src_path = staged if staged.is_file() else wd
data = load_json(src_path)
if data:
inner = data.get("WorldDescription", data)
w["worldName"] = inner.get("WorldName", "")
w["worldPresetType"] = inner.get("WorldPresetType", "")
w["creationTime"] = inner.get("CreationTime")
worlds.append(w)
return worlds
def current_save_version() -> str:
if not SAVE_ROOT.exists():
return ""
versions = sorted([p.name for p in SAVE_ROOT.iterdir() if p.is_dir()])
return versions[-1] if versions else ""
# --- Backup handling --------------------------------------------------------
# --- idle-CPU patch helpers -------------------------------------------------
def read_idle_patch_override() -> str:
"""Return the trimmed content of the override file, or '' if absent."""
try:
v = IDLE_PATCH_OVERRIDE_FILE.read_text(encoding="ascii", errors="replace").strip()
return v if v in ("enabled", "disabled") else ""
except FileNotFoundError:
return ""
except OSError:
return ""
def write_idle_patch_override(value: str) -> None:
"""Write 'enabled' or 'disabled' to the override file, or delete it if
value is falsy. Raises ValueError on unknown value."""
if value in (None, "", "auto", "clear"):
try:
IDLE_PATCH_OVERRIDE_FILE.unlink()
except FileNotFoundError:
pass
return
if value not in ("enabled", "disabled"):
raise ValueError(f"override must be 'enabled', 'disabled', or 'auto'; got {value!r}")
IDLE_PATCH_OVERRIDE_FILE.parent.mkdir(parents=True, exist_ok=True)
# Atomic write. `.with_suffix(".tmp")` misbehaves on dot-leading names
# (`.idle-patch-override` → `.tmp`, not `.idle-patch-override.tmp`),
# so build the sibling path explicitly.
tmp = IDLE_PATCH_OVERRIDE_FILE.parent / (IDLE_PATCH_OVERRIDE_FILE.name + ".tmp")
tmp.write_text(value + "\n", encoding="ascii")
tmp.replace(IDLE_PATCH_OVERRIDE_FILE)
def _find_patch_script() -> Path | None:
for c in PATCH_SCRIPT_CANDIDATES:
if c.is_file():
return c
return None
def idle_patch_binary_state() -> dict:
"""Report the idle-CPU patch state under the sidecar model. The
entrypoint maintains a sibling `<exe>.patched.exe` alongside the
Steam-managed original, plus a `<exe>.patched-source.md5` file
recording the md5 of the source it was built from.
States:
missing — original EXE isn't on disk yet
unpatched — no sibling (patch is OFF or has never been built)
patched — sibling exists and its recorded source md5 matches
the current original's md5; will launch on next start
stale — sibling exists but source md5 differs (Windrose
updated since last patch build). Next boot rebuilds.
"""
try:
src_st = GAME_EXE_PATH.stat()
except FileNotFoundError:
return {"state": "missing", "md5": None}
patched_exe = GAME_EXE_PATH.parent / (GAME_EXE_PATH.stem + ".patched.exe")
source_md5_file = GAME_EXE_PATH.parent / (GAME_EXE_PATH.stem + ".patched-source.md5")
cache_key = (src_st.st_mtime_ns, src_st.st_size,
patched_exe.stat().st_mtime_ns if patched_exe.is_file() else 0)
with _PATCH_STATE_LOCK:
if _PATCH_STATE_CACHE.get("mtime") == cache_key and _PATCH_STATE_CACHE.get("state"):
return {
"state": _PATCH_STATE_CACHE["state"],
"md5": _PATCH_STATE_CACHE["md5"],
"reason": _PATCH_STATE_CACHE.get("reason"),
}
src_md5 = _file_md5_streaming(GAME_EXE_PATH)
if not patched_exe.is_file():
state, reason = "unpatched", None
else:
try:
cached = source_md5_file.read_text(encoding="ascii").strip()
except OSError:
cached = ""
if cached == src_md5:
state, reason = "patched", None
else:
state = "stale"
reason = f"source md5 moved from {cached or '(none)'} to {src_md5}; will rebuild on next restart"
with _PATCH_STATE_LOCK:
_PATCH_STATE_CACHE.update({"mtime": cache_key, "md5": src_md5, "state": state, "reason": reason})
return {"state": state, "md5": src_md5, "reason": reason}
def _file_md5_streaming(path: Path) -> str:
h = hashlib.md5()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def idle_patch_full_status() -> dict:
"""Full state payload for GET /api/idle-cpu-patch."""
override = read_idle_patch_override()
binary = idle_patch_binary_state()
# The operator-configured env var on the GAME container. Read from the
# game process's /proc/<pid>/environ so the UI reflects the deployed
# value, not the UI container's own env (which doesn't set it).
env_requested = _read_game_container_env("WINDROSE_PATCH_IDLE_CPU") == "1"
effective_on = env_requested
if override == "enabled":
effective_on = True
elif override == "disabled":
effective_on = False
binary_state_val = binary.get("state")
# Whether a game restart is needed for effective state to match the binary.
needs_restart = (
(effective_on and binary_state_val == "unpatched") or
(not effective_on and binary_state_val == "patched")
)
return {
"envRequested": env_requested,
"override": override or "auto",
"overrideFile": str(IDLE_PATCH_OVERRIDE_FILE),
"effectiveOn": effective_on,
"binaryMd5": binary.get("md5"),
"binaryState": binary_state_val,
"binaryReason": binary.get("reason"),
"needsRestart": needs_restart,
}
def _read_game_container_env(var_name: str) -> str:
"""Best-effort read of an env var from the game container's process via
the shared PID namespace. Returns '' on any failure."""
try:
for pid_dir in Path("/proc").iterdir():
if not pid_dir.name.isdigit():
continue
try:
cmdline = (pid_dir / "cmdline").read_bytes()
except OSError:
continue
if b"WindroseServer-Win64-Shipping" not in cmdline and b"proton" not in cmdline:
continue
try:
environ = (pid_dir / "environ").read_bytes()
except OSError:
continue
for entry in environ.split(b"\x00"):
if entry.startswith(var_name.encode() + b"="):
return entry.split(b"=", 1)[1].decode("ascii", "replace")
except OSError:
pass
return ""
def list_backups() -> list[dict]:
out: list[dict] = []
if not BACKUP_ROOT.exists():
return out
# Gather metadata first so we can sort by mtime (true chronological)
# instead of by dir name. Sorting by name groups the "manual-*"
# prefix lexicographically separately from bare-timestamp auto
# entries, producing a confusing interleave where a pinned backup
# from last week appears above an auto-backup from five minutes ago.
rows: list[dict] = []
for d in BACKUP_ROOT.iterdir():
try:
if not d.is_dir():
continue
except OSError:
continue # Python 3.13 is_dir() propagates OSError; skip unreadable entries
try:
st = d.stat()
size = sum(p.stat().st_size for p in d.rglob("*") if p.is_file())
mtime = st.st_mtime
except OSError:
size = 0; st = None; mtime = 0.0
pinned = d.name.startswith(BACKUP_PIN_PREFIX)
auto = (d / AUTO_BACKUP_MARKER_NAME).is_file()
rows.append({
"_mtime": mtime, # sort key, stripped before returning
"id": d.name,
"createdAt": datetime.fromtimestamp(mtime, tz=timezone.utc).isoformat() if st else "",
"sizeBytes": size,
"pinned": pinned,
"source": "auto" if auto else ("manual-pinned" if pinned else "manual"),
})
rows.sort(key=lambda r: r["_mtime"], reverse=True)
for r in rows:
r.pop("_mtime", None)
out.append(r)
return out
def create_backup(pin: bool = False) -> dict:
"""Snapshot the current R5/Saved tree + identity files into a
timestamped backup directory, then run retention.
Do NOT use raw `cp` or piecewise file swaps to recover from a backup
— RocksDB + the game's internal book-keeping under Saved/ expect
the whole subtree to be consistent. Use `restore_backup()` /
`POST /api/backups/{id}/restore`.
"""
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
dir_name = (BACKUP_PIN_PREFIX + ts) if pin else ts
dst = BACKUP_ROOT / dir_name
dst.mkdir(parents=True, exist_ok=True)
saved = R5_DIR / "Saved"
if saved.is_dir():
shutil.copytree(saved, dst / "Saved", dirs_exist_ok=True)
# Identity + operator-owned runtime settings. Staged configs are
# intentionally NOT captured — those represent pending intent, not
# live state, and mixing them into restore semantics gets confusing.
for name in (
"ServerDescription.json",
"WorldDescription.json",
".backup-config.json",
".idle-patch-override",
MODS_METADATA_NAME,
MODS_STAGED_METADATA_NAME,
):
src = R5_DIR / name
if src.is_file():
shutil.copy2(src, dst / name)
for src, rel in (
(mods_enabled_dir(), Path("Content/Paks/~mods")),
(mods_disabled_dir(), Path("Content/Paks/~mods.disabled")),
(mods_stage_root(), Path(MODS_STAGE_DIR_NAME)),
):
if src.is_dir():
shutil.copytree(src, dst / rel, dirs_exist_ok=True)
(dst / MODS_BACKUP_MARKER_NAME).write_text(
datetime.now(timezone.utc).isoformat() + "\n", encoding="ascii"
)
_prune_backups()
return {"id": dir_name, "path": str(dst), "pinned": pin}
def _prune_backups() -> None:
"""Retention policy, evaluated per call to `create_backup()`:
1. Directories whose name starts with BACKUP_PIN_PREFIX are never
pruned ("pinned"). Operators use this to protect a known-good
snapshot from being auto-evicted.
2. Among the non-pinned, keep the most-recent BACKUP_RETAIN by
name-sort (timestamp dirs sort chronologically by design).
3. Also keep anything mtime-younger than BACKUP_RETAIN_DAYS,
regardless of count — so a burst of backups in one hour
doesn't push out last week's quiet snapshot.
A backup survives if EITHER (2) or (3) keeps it. Rule (1) overrides
everything else.
"""
if not BACKUP_ROOT.is_dir():
return
cfg = effective_backup_config()
retain_count = int(cfg["retainCount"])
retain_days = float(cfg["retainDays"])
now = time.time()
age_cutoff = now - retain_days * 86400
# Python 3.13's Path.is_dir() stopped swallowing OSError, so a single
# unreadable entry during iterdir() would abort the entire prune sweep.
# Guard each is_dir() call so the sweep still completes.
entries = []
for p in BACKUP_ROOT.iterdir():
try:
if p.is_dir():
entries.append(p)
except OSError:
pass # can't read — leave it alone, move on
unpinned = [p for p in entries if not p.name.startswith(BACKUP_PIN_PREFIX)]
# Rule 2: top-N by name-sort descending.
keep_names = {p.name for p in sorted(unpinned, key=lambda p: p.name, reverse=True)[:retain_count]}
# Rule 3: keep anything within the age window.
for p in unpinned:
try:
if p.stat().st_mtime >= age_cutoff:
keep_names.add(p.name)
except OSError:
keep_names.add(p.name) # If we can't stat, don't be the one to delete it.
for p in unpinned:
if p.name not in keep_names:
shutil.rmtree(p, ignore_errors=True)
def pin_backup(bid: str) -> str:
"""Rename an existing backup dir with BACKUP_PIN_PREFIX so retention
skips it. Returns the new id. No-op if already pinned."""
src = BACKUP_ROOT / bid
if not src.is_dir():
raise FileNotFoundError(bid)
if bid.startswith(BACKUP_PIN_PREFIX):
return bid
new_name = BACKUP_PIN_PREFIX + bid
dst = BACKUP_ROOT / new_name
if dst.exists():
raise FileExistsError(new_name)
src.rename(dst)
return new_name
def unpin_backup(bid: str) -> str:
"""Strip BACKUP_PIN_PREFIX from a backup dir name so retention treats
it normally. No-op if not currently pinned."""
src = BACKUP_ROOT / bid
if not src.is_dir():
raise FileNotFoundError(bid)
if not bid.startswith(BACKUP_PIN_PREFIX):
return bid
new_name = bid[len(BACKUP_PIN_PREFIX):]
dst = BACKUP_ROOT / new_name
if dst.exists():
raise FileExistsError(new_name)
src.rename(dst)
return new_name
def list_game_backups() -> list[dict]:
"""Enumerate Windrose's own Default_Backups/ entries. Same shape as
list_backups() plus `source="game"` so the UI can tag the row."""
out: list[dict] = []
if not GAME_BACKUPS_DIR.is_dir():
return out
# Sort by mtime rather than name so the UI presents chronological
# order regardless of the game engine's future naming conventions.
rows = []
for d in GAME_BACKUPS_DIR.iterdir():
try:
if not d.is_dir():
continue
except OSError:
continue
try:
st = d.stat()
size = sum(p.stat().st_size for p in d.rglob("*") if p.is_file())
mtime = st.st_mtime
except OSError:
size = 0; st = None; mtime = 0.0
rows.append((mtime, d, st, size))
rows.sort(key=lambda r: r[0], reverse=True)
for _mtime, d, st, size in rows:
out.append({
"id": d.name,
"createdAt": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat() if st else "",
"sizeBytes": size,
"source": "game",
})
return out
def restore_game_backup(ts: str) -> None:
"""Merge-restore a Default_Backups/<ts>/ entry onto the live RocksDB
tree. Follows the recipe in the Windrose release notes: copy the
backup contents on top of SaveProfiles/Default/RocksDB/ replacing
matching files. Not a wipe-and-replace — the game's own Default_Backups
layout is scoped per-version/per-world, and we merge so multiple
worlds in other subtrees aren't inadvertently wiped.
Creates a snapshot of the current live state via create_backup()
FIRST (with a pin prefix so it survives retention) — if this restore
lands wrong, operator has a one-click rollback in the UI backup list.
"""
src = GAME_BACKUPS_DIR / ts
if not src.is_dir():
raise FileNotFoundError(ts)
# Pre-restore safety snapshot — pinned so retention can't evict it.
create_backup(pin=True)
GAME_ROCKSDB_DIR.mkdir(parents=True, exist_ok=True)
shutil.copytree(src, GAME_ROCKSDB_DIR, dirs_exist_ok=True)
def _backup_config_path() -> Path:
return R5_DIR / ".backup-config.json"
def effective_backup_config() -> dict:
"""Resolve the runtime backup config. File (UI-owned) wins over env.
Always returns a fully-populated dict so callers can pull fields
without defensive `.get` noise.
"""
cfg = {
"idleMinutes": AUTO_BACKUP_IDLE_MINUTES_DEFAULT,
"floorHours": AUTO_BACKUP_FLOOR_HOURS_DEFAULT,
"retainCount": BACKUP_RETAIN,
"retainDays": BACKUP_RETAIN_DAYS,
}
path = _backup_config_path()
try:
if path.is_file():
overrides = json.loads(path.read_text())
for k in list(cfg.keys()):
if k in overrides and overrides[k] is not None:
cfg[k] = type(cfg[k])(overrides[k])
except Exception as e: # noqa: BLE001 — bad file shouldn't brick pruning
print(f"[backup-config] failed to load overrides: {e}", file=sys.stderr, flush=True)
return cfg
def _validate_backup_config(payload: dict) -> dict:
"""Coerce + clamp incoming config PUT body. Raises ValueError on shape errors."""
if not isinstance(payload, dict):
raise ValueError("body must be an object")
out: dict = {}
for k, lo, hi, caster in (
("idleMinutes", 0.0, 24 * 60, float),
("floorHours", 0.0, 24 * 30, float),
("retainCount", 0, 10_000, int),
("retainDays", 0.0, 365.0 * 5, float),
):
if k not in payload:
continue
try:
v = caster(payload[k])
except (TypeError, ValueError):
raise ValueError(f"{k} must be a number")
if v < lo or v > hi:
raise ValueError(f"{k} out of range [{lo}, {hi}]")
out[k] = v
return out
def save_backup_config(overrides: dict) -> dict:
"""Atomic-replace the override file with the given dict. Caller should
pre-validate via _validate_backup_config. Returns the effective config
after the write."""
path = _backup_config_path()
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(path.name + ".tmp")
tmp.write_text(json.dumps(overrides, indent=2) + "\n", encoding="utf-8")
tmp.replace(path)
return effective_backup_config()
def _mark_auto_backup(bkp_dir: Path) -> None:
"""Drop the marker file inside a backup dir so list_backups() can tag it."""
try:
(bkp_dir / AUTO_BACKUP_MARKER_NAME).write_text(
datetime.now(timezone.utc).isoformat() + "\n", encoding="ascii"
)
except Exception as e: # noqa: BLE001 — cosmetic tag, don't fail the backup over it
print(f"[auto-backup] failed to drop marker: {e}", file=sys.stderr, flush=True)
# In-memory state for the scheduler thread. Reset on UI container restart;
# _bootstrap_auto_backup_state() re-derives the last-backup timestamp from
# on-disk markers so restarts don't cause spurious immediate backups.
_auto_state_lock = threading.Lock()
_auto_state: dict = {
"lastAutoBackupAt": None, # epoch float or None
"playersZeroSince": None, # epoch float or None
"lastResult": "",
}
def _bootstrap_auto_backup_state() -> None:
"""On process start, scan BACKUP_ROOT for the newest .auto-marked dir
and seed lastAutoBackupAt from it. Prevents the scheduler from firing
immediately after a UI container restart."""
latest = 0.0
try:
if BACKUP_ROOT.is_dir():
for d in BACKUP_ROOT.iterdir():
try:
if d.is_dir() and (d / AUTO_BACKUP_MARKER_NAME).is_file():
latest = max(latest, d.stat().st_mtime)
except OSError:
continue
except OSError:
pass
with _auto_state_lock:
_auto_state["lastAutoBackupAt"] = latest or None
def trigger_auto_backup(reason: str) -> dict | None:
"""Create an auto-backup and record state. Returns the backup dict or
None on failure. Reason goes into stderr log + webhook payload so
operators can distinguish idle vs floor triggers."""
try:
bkp = create_backup(pin=False)
except Exception as e: # noqa: BLE001 — log, don't crash scheduler
msg = f"auto-backup failed: {e}"
print(f"[auto-backup] {msg}", file=sys.stderr, flush=True)
with _auto_state_lock:
_auto_state["lastResult"] = msg
fire_event("backup.failed", source="auto", reason=str(e), trigger=reason)
return None
bkp_dir = Path(bkp["path"])
_mark_auto_backup(bkp_dir)