Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/sdk-release.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"engine_commit": "8fd3e8780c985721730855396de987e59e268fcf",
"attempt": 2
"attempt": 3
}
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,12 @@ python release.py stage --input downloaded-sdk-assets --output approved-sdk --co

The automated tests use deliberate miniature fixtures; they do not build SDKs
locally. Real native compilation and binary consumer qualification run in CI.
If compilation fails, reproduce in the private repository to inspect diagnostics.
Do not add public compiler logs to troubleshoot a private build.
Expected audit failures report only fixed public rule labels, never rejected
file names or contents. Unexpected parser errors remain generic. PEM detection
requires encoded key material and a matching end marker; a cryptographic format
marker embedded in Godot is not itself a key. If compilation fails, reproduce
in the private repository to inspect diagnostics. Do not add public compiler
logs to troubleshoot a private build.

Automation in this repository is AGPL-3.0-only. SDK components retain their own
licences and required notices; this repository's licence does not relicense them.
78 changes: 42 additions & 36 deletions release.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@


class SDKRelease:
class PolicyError(ValueError):
"""A fixed public rule label, never archive content or parser diagnostics."""

TARGETS = {"linux-x64": ("linux", "x86_64", "so"),
"windows-x64": ("windows", "x86_64", "dll"),
"macos-universal": ("macos", "universal", "dylib")}
Expand All @@ -24,20 +27,20 @@ class SDKRelease:
FORBIDDEN = {".c", ".cpp", ".cc", ".cxx", ".h", ".hpp", ".hxx", ".gd", ".gdshader", ".gdshaderinc",
".glsl", ".hlsl", ".shader", ".pdb", ".snupkg", ".csproj", ".sln", ".map", ".py", ".sh", ".ps1", ".js", ".ts", ".rs", ".log"}
SOURCE_TEXT = re.compile(rb'(?m)^\s*(?:namespace\s+Wol\.Engine\b|(?:public|internal)\s+(?:(?:partial|sealed|static|abstract)\s+)*(?:class|struct|interface|record)\s+\w+|shader_type\s+\w+\s*;|void\s+(?:vertex|fragment)\s*\(|#include\s*[<"])')
SECRET_TEXT = re.compile(rb'(?:-----BEGIN (?:OPENSSH |RSA |EC )?PRIVATE KEY-----|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9_]{20,})')
SECRET_TEXT = re.compile(rb'(?:-----BEGIN (?P<pem>(?:OPENSSH |RSA |EC |DSA |ENCRYPTED )?PRIVATE KEY)-----[ \t\r\n]+(?:[A-Za-z0-9+/]{20,}={0,2}[ \t\r\n]+)+-----END (?P=pem)-----|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9_]{20,})')

@classmethod
def requested_commit(cls, root, override=None):
if override is None:
request = json.loads((Path(root) / ".github/sdk-release.json").read_text())
if not isinstance(request, dict) or set(request) not in ({"engine_commit"}, {"engine_commit", "attempt"}):
raise ValueError("Unexpected release request fields")
raise cls.PolicyError("Unexpected release request fields")
attempt = request.get("attempt", 1)
if type(attempt) is not int or not 1 <= attempt <= 100000:
raise ValueError("Release attempt must be a positive integer")
raise cls.PolicyError("Release attempt must be a positive integer")
override = request["engine_commit"]
if not isinstance(override, str) or not re.fullmatch(r"[0-9a-f]{40}", override):
raise ValueError("Release requests require a full engine commit")
raise cls.PolicyError("Release requests require a full engine commit")
return override

@classmethod
Expand All @@ -50,38 +53,38 @@ def check_repository(cls, root):
for revision in revisions:
paths.update(subprocess.check_output(["git", "ls-tree", "-r", "--name-only", "-z", revision], cwd=root).decode().split("\0"))
if paths - cls.PUBLIC_FILES - {""}:
raise ValueError("Repository contains an unapproved path")
raise cls.PolicyError("Repository contains an unapproved path")
blobs = set()
for revision in revisions:
for row in subprocess.check_output(["git", "ls-tree", "-r", revision], cwd=root).splitlines():
metadata, _ = row.split(b"\t", 1)
mode, kind, oid = metadata.split()
if kind != b"blob" or mode not in (b"100644", b"100755"):
raise ValueError("Repository contains a symlink or submodule")
raise cls.PolicyError("Repository contains a symlink or submodule")
blobs.add(oid.decode())
for oid in blobs:
data = subprocess.check_output(["git", "cat-file", "blob", oid], cwd=root)
if len(data) > 100_000 or cls.SOURCE_TEXT.search(data) or cls.SECRET_TEXT.search(data):
raise ValueError("Repository content failed the publication policy")
raise cls.PolicyError("Repository content failed the publication policy")
for name in paths - {""}:
path = root / name
if path.exists() and (path.is_symlink() or cls.SOURCE_TEXT.search(path.read_bytes()) or cls.SECRET_TEXT.search(path.read_bytes())):
raise ValueError("Working tree failed the publication policy")
raise cls.PolicyError("Working tree failed the publication policy")

@classmethod
def check_sdk(cls, archive, commit, target):
if not re.fullmatch(r"[0-9a-f]{40}", commit) or target not in cls.TARGETS:
raise ValueError("Invalid SDK identity")
raise cls.PolicyError("Invalid SDK identity")
archive = Path(archive)
if archive.name != f"wol-engine-{commit}-{target}.zip":
raise ValueError("Unexpected SDK archive name")
raise cls.PolicyError("Unexpected SDK archive name")
sidecar = archive.with_suffix(".zip.sha256").read_bytes()
checksum = sidecar[:64].decode("ascii")
if not re.fullmatch(r"[0-9a-f]{64}", checksum) or sidecar != (checksum + " " + archive.name + "\n").encode("ascii"):
raise ValueError("Unexpected checksum sidecar content")
raise cls.PolicyError("Unexpected checksum sidecar content")
with archive.open("rb") as stream:
if hashlib.file_digest(stream, "sha256").hexdigest() != checksum:
raise ValueError("SDK archive checksum mismatch")
raise cls.PolicyError("SDK archive checksum mismatch")
native_platform, arch, extension = cls.TARGETS[target]
required = {"WolEngine.props", "GodotApi.props", "native/spatial.gdextension",
f"native/bin/libspatial.{native_platform}.template_release.{arch}.{extension}",
Expand All @@ -94,31 +97,31 @@ def check_sdk(cls, archive, commit, target):
members = packed.infolist()
names = [m.filename for m in members]
if len(names) != len(set(names)) or len(names) > 4000 or sum(m.file_size for m in members) > 4 * 1024**3:
raise ValueError("Invalid SDK archive structure")
raise cls.PolicyError("Invalid SDK archive structure")
manifest = json.loads(packed.read("RUNTIME.json"))
if set(manifest) != {"schema", "target", "godot", "commit", "release", "editor", "files"}:
raise ValueError("Unexpected SDK manifest fields")
raise cls.PolicyError("Unexpected SDK manifest fields")
if manifest.get("schema") != 1 or manifest.get("commit") != commit or manifest.get("target") != target or manifest.get("godot") != "4.7.2":
raise ValueError("SDK manifest identity mismatch")
raise cls.PolicyError("SDK manifest identity mismatch")
expected_release = f"https://github.com/RakuenSoftware/wol-engine-release/releases/tag/runtime-{commit}"
if manifest.get("release") != expected_release:
raise ValueError("SDK release origin mismatch")
raise cls.PolicyError("SDK release origin mismatch")
files = manifest["files"]
if set(names) != set(files) | {"RUNTIME.json"} or not required <= set(files) or manifest.get("editor") != editor:
raise ValueError("SDK file manifest mismatch")
raise cls.PolicyError("SDK file manifest mismatch")
for member in members:
name = member.filename
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or "\\" in name or ":" in name or path.as_posix() != name or member.is_dir() or stat.S_ISLNK(member.external_attr >> 16):
raise ValueError("Unsafe SDK member")
raise cls.PolicyError("Unsafe SDK member")
if name == "RUNTIME.json":
continue
data = packed.read(member)
if hashlib.sha256(data).hexdigest() != files[name]:
raise ValueError("SDK member checksum mismatch")
raise cls.PolicyError("SDK member checksum mismatch")
suffix = path.suffix.lower()
if suffix in cls.FORBIDDEN or cls.SECRET_TEXT.search(data):
raise ValueError("Implementation, diagnostics or credentials in SDK")
raise cls.PolicyError("Implementation, diagnostics or credentials in SDK")
allowed = name in required
allowed |= bool(re.fullmatch(r"\.dependencies/godot-api/api/(Debug|Release)/GodotSharp(?:Editor)?\.dll", name))
allowed |= bool(re.fullmatch(r"licenses/[A-Za-z0-9_.-]+\.(?:txt|md)", name))
Expand All @@ -133,31 +136,31 @@ def check_sdk(cls, archive, commit, target):
if name == "editor/GodotSharp/Tools/GodotTools.OpenVisualStudio":
allowed = target == "windows-x64" and data.startswith((b"MZ", b"\x7fELF"))
if not allowed:
raise ValueError("Unapproved SDK file")
raise cls.PolicyError("Unapproved SDK file")
if suffix == ".nupkg":
with zipfile.ZipFile(io.BytesIO(data)) as nested:
for entry in nested.infolist():
inner = PurePosixPath(entry.filename)
if inner.is_absolute() or ".." in inner.parts or "\\" in entry.filename or stat.S_ISLNK(entry.external_attr >> 16):
raise ValueError("Unsafe dependency package")
raise cls.PolicyError("Unsafe dependency package")
content = nested.read(entry)
if inner.suffix.lower() in {".pdb", ".gdshader", ".gdshaderinc", ".glsl", ".hlsl"} or cls.SECRET_TEXT.search(content) or re.search(rb"namespace\s+Wol\.Engine\b", content):
raise ValueError("Private implementation or symbols in dependency package")
raise cls.PolicyError("Private implementation or symbols in dependency package")
if suffix == ".cs" and data.strip():
raise ValueError("Nonempty C# implementation in SDK")
raise cls.PolicyError("Nonempty C# implementation in SDK")
if suffix in {".dll", ".exe"} and not data.startswith(b"MZ"):
raise ValueError("Invalid PE binary")
raise cls.PolicyError("Invalid PE binary")
if name.startswith("managed/"):
# Owned assemblies must not embed portable PDBs or CodeView paths.
pe = struct.unpack_from("<I", data, 0x3c)[0]
if data[pe:pe + 4] != b"PE\0\0":
raise ValueError("Invalid managed PE header")
raise cls.PolicyError("Invalid managed PE header")
sections = struct.unpack_from("<H", data, pe + 6)[0]
optional_size = struct.unpack_from("<H", data, pe + 20)[0]
optional = pe + 24
magic = struct.unpack_from("<H", data, optional)[0]
if magic not in (0x10b, 0x20b):
raise ValueError("Unknown managed PE format")
raise cls.PolicyError("Unknown managed PE format")
directories = optional + (96 if magic == 0x10b else 112)
debug_rva, debug_size = struct.unpack_from("<II", data, directories + 6 * 8)
if debug_size:
Expand All @@ -168,32 +171,32 @@ def check_sdk(cls, archive, commit, target):
if address <= debug_rva < address + max(size, raw_size):
debug_offset = raw + debug_rva - address
if debug_offset is None or debug_size % 28:
raise ValueError("Invalid managed debug directory")
raise cls.PolicyError("Invalid managed debug directory")
for offset in range(debug_offset, debug_offset + debug_size, 28):
if struct.unpack_from("<I", data, offset + 12)[0] in (2, 17):
raise ValueError("Private debug metadata in managed assembly")
raise cls.PolicyError("Private debug metadata in managed assembly")
if suffix == ".so" and not data.startswith(b"\x7fELF"):
raise ValueError("Invalid ELF binary")
raise cls.PolicyError("Invalid ELF binary")
if suffix == ".dylib" and data[:4] not in {b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf"}:
raise ValueError("Invalid Mach-O binary")
raise cls.PolicyError("Invalid Mach-O binary")
if name == editor and target != "windows-x64" and data[:4] not in ({b"\x7fELF"} if target == "linux-x64" else {b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf"}):
raise ValueError("Invalid editor binary")
raise cls.PolicyError("Invalid editor binary")
if name != editor and suffix not in {".dll", ".exe", ".so", ".dylib", ".nupkg", ".png", ".icns", ".car"} and cls.SOURCE_TEXT.search(data):
raise ValueError("Implementation text disguised as an SDK resource")
raise cls.PolicyError("Implementation text disguised as an SDK resource")
return archive

@classmethod
def stage(cls, source, destination, commit, targets):
source, destination = Path(source), Path(destination)
expected = {f"wol-engine-{commit}-{t}.zip{s}" for t in targets for s in ("", ".sha256")}
if {p.name for p in source.iterdir() if p.is_file() or p.is_symlink()} != expected:
raise ValueError("Expected only SDK archives and their checksums")
raise cls.PolicyError("Expected only SDK archives and their checksums")
if any((source / name).is_symlink() for name in expected):
raise ValueError("SDK input must not be a symlink")
raise cls.PolicyError("SDK input must not be a symlink")
for target in targets:
cls.check_sdk(source / f"wol-engine-{commit}-{target}.zip", commit, target)
if destination.exists():
raise ValueError("Choose an unused audit staging directory")
raise cls.PolicyError("Choose an unused audit staging directory")
destination.mkdir(parents=True)
for name in sorted(expected):
shutil.copyfile(source / name, destination / name)
Expand All @@ -215,6 +218,9 @@ def stage(cls, source, destination, commit, targets):
print("commit=" + SDKRelease.requested_commit(args.root, args.commit))
else:
SDKRelease.stage(args.input, args.output, args.commit, (args.target,) if args.target else tuple(SDKRelease.TARGETS))
except SDKRelease.PolicyError as error:
print("PUBLICATION_POLICY_FAILED: " + str(error) + "; output withheld.", file=sys.stderr)
sys.exit(1)
except Exception:
print("PUBLICATION_POLICY_FAILED: output withheld; inspect privately.", file=sys.stderr)
sys.exit(1)
Expand Down
22 changes: 22 additions & 0 deletions test_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,28 @@ def test_release_request_cli_emits_only_github_job_output(self):
self.assertEqual(result.stdout, '')
self.assertNotIn('Traceback', result.stderr)

def test_pem_format_markers_are_not_keys_but_encoded_key_blocks_are_rejected(self):
header = b'-----BEGIN PRIVATE KEY-----'
name = 'editor/Godot_v4.7.2-stable_mono_linux.x86_64'
SDKRelease.check_sdk(self.sdk(extra={name: b'\x7fELF' + header + b'\x00'}), self.commit, 'linux-x64')
for kind in (b'PRIVATE KEY', b'RSA PRIVATE KEY', b'EC PRIVATE KEY', b'OPENSSH PRIVATE KEY', b'ENCRYPTED PRIVATE KEY'):
payload = b'-----BEGIN ' + kind + b'-----\n' + b'A' * 64 + b'\n-----END ' + kind + b'-----\n'
with self.subTest(kind=kind), self.assertRaises(SDKRelease.PolicyError):
SDKRelease.check_sdk(self.sdk(extra={name: b'\x7fELF' + payload}), self.commit, 'linux-x64')

def test_cli_reports_fixed_audit_rules_without_echoing_rejected_content(self):
archive = self.sdk(extra={'runtime/Secret.cs': b'PRIVATE_FIXTURE_CONTENT'})
command = [sys.executable, str(Path(__file__).with_name('release.py').resolve()), 'stage',
'--input', str(self.root), '--output', str(self.root / 'approved'), '--commit', self.commit, '--target', 'linux-x64']
result = subprocess.run(command, capture_output=True, text=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn('Nonempty C# implementation in SDK', result.stderr)
self.assertEqual(result.stdout, '')
self.assertNotIn('Secret.cs', result.stderr)
self.assertNotIn('PRIVATE_FIXTURE_CONTENT', result.stderr)
self.assertNotIn('Traceback', result.stderr)
self.assertFalse((self.root / 'approved').exists())

def test_repository_history_keeps_deleted_source_visible_to_the_audit(self):
repo = self.root / 'repo'; repo.mkdir()
subprocess.run(['git', 'init', '-q', str(repo)], check=True)
Expand Down