add/data/check-bin-data

README.RU.md

@@ -24,6 +24,27 @@
 
 Работа с данными построена на принципе proto-first, но с сохранением поддержки старых json-форматов экспорта. Если Вы планируете создавать свое решение с использованием данного генератора, рекомендую рассчитывать только на бинарные proto-файлы так как импорт/экспорт в json сфокусирован в первую очередь на поддержке старых вариантов и первоначальных watabou-генераторов. proto-файлы в свою очередь гарантируют долгосрочную поддержку даже в случае добавления нового функционала - старые сохраненные файлы будут поддерживаться и в новых версиях, без потери обратной совместимости.
 
+### Формирование бинарных файлов
+
+Все бинарные файлы формируются на основании proto-файлов которые лежат в папке: [protobuf](./protobuf)
+
+Импорт принимает как прямые бинарные прото-обьекты из корневых структур, так и обернутые в указатель типа с CRC32.
+
+Обертка формируется простым правилом: `|DataType uint32|{protobuf binary}|crc32IEEE({protobuf binary})`
+
+Указатели так же в proto: [DataType](./protobuf/data/enum.proto)
+
+Список корневых структур:
+
+- [GeoObj](./protobuf/data/geo/obj.proto)
+- [DwellingsObj](./protobuf/data/dwellings/obj.proto)
+- [PaletteMfcgObj](./protobuf/data/palette/mfcg.proto)
+- [PaletteVillageObj](./protobuf/data/palette/village.proto)
+- [PaletteDwellingsObj](./protobuf/data/palette/dwellings.proto)
+- [PaletteViewerObj](./protobuf/data/palette/viewer.proto)
+- [PaletteCaveObj](./protobuf/data/palette/cave.proto)
+- [PaletteGladeObj](./protobuf/data/palette/glade.proto)
+
 ## О проекте
 
 Этот проект — объединение в один проект генераторов карт от [watabou](https://github.com/watabou/), которые он публиковал на [watabou.github.io](https://watabou.github.io). Не все из них доступны публично, а даже то, что доступно, написано на не самом популярном языке.

README.md

@@ -24,6 +24,27 @@ If needed, you can “cut out” only one specific generator — at the code lev
 
 Work with data is built on the proto-first principle, while keeping support for older JSON export formats. If you plan to create your own solution using this generator, I recommend relying only on binary proto files, because JSON import/export is primarily focused on supporting older variants and the original watabou generators. Proto files, in turn, guarantee long-term support even if new functionality is added — old saved files will be supported in new versions without losing backward compatibility.
 
+### Binary file structure
+
+All binary files are generated from proto definitions located in the [protobuf](./protobuf) directory.
+
+Import accepts both raw binary proto objects from root structures and objects wrapped in a typed envelope with CRC32.
+
+The envelope follows a simple format: `|DataType uint32|{protobuf binary}|crc32IEEE({protobuf binary})`
+
+Type identifiers are also defined in proto: [DataType](./protobuf/data/enum.proto)
+
+Root structures:
+
+- [GeoObj](./protobuf/data/geo/obj.proto)
+- [DwellingsObj](./protobuf/data/dwellings/obj.proto)
+- [PaletteMfcgObj](./protobuf/data/palette/mfcg.proto)
+- [PaletteVillageObj](./protobuf/data/palette/village.proto)
+- [PaletteDwellingsObj](./protobuf/data/palette/dwellings.proto)
+- [PaletteViewerObj](./protobuf/data/palette/viewer.proto)
+- [PaletteCaveObj](./protobuf/data/palette/cave.proto)
+- [PaletteGladeObj](./protobuf/data/palette/glade.proto)
+
 ## About the project
 
 This project is a consolidation into one project of map generators by [watabou](https://github.com/watabou/) that he published at [watabou.github.io](https://watabou.github.io). Not all of them are publicly available, and even what is available is written in a not-so-popular language.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "fantasy-maphub",
   "name_full": "Fantasy MapHub Generators",
-  "version": "1.2.0",
+  "version": "1.2.1-add-data-check-bin-data.1",
   "description": "An offline-first hub that bundles several Watabou map generators into one consistent web app: same UI patterns, local assets, and modern import/export. It adds OpenAPI docs, proto-first serialization (including pure protobuf files), and a PWA build so everything works without an internet connection.",
   "license": "MPL-2.0",
   "author": "mail@sunsung.fun",

protobuf/data/enum.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package data;
+
+// // // //
+
+enum DataType {
+  DATA_UNSPECIFIED = 0;
+  geo = 1;
+  dwellings = 2;
+  reserved 3 to 19;
+
+  palette_cave = 20;
+  palette_glade = 21;
+  palette_dwellings = 22;
+  palette_mfcg = 23;
+  palette_village = 24;
+  palette_viewer = 25;
+  reserved 26 to 59;
+};

protobuf/data/palette/village.proto

@@ -47,47 +47,47 @@ message PaletteVillageTerrainObj {
 }
 
 message PaletteVillageHousesObj {
-  repeated PaletteRgbObj roofLight = 1;
-  PaletteRgbObj roofStroke = 2;
-  float roofVariance = 3;
-  float roofSlope = 4;
-  PaletteVillageRoofType roofType = 5;
+  repeated PaletteRgbObj roof_light = 1;
+  PaletteRgbObj roof_stroke = 2;
+  float roof_variance = 3;
+  float roof_slope = 4;
+  PaletteVillageRoofType roof_type = 5;
 }
 
 message PaletteVillageRoadsObj {
   PaletteRgbObj road = 1;
-  float largeRoad = 2;
-  float smallRoad = 3;
-  PaletteVillageOutlineType outlineRoads = 4;
-  bool mergeRoads = 5;
+  float large_road = 2;
+  float small_road = 3;
+  PaletteVillageOutlineType outline_roads = 4;
+  bool merge_roads = 5;
 }
 
 message PaletteVillageFieldsObj {
-  repeated PaletteRgbObj fieldLight = 1;
-  PaletteRgbObj fieldFurrow = 2;
-  float fieldVariance = 3;
-  PaletteVillageOutlineType outlineFields = 4;
+  repeated PaletteRgbObj field_light = 1;
+  PaletteRgbObj field_furrow = 2;
+  float field_variance = 3;
+  PaletteVillageOutlineType outline_fields = 4;
 }
 
 message PaletteVillageWaterObj {
-  PaletteRgbObj waterShallow = 1;
-  PaletteRgbObj waterDeep = 2;
-  PaletteRgbObj waterTide = 3;
-  uint32 shallowBands = 4;
+  PaletteRgbObj water_shallow = 1;
+  PaletteRgbObj water_deep = 2;
+  PaletteRgbObj water_tide = 3;
+  uint32 shallow_bands = 4;
 }
 
 message PaletteVillageTreesObj {
   repeated PaletteRgbObj tree = 1;
   PaletteRgbObj thicket = 2;
-  PaletteRgbObj treeDetails = 3;
+  PaletteRgbObj tree_details = 3;
   float treeVariance = 4;
-  PaletteVillageTreeShapeType treeShape = 5;
+  PaletteVillageTreeShapeType tree_shape = 5;
 }
 
 message PaletteVillageLightingObj {
-  PaletteRgbObj shadowColor = 1;
-  float shadowLength = 2;
-  uint32 shadowAngleDeg = 3;
+  PaletteRgbObj shadow_color = 1;
+  float shadow_length = 2;
+  uint32 shadow_angle_deg = 3;
   PaletteRgbObj lights = 4;
 }
 
@@ -100,8 +100,8 @@ message PaletteVillageTextObj {
 message PaletteVillageMiscObj {
   PaletteRgbObj ink = 1;
   PaletteRgbObj paper = 2;
-  float strokeNormal = 3;
-  float strokeThin = 4;
+  float stroke_normal = 3;
+  float stroke_thin = 4;
 }
 
 //

src/js/Dwellings.js

@@ -7,6 +7,7 @@ import * as FuncProto from "./shared/proto.js";
 import * as DataDwellings from "./shared/data/Dwellings.js";
 
 import * as DataProto from "./struct/data.js";
+import * as FuncBin from "./shared/data/bin-verify.js";
 
 const params = FuncProto.initParams(JSON.parse(String.raw`{{EMBED_PARAMETERS_JSON_DWELLINGS}}`));
 
@@ -5019,7 +5020,7 @@ var $lime_init = function (K, v) {
                 onSave: function (a, fmt) {
                     var pdo = DataDwellings.paletteObjFromLegacyJsonText(a.json());
                     if (fmt === "proto") {
-                        var bytes = DataProto.data.PaletteDwellingsObj.encode(pdo).finish();
+                        var bytes = DataDwellings.paletteProtoBytesFromObj(pdo);
                         id.saveText(bytes, this.getName(a) + ".palette.dw.pb", "application/octet-stream");
                     } else {
                         var json = DataDwellings.paletteLegacyJsonFromObj(pdo);
@@ -6567,10 +6568,9 @@ var $lime_init = function (K, v) {
                 return b
             };
             sb.exportAsProto=function(a){
-                var b=sb.exportProto(a),c=DataProto.data.DwellingsObj.encode(b).finish(),d=c.buffer.slice(c.byteOffset,c.byteOffset+c.byteLength);
-                c=Pd.fromArrayBuffer(d);
-                d=id.fixName(a.name);
-                id.saveBinary(c,d+".dw.pb","application/x-protobuf");
+                var b=sb.exportProto(a),c=DataProto.data.DwellingsObj.encode(b).finish();
+                var d = FuncBin.exportBin(c, DataProto.data.DataType.dwellings);
+                id.saveBinary(Pd.fromArrayBuffer(d),id.fixName(a.name)+".dw.pb","application/x-protobuf");
                 return b
             };
             sb.house2proto=function(a){

src/js/ToyTown2.js

@@ -15721,7 +15721,7 @@ if (params !== null) (function (S, u) {
                 onSave: function (a, fmt) {
                     var pvo = DataViewer.paletteObjFromLegacyJsonText(a.json());
                     if (fmt === "proto") {
-                        var bytes = DataProto.data.PaletteViewerObj.encode(pvo).finish();
+                        var bytes = DataViewer.paletteProtoBytesFromObj(pvo);
                         Og.saveText(bytes, this.getName(a) + ".palette.vr.pb", "application/octet-stream");
                     } else {
                         var json = DataViewer.paletteLegacyJsonFromObj(pvo);

src/js/Village.js

@@ -7,6 +7,7 @@ import * as FuncProto from "./shared/proto.js";
 import * as DataVillage from "./shared/data/Village.js";
 
 import * as DataProto from "./struct/data.js";
+import * as FuncBin from "./shared/data/bin-verify.js";
 
 const params = FuncProto.initParams(JSON.parse(String.raw`{{EMBED_PARAMETERS_JSON_VILLAGE}}`));
 
@@ -4922,15 +4923,14 @@ var $lime_init = function (E, u) {
                     var self = this;
 
                     var doSave = function(fmt) {
+                        var pvo = DataVillage.paletteObjFromLegacyJsonText(a.json());
                         if (fmt === "JSON") {
-                            var pvo = DataVillage.paletteObjFromLegacyJsonText(a.json());
                             var json = DataVillage.paletteLegacyJsonFromObj(pvo);
                             Sd.saveText(json, self.getName(a) + ".palette.vg.json", "application/json");
                             return;
                         }
                         if (fmt === "PROTO") {
-                            var pvo = DataVillage.paletteObjFromLegacyJsonText(a.json());
-                            var bytes = DataProto.data.PaletteVillageObj.encode(pvo).finish();
+                            var bytes = DataVillage.paletteProtoBytesFromObj(pvo);
                             Sd.saveText(bytes, self.getName(a) + ".palette.vg.pb", "application/octet-stream");
                             return;
                         }
@@ -9406,9 +9406,9 @@ var $lime_init = function (E, u) {
                     var b = Hh.getProto(a), c = Hh.stringifyProto(b);
                     return Sd.saveText(c, a.name + ".vg.json", "application/json"), b
                 }, Hh.exportBinary = function (a) {
-                var b = Hh.getProto(a), c = DataProto.data.GeoObj.encode(b).finish(),
-                    d = wd.fromArrayBuffer(c.buffer.slice(c.byteOffset, c.byteOffset + c.byteLength));
-                return Sd.saveBinary(d, a.name + ".vg.pb", "application/x-protobuf"), b
+                var b = Hh.getProto(a), c = DataProto.data.GeoObj.encode(b).finish();
+                var d = FuncBin.exportBin(c, DataProto.data.DataType.geo);
+                return Sd.saveBinary(wd.fromArrayBuffer(d), a.name + ".vg.pb", "application/x-protobuf"), b
             },
                 Hh.getData = function (a) {
                     Ua.CX = 0, Ua.CY = 0, Ua.SCALE = .7;

src/js/mfcg.js

@@ -5,8 +5,11 @@ import * as OthersShared from "./shared/others.js";
 import * as FuncProto from "./shared/proto.js";
 
 import * as DataMfcg from "./shared/data/mfcg.js";
+import * as FuncBin from "./shared/data/bin-verify.js";
 
 import * as DataProto from "./struct/data.js";
+import {paletteProtoBytesFromObj} from "./shared/data/mfcg.js";
+import {exportBin} from "./shared/data/bin-verify.js";
 
 const params = FuncProto.initParams(JSON.parse(String.raw`{{EMBED_PARAMETERS_JSON_MFCG}}`));
 
@@ -7882,7 +7885,8 @@ var $lime_init = function (A, t) {
             };
             be.asPROTO = function () {
                 var a = Ub.instance, b = lg.export(a), c = DataProto.data.GeoObj.encode(b).finish();
-                a = a.name, ge.saveBinary(Td.fromArrayBuffer(c.buffer), "" + a + ".mf.pb", "application/x-protobuf")
+                var bb = FuncBin.exportBin(c, DataProto.data.DataType.geo);
+                a = a.name, ge.saveBinary(Td.fromArrayBuffer(bb), "" + a + ".mf.pb", "application/x-protobuf")
             };
             var lg = function () {
             };

src/js/shared/data/Cave.js

@@ -1,5 +1,5 @@
 import * as DataProto from "../../struct/data.js";
-import { assertExpectedLegacyRootType, decodeDataFromFile } from "./data.js";
+import { assertExpectedLegacyRootType, decodeDataFromFile, encodeDataToBytes } from "./data.js";
 import * as PaletteFunc from "./palette.js";
 
 const LEGACY_KEYS = [
@@ -85,7 +85,7 @@ export function paletteObjFromLegacyJsonText(text) {
 
     if (!isPlainObject(obj)) throw PaletteFunc.unknownPalette();
 
-    assertExpectedLegacyRootType("PaletteCaveObj", obj);
+    assertExpectedLegacyRootType(DataProto.data.DataType.palette_cave, obj);
 
     if (isPlainObject(obj.colors) && isPlainObject(obj.shadow) && isPlainObject(obj.strokes) && isPlainObject(obj.hatching)) {
         return normalizePaletteCaveObjLike(obj);
@@ -152,10 +152,11 @@ export function paletteLegacyJsonFromObj(m) {
 
 export function paletteProtoBytesFromObj(m) {
     let n = normalizePaletteCaveObjLike(m);
-    return DataProto.data.PaletteCaveObj.encode(n).finish();
+    let raw = DataProto.data.PaletteCaveObj.encode(n).finish();
+    return encodeDataToBytes(DataProto.data.DataType.palette_cave, raw);
 }
 
 export function decodePaletteFile(name, data) {
-    let msg = decodeDataFromFile("PaletteCaveObj", paletteObjFromLegacyJsonText, data);
+    let msg = decodeDataFromFile(DataProto.data.DataType.palette_cave, paletteObjFromLegacyJsonText, data);
     return normalizePaletteCaveObjLike(msg);
 }

src/js/shared/data/Dwellings.js

@@ -1,6 +1,7 @@
 import * as DataProto from "../../struct/data.js";
-import { assertExpectedLegacyRootType, decodeDataFromFile } from "./data.js";
+import { assertExpectedLegacyRootType, decodeDataFromFile, encodeDataToBytes } from "./data.js";
 import * as PaletteFunc from "./palette.js";
+import * as FuncBin from "./bin-verify.js";
 
 const COLOR_SPECS = [
     { legacy: "colorInk", proto: "ink" },
@@ -120,7 +121,7 @@ function paletteDwellingsObjFromLegacyJsonInternal(obj) {
     if (!isPlainObject(obj)) throw PaletteFunc.unknownPalette();
     if (Array.isArray(obj.floors) && obj.features == null) throw new Error("Dwellings, not Palette.");
 
-    assertExpectedLegacyRootType("PaletteDwellingsObj", obj);
+    assertExpectedLegacyRootType(DataProto.data.DataType.palette_dwellings, obj);
 
     if (isPlainObject(obj.colors) && isPlainObject(obj.strokes) && isPlainObject(obj.misc)) {
         return normalizePaletteDwellingsObjLike(obj);
@@ -212,10 +213,11 @@ export function paletteLegacyJsonFromObj(pdo) {
 
 export function paletteProtoBytesFromObj(pdo) {
     let n = normalizePaletteDwellingsObjLike(pdo);
-    return DataProto.data.PaletteDwellingsObj.encode(n).finish();
+    let raw = DataProto.data.PaletteDwellingsObj.encode(n).finish();
+    return encodeDataToBytes(DataProto.data.DataType.palette_dwellings, raw);
 }
 
 export function decodePaletteFile(name, data) {
-    let msg = decodeDataFromFile("PaletteDwellingsObj", paletteObjFromLegacyJsonText, data);
+    let msg = decodeDataFromFile(DataProto.data.DataType.palette_dwellings, paletteObjFromLegacyJsonText, data);
     return normalizePaletteDwellingsObjLike(msg);
 }

src/js/shared/data/Glade.js

@@ -1,6 +1,7 @@
 import * as DataProto from "../../struct/data.js";
-import { assertExpectedLegacyRootType, decodeDataFromFile } from "./data.js";
+import { assertExpectedLegacyRootType, decodeDataFromFile, encodeDataToBytes } from "./data.js";
 import * as PaletteFunc from "./palette.js";
+import * as FuncBin from "./bin-verify.js";
 
 const LEGACY_KEYS = [
     "ink", "marks", "tree", "ground", "treeDetails", "thicket",
@@ -147,7 +148,7 @@ export function paletteObjFromLegacyJsonText(text) {
     let obj = null;
     try { obj = JSON.parse(text); } catch (e) { throw new Error("An error occurred while parsing: " + (e && e.message ? e.message : String(e))); }
 
-    assertExpectedLegacyRootType("PaletteGladeObj", obj);
+    assertExpectedLegacyRootType(DataProto.data.DataType.palette_glade, obj);
 
     if (isPlainObject(obj) && isPlainObject(obj.colors) && isPlainObject(obj.trees) && isPlainObject(obj.shadow) && isPlainObject(obj.strokes) && isPlainObject(obj.misc)) {
         return normalizePaletteGladeObjLike(obj);
@@ -329,10 +330,11 @@ export function paletteLegacyJsonFromObj(p) {
 
 export function paletteProtoBytesFromObj(m) {
     let n = normalizePaletteGladeObjLike(m);
-    return DataProto.data.PaletteGladeObj.encode(n).finish();
+    let raw = DataProto.data.PaletteGladeObj.encode(n).finish();
+    return encodeDataToBytes(DataProto.data.DataType.palette_glade, raw);
 }
 
 export function decodePaletteFile(name, data) {
-    let msg = decodeDataFromFile("PaletteGladeObj", paletteObjFromLegacyJsonText, data);
+    let msg = decodeDataFromFile(DataProto.data.DataType.palette_glade, paletteObjFromLegacyJsonText, data);
     return normalizePaletteGladeObjLike(msg);
 }

src/js/shared/data/Viewer.js

@@ -1,6 +1,7 @@
 import * as DataProto from "../../struct/data.js";
-import { assertExpectedLegacyRootType, decodeDataFromFile } from "./data.js";
+import { assertExpectedLegacyRootType, decodeDataFromFile, encodeDataToBytes } from "./data.js";
 import * as PaletteFunc from "./palette.js";
+import * as FuncBin from "./bin-verify.js";
 
 const COLOR_KEYS = ["ground", "fields", "greens", "foliage", "roads", "water", "walls1", "walls2", "roofs1", "roofs2"];
 const LIGHTING_COLOR_KEYS = ["sky1", "sky2", "sun", "windows"];
@@ -106,7 +107,7 @@ export function paletteObjFromLegacyJsonText(text) {
     let obj = null;
     try { obj = JSON.parse(text); } catch (e) { throw new Error("An error occurred while parsing: " + (e && e.message ? e.message : String(e))); }
 
-    assertExpectedLegacyRootType("PaletteViewerObj", obj);
+    assertExpectedLegacyRootType(DataProto.data.DataType.palette_viewer, obj);
 
     if (isPlainObject(obj) && isPlainObject(obj.colors) && isPlainObject(obj.lighting) && isPlainObject(obj.shapes)) {
         return normalizePaletteViewerObjLike(obj);
@@ -202,10 +203,11 @@ export function paletteLegacyJsonFromObj(pvo) {
 
 export function paletteProtoBytesFromObj(pvo) {
     let n = normalizePaletteViewerObjLike(pvo);
-    return DataProto.data.PaletteViewerObj.encode(n).finish();
+    let raw = DataProto.data.PaletteViewerObj.encode(n).finish();
+    return encodeDataToBytes(DataProto.data.DataType.palette_viewer, raw);
 }
 
 export function decodePaletteFile(name, data) {
-    let msg = decodeDataFromFile("PaletteViewerObj", paletteObjFromLegacyJsonText, data);
+    let msg = decodeDataFromFile(DataProto.data.DataType.palette_viewer, paletteObjFromLegacyJsonText, data);
     return normalizePaletteViewerObjLike(msg);
 }