benchmarking: walk RAM after resume (--mem-read) and rotate the churn window

benchmarking/automation/tests.yaml

@@ -97,9 +97,13 @@ tests:
   # cycle is measured against a realistically-sized actor. actorMemory =
   # target + headroom (glutton itself, the guest, and on microvm the 128Mi
   # VMM reserve). The golden snapshot and cold boot stay small: the working
-  # set exists from the first fill onward. --mem-churn re-randomizes 64Mi in
-  # place each cycle, so repeated suspends snapshot changing memory rather
-  # than a static set.
+  # set exists from the first fill onward. --mem-churn re-randomizes 64Mi
+  # each cycle through an advancing window (WriteRAM rotate), so repeated
+  # suspends snapshot changing memory rather than a static set and the dirty
+  # window moves across the array. --mem-read walks the whole working set
+  # right after every resume, one byte per page, so under a demand-paged
+  # restore the GluttonReadRAM row is the cost of paging the previous
+  # snapshot's memory back in.
   - name: glutton_mem_1gi_gvisor
     type: locust
     description: "Large-memory: 1Gi resident working set, gvisor"
@@ -114,6 +118,8 @@ tests:
       - "1Gi"
       - "--mem-churn"
       - "64Mi"
+      - "--mem-read"
+      - "all"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -133,6 +139,8 @@ tests:
       - "1Gi"
       - "--mem-churn"
       - "64Mi"
+      - "--mem-read"
+      - "all"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -151,6 +159,8 @@ tests:
       - "2Gi"
       - "--mem-churn"
       - "64Mi"
+      - "--mem-read"
+      - "all"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"
@@ -170,6 +180,8 @@ tests:
       - "2Gi"
       - "--mem-churn"
       - "64Mi"
+      - "--mem-read"
+      - "all"
       - "--min-wait-time"
       - "1.0"
       - "--max-wait-time"

benchmarking/locust/common/boomer_config.py

@@ -19,7 +19,7 @@
   * --min-wait-time / --max-wait-time → common.wait_time.init_wait_time
   * --resume-mode                   → common.resume_mode.add_resume_mode_arguments
   * --durdir-*                      → common.durdir_config.add_durdir_arguments
-  * --mem-target / --mem-churn      → common.memload_config.add_memload_arguments
+  * --mem-target / --mem-churn / --mem-read → common.memload_config.add_memload_arguments
 
 This module ties them together so boomer-Go workers can pick up the values
 the operator set in the web UI form:
@@ -55,6 +55,7 @@
     "--durdir-template": str,
     "--mem-target": str,
     "--mem-churn": str,
+    "--mem-read": str,
 }
 
 

benchmarking/locust/common/glutton_pb2.py

@@ -38,44 +38,48 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"1\n\x11WriteDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\"D\n\x0fReadDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\tread_mode\x18\x02 \x01(\x0e\x32\x11.glutton.ReadMode\">\n\x10ReadDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*>\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01*9\n\x08ReadMode\x12\x12\n\x0eREAD_MODE_DATA\x10\x00\x12\x19\n\x15READ_MODE_DIGEST_ONLY\x10\x01\x32\x86\x03\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12\x41\n\x08ReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"+\n\x0eReadRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\"1\n\x0fReadRAMResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x10\n\x08\x63hecksum\x18\x02 \x01(\r\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"1\n\x11WriteDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\"D\n\x0fReadDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\tread_mode\x18\x02 \x01(\x0e\x32\x11.glutton.ReadMode\">\n\x10ReadDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*_\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01\x12\x1f\n\x1bWRITE_MODE_OVERWRITE_ROTATE\x10\x02*9\n\x08ReadMode\x12\x12\n\x0eREAD_MODE_DATA\x10\x00\x12\x19\n\x15READ_MODE_DIGEST_ONLY\x10\x01\x32\xc6\x03\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12>\n\x07ReadRAM\x12\x17.glutton.ReadRAMRequest\x1a\x18.glutton.ReadRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12\x41\n\x08ReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'glutton_pb2', _globals)
 if not _descriptor._USE_C_DESCRIPTORS:
   _globals['DESCRIPTOR']._loaded_options = None
   _globals['DESCRIPTOR']._serialized_options = b'Z;github.com/agent-substrate/substrate/internal/proto/glutton'
-  _globals['_WRITEMODE']._serialized_start=624
-  _globals['_WRITEMODE']._serialized_end=686
-  _globals['_READMODE']._serialized_start=688
-  _globals['_READMODE']._serialized_end=745
+  _globals['_WRITEMODE']._serialized_start=720
+  _globals['_WRITEMODE']._serialized_end=815
+  _globals['_READMODE']._serialized_start=817
+  _globals['_READMODE']._serialized_end=874
   _globals['_WRITERAMREQUEST']._serialized_start=26
   _globals['_WRITERAMREQUEST']._serialized_end=110
   _globals['_WRITERAMRESPONSE']._serialized_start=112
   _globals['_WRITERAMRESPONSE']._serialized_end=130
-  _globals['_WRITEDISKREQUEST']._serialized_start=132
-  _globals['_WRITEDISKREQUEST']._serialized_end=217
-  _globals['_WRITEDISKRESPONSE']._serialized_start=219
-  _globals['_WRITEDISKRESPONSE']._serialized_end=268
-  _globals['_READDISKREQUEST']._serialized_start=270
-  _globals['_READDISKREQUEST']._serialized_end=338
-  _globals['_READDISKRESPONSE']._serialized_start=340
-  _globals['_READDISKRESPONSE']._serialized_end=402
-  _globals['_OPENFDREQUEST']._serialized_start=404
-  _globals['_OPENFDREQUEST']._serialized_end=434
-  _globals['_OPENFDRESPONSE']._serialized_start=436
-  _globals['_OPENFDRESPONSE']._serialized_end=452
-  _globals['_PINGREQUEST']._serialized_start=454
-  _globals['_PINGREQUEST']._serialized_end=484
-  _globals['_PINGRESPONSE']._serialized_start=486
-  _globals['_PINGRESPONSE']._serialized_end=517
-  _globals['_GOSSIPREQUEST']._serialized_start=519
-  _globals['_GOSSIPREQUEST']._serialized_end=564
-  _globals['_GOSSIPRESPONSE']._serialized_start=566
-  _globals['_GOSSIPRESPONSE']._serialized_end=582
-  _globals['_PEER']._serialized_start=584
-  _globals['_PEER']._serialized_end=622
-  _globals['_GLUTTON']._serialized_start=748
-  _globals['_GLUTTON']._serialized_end=1138
+  _globals['_READRAMREQUEST']._serialized_start=132
+  _globals['_READRAMREQUEST']._serialized_end=175
+  _globals['_READRAMRESPONSE']._serialized_start=177
+  _globals['_READRAMRESPONSE']._serialized_end=226
+  _globals['_WRITEDISKREQUEST']._serialized_start=228
+  _globals['_WRITEDISKREQUEST']._serialized_end=313
+  _globals['_WRITEDISKRESPONSE']._serialized_start=315
+  _globals['_WRITEDISKRESPONSE']._serialized_end=364
+  _globals['_READDISKREQUEST']._serialized_start=366
+  _globals['_READDISKREQUEST']._serialized_end=434
+  _globals['_READDISKRESPONSE']._serialized_start=436
+  _globals['_READDISKRESPONSE']._serialized_end=498
+  _globals['_OPENFDREQUEST']._serialized_start=500
+  _globals['_OPENFDREQUEST']._serialized_end=530
+  _globals['_OPENFDRESPONSE']._serialized_start=532
+  _globals['_OPENFDRESPONSE']._serialized_end=548
+  _globals['_PINGREQUEST']._serialized_start=550
+  _globals['_PINGREQUEST']._serialized_end=580
+  _globals['_PINGRESPONSE']._serialized_start=582
+  _globals['_PINGRESPONSE']._serialized_end=613
+  _globals['_GOSSIPREQUEST']._serialized_start=615
+  _globals['_GOSSIPREQUEST']._serialized_end=660
+  _globals['_GOSSIPRESPONSE']._serialized_start=662
+  _globals['_GOSSIPRESPONSE']._serialized_end=678
+  _globals['_PEER']._serialized_start=680
+  _globals['_PEER']._serialized_end=718
+  _globals['_GLUTTON']._serialized_start=877
+  _globals['_GLUTTON']._serialized_end=1331
 # @@protoc_insertion_point(module_scope)

benchmarking/locust/common/glutton_pb2_grpc.py

@@ -55,6 +55,11 @@ def __init__(self, channel):
                 request_serializer=glutton__pb2.WriteRAMRequest.SerializeToString,
                 response_deserializer=glutton__pb2.WriteRAMResponse.FromString,
                 _registered_method=True)
+        self.ReadRAM = channel.unary_unary(
+                '/glutton.Glutton/ReadRAM',
+                request_serializer=glutton__pb2.ReadRAMRequest.SerializeToString,
+                response_deserializer=glutton__pb2.ReadRAMResponse.FromString,
+                _registered_method=True)
         self.WriteDisk = channel.unary_unary(
                 '/glutton.Glutton/WriteDisk',
                 request_serializer=glutton__pb2.WriteDiskRequest.SerializeToString,
@@ -96,6 +101,15 @@ def WriteRAM(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def ReadRAM(self, request, context):
+        """Walks RAM previously allocated by WriteRAM: reads one byte per 4KiB
+        page across the requested size, so every touched page must be resident
+        before the response returns.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
     def WriteDisk(self, request, context):
         """Tells glutton to write to disk using the specified mode. Data
         written will be random bytes.
@@ -144,6 +158,11 @@ def add_GluttonServicer_to_server(servicer, server):
                     request_deserializer=glutton__pb2.WriteRAMRequest.FromString,
                     response_serializer=glutton__pb2.WriteRAMResponse.SerializeToString,
             ),
+            'ReadRAM': grpc.unary_unary_rpc_method_handler(
+                    servicer.ReadRAM,
+                    request_deserializer=glutton__pb2.ReadRAMRequest.FromString,
+                    response_serializer=glutton__pb2.ReadRAMResponse.SerializeToString,
+            ),
             'WriteDisk': grpc.unary_unary_rpc_method_handler(
                     servicer.WriteDisk,
                     request_deserializer=glutton__pb2.WriteDiskRequest.FromString,
@@ -209,6 +228,33 @@ def WriteRAM(request,
             metadata,
             _registered_method=True)
 
+    @staticmethod
+    def ReadRAM(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/glutton.Glutton/ReadRAM',
+            glutton__pb2.ReadRAMRequest.SerializeToString,
+            glutton__pb2.ReadRAMResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
     @staticmethod
     def WriteDisk(request,
             target,

benchmarking/locust/common/memload_config.py

@@ -35,9 +35,21 @@ def add_memload_arguments(parser: LocustArgumentParser) -> None:
         "--mem-churn",
         type=str,
         default="",
-        help="How much of the working set each GluttonUser re-randomizes in "
-        "place every cycle (WriteRAM overwrite), with an optional unit "
-        "suffix (e.g. '64Mi'), so repeated suspends snapshot changing "
-        "memory like a live application's (default: empty = disabled). "
+        help="How much of the working set each GluttonUser re-randomizes "
+        "every cycle (WriteRAM rotate: the dirty window advances through "
+        "the array, wrapping at the end), with an optional unit suffix "
+        "(e.g. '64Mi'), so repeated suspends snapshot changing memory "
+        "like a live application's (default: empty = disabled). "
         "Requires --mem-target.",
     )
+    group.add_argument(
+        "--mem-read",
+        type=str,
+        default="",
+        help="How much of the working set each GluttonUser walks (ReadRAM, "
+        "one byte per page) right after every resume, with an optional "
+        "unit suffix (e.g. '1Gi') or 'all' for the whole array. Under a "
+        "demand-paged restore the walk's latency is the cost of paging "
+        "the previous snapshot's memory back in (default: empty = "
+        "disabled). Requires --mem-target.",
+    )

cmd/benchmarking/boomer-glutton/main.go

@@ -42,7 +42,7 @@ func main() {
 		routerURL          = flag.String("router-url", "http://atenet-router.ate-system.svc.cluster.local", "atenet HTTP router base URL (no trailing slash).")
 		atespace           = flag.String("atespace", "benchmark", "Atespace every actor this worker creates lives in. Ensured (CreateAtespace, AlreadyExists is ok) at startup.")
 		promAddr           = flag.String("prometheus-addr", ":8001", "Address for the Prometheus /metrics endpoint.")
-		configJSON         = flag.String("config-json", "", "Initial dynconfig as a JSON object (keys: trace_probability, min_wait_time, max_wait_time in seconds, durdir_file_size_bytes, resume_mode, durdir_read_mode, durdir_template, mem_target, mem_churn). Unset fields keep their built-in defaults.")
+		configJSON         = flag.String("config-json", "", "Initial dynconfig as a JSON object (keys: trace_probability, min_wait_time, max_wait_time in seconds, durdir_file_size_bytes, resume_mode, durdir_read_mode, durdir_template, mem_target, mem_churn, mem_read). Unset fields keep their built-in defaults.")
 		masterWebPort      = flag.Int("master-web-port", 0, "If non-zero, fetch dynconfig from http://{master-host}:{master-web-port}/boomer-config on each spawn message and fail fatally on error. {master-host} comes from boomer's existing --master-host flag.")
 		configPollInterval = flag.Duration("config-poll-interval", 10*time.Second, "With --master-web-port, also fetch dynconfig on this interval. A spawn message comes only when the number of users or the spawn rate changes, thus a load shape that changes the sample rate alone needs this. Zero stops the polling.")
 		userClass          = flag.String("user-class", "glutton", fmt.Sprintf("Locust user class to run, lowercase; one of %s.", strings.Join(userclass.Names(), "|")))