diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index 5f808a4752..ff31413f89 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -97,9 +97,13 @@ tests: # cycle is measured against a realistically-sized actor. actorMemory = # target + headroom (glutton itself, the guest, and on microvm the 128Mi # VMM reserve). The golden snapshot and cold boot stay small: the working - # set exists from the first fill onward. --mem-churn re-randomizes 64Mi in - # place each cycle, so repeated suspends snapshot changing memory rather - # than a static set. + # set exists from the first fill onward. --mem-churn re-randomizes 64Mi + # each cycle through an advancing window (WriteRAM rotate), so repeated + # suspends snapshot changing memory rather than a static set and the dirty + # window moves across the array. --mem-read walks the whole working set + # right after every resume, one byte per page, so under a demand-paged + # restore the GluttonReadRAM row is the cost of paging the previous + # snapshot's memory back in. - name: glutton_mem_1gi_gvisor type: locust description: "Large-memory: 1Gi resident working set, gvisor" @@ -114,6 +118,8 @@ tests: - "1Gi" - "--mem-churn" - "64Mi" + - "--mem-read" + - "all" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -133,6 +139,8 @@ tests: - "1Gi" - "--mem-churn" - "64Mi" + - "--mem-read" + - "all" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -151,6 +159,8 @@ tests: - "2Gi" - "--mem-churn" - "64Mi" + - "--mem-read" + - "all" - "--min-wait-time" - "1.0" - "--max-wait-time" @@ -170,6 +180,8 @@ tests: - "2Gi" - "--mem-churn" - "64Mi" + - "--mem-read" + - "all" - "--min-wait-time" - "1.0" - "--max-wait-time" diff --git a/benchmarking/locust/common/boomer_config.py b/benchmarking/locust/common/boomer_config.py index bfde3dbc13..9494a085d5 100644 --- a/benchmarking/locust/common/boomer_config.py +++ b/benchmarking/locust/common/boomer_config.py @@ -19,7 +19,7 @@ * --min-wait-time / --max-wait-time → common.wait_time.init_wait_time * --resume-mode → common.resume_mode.add_resume_mode_arguments * --durdir-* → common.durdir_config.add_durdir_arguments - * --mem-target / --mem-churn → common.memload_config.add_memload_arguments + * --mem-target / --mem-churn / --mem-read → common.memload_config.add_memload_arguments This module ties them together so boomer-Go workers can pick up the values the operator set in the web UI form: @@ -55,6 +55,7 @@ "--durdir-template": str, "--mem-target": str, "--mem-churn": str, + "--mem-read": str, } diff --git a/benchmarking/locust/common/glutton_pb2.py b/benchmarking/locust/common/glutton_pb2.py index 2fdc341a11..57c4ddf1b0 100644 --- a/benchmarking/locust/common/glutton_pb2.py +++ b/benchmarking/locust/common/glutton_pb2.py @@ -38,7 +38,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"1\n\x11WriteDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\"D\n\x0fReadDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\tread_mode\x18\x02 \x01(\x0e\x32\x11.glutton.ReadMode\">\n\x10ReadDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*>\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01*9\n\x08ReadMode\x12\x12\n\x0eREAD_MODE_DATA\x10\x00\x12\x19\n\x15READ_MODE_DIGEST_ONLY\x10\x01\x32\x86\x03\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12\x41\n\x08ReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"+\n\x0eReadRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\t\"1\n\x0fReadRAMResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x10\n\x08\x63hecksum\x18\x02 \x01(\r\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"1\n\x11WriteDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\"D\n\x0fReadDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\tread_mode\x18\x02 \x01(\x0e\x32\x11.glutton.ReadMode\">\n\x10ReadDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*_\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01\x12\x1f\n\x1bWRITE_MODE_OVERWRITE_ROTATE\x10\x02*9\n\x08ReadMode\x12\x12\n\x0eREAD_MODE_DATA\x10\x00\x12\x19\n\x15READ_MODE_DIGEST_ONLY\x10\x01\x32\xc6\x03\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12>\n\x07ReadRAM\x12\x17.glutton.ReadRAMRequest\x1a\x18.glutton.ReadRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12\x41\n\x08ReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -46,36 +46,40 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'Z;github.com/agent-substrate/substrate/internal/proto/glutton' - _globals['_WRITEMODE']._serialized_start=624 - _globals['_WRITEMODE']._serialized_end=686 - _globals['_READMODE']._serialized_start=688 - _globals['_READMODE']._serialized_end=745 + _globals['_WRITEMODE']._serialized_start=720 + _globals['_WRITEMODE']._serialized_end=815 + _globals['_READMODE']._serialized_start=817 + _globals['_READMODE']._serialized_end=874 _globals['_WRITERAMREQUEST']._serialized_start=26 _globals['_WRITERAMREQUEST']._serialized_end=110 _globals['_WRITERAMRESPONSE']._serialized_start=112 _globals['_WRITERAMRESPONSE']._serialized_end=130 - _globals['_WRITEDISKREQUEST']._serialized_start=132 - _globals['_WRITEDISKREQUEST']._serialized_end=217 - _globals['_WRITEDISKRESPONSE']._serialized_start=219 - _globals['_WRITEDISKRESPONSE']._serialized_end=268 - _globals['_READDISKREQUEST']._serialized_start=270 - _globals['_READDISKREQUEST']._serialized_end=338 - _globals['_READDISKRESPONSE']._serialized_start=340 - _globals['_READDISKRESPONSE']._serialized_end=402 - _globals['_OPENFDREQUEST']._serialized_start=404 - _globals['_OPENFDREQUEST']._serialized_end=434 - _globals['_OPENFDRESPONSE']._serialized_start=436 - _globals['_OPENFDRESPONSE']._serialized_end=452 - _globals['_PINGREQUEST']._serialized_start=454 - _globals['_PINGREQUEST']._serialized_end=484 - _globals['_PINGRESPONSE']._serialized_start=486 - _globals['_PINGRESPONSE']._serialized_end=517 - _globals['_GOSSIPREQUEST']._serialized_start=519 - _globals['_GOSSIPREQUEST']._serialized_end=564 - _globals['_GOSSIPRESPONSE']._serialized_start=566 - _globals['_GOSSIPRESPONSE']._serialized_end=582 - _globals['_PEER']._serialized_start=584 - _globals['_PEER']._serialized_end=622 - _globals['_GLUTTON']._serialized_start=748 - _globals['_GLUTTON']._serialized_end=1138 + _globals['_READRAMREQUEST']._serialized_start=132 + _globals['_READRAMREQUEST']._serialized_end=175 + _globals['_READRAMRESPONSE']._serialized_start=177 + _globals['_READRAMRESPONSE']._serialized_end=226 + _globals['_WRITEDISKREQUEST']._serialized_start=228 + _globals['_WRITEDISKREQUEST']._serialized_end=313 + _globals['_WRITEDISKRESPONSE']._serialized_start=315 + _globals['_WRITEDISKRESPONSE']._serialized_end=364 + _globals['_READDISKREQUEST']._serialized_start=366 + _globals['_READDISKREQUEST']._serialized_end=434 + _globals['_READDISKRESPONSE']._serialized_start=436 + _globals['_READDISKRESPONSE']._serialized_end=498 + _globals['_OPENFDREQUEST']._serialized_start=500 + _globals['_OPENFDREQUEST']._serialized_end=530 + _globals['_OPENFDRESPONSE']._serialized_start=532 + _globals['_OPENFDRESPONSE']._serialized_end=548 + _globals['_PINGREQUEST']._serialized_start=550 + _globals['_PINGREQUEST']._serialized_end=580 + _globals['_PINGRESPONSE']._serialized_start=582 + _globals['_PINGRESPONSE']._serialized_end=613 + _globals['_GOSSIPREQUEST']._serialized_start=615 + _globals['_GOSSIPREQUEST']._serialized_end=660 + _globals['_GOSSIPRESPONSE']._serialized_start=662 + _globals['_GOSSIPRESPONSE']._serialized_end=678 + _globals['_PEER']._serialized_start=680 + _globals['_PEER']._serialized_end=718 + _globals['_GLUTTON']._serialized_start=877 + _globals['_GLUTTON']._serialized_end=1331 # @@protoc_insertion_point(module_scope) diff --git a/benchmarking/locust/common/glutton_pb2_grpc.py b/benchmarking/locust/common/glutton_pb2_grpc.py index 57a7fae6d6..decf881fb5 100644 --- a/benchmarking/locust/common/glutton_pb2_grpc.py +++ b/benchmarking/locust/common/glutton_pb2_grpc.py @@ -55,6 +55,11 @@ def __init__(self, channel): request_serializer=glutton__pb2.WriteRAMRequest.SerializeToString, response_deserializer=glutton__pb2.WriteRAMResponse.FromString, _registered_method=True) + self.ReadRAM = channel.unary_unary( + '/glutton.Glutton/ReadRAM', + request_serializer=glutton__pb2.ReadRAMRequest.SerializeToString, + response_deserializer=glutton__pb2.ReadRAMResponse.FromString, + _registered_method=True) self.WriteDisk = channel.unary_unary( '/glutton.Glutton/WriteDisk', request_serializer=glutton__pb2.WriteDiskRequest.SerializeToString, @@ -96,6 +101,15 @@ def WriteRAM(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ReadRAM(self, request, context): + """Walks RAM previously allocated by WriteRAM: reads one byte per 4KiB + page across the requested size, so every touched page must be resident + before the response returns. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def WriteDisk(self, request, context): """Tells glutton to write to disk using the specified mode. Data written will be random bytes. @@ -144,6 +158,11 @@ def add_GluttonServicer_to_server(servicer, server): request_deserializer=glutton__pb2.WriteRAMRequest.FromString, response_serializer=glutton__pb2.WriteRAMResponse.SerializeToString, ), + 'ReadRAM': grpc.unary_unary_rpc_method_handler( + servicer.ReadRAM, + request_deserializer=glutton__pb2.ReadRAMRequest.FromString, + response_serializer=glutton__pb2.ReadRAMResponse.SerializeToString, + ), 'WriteDisk': grpc.unary_unary_rpc_method_handler( servicer.WriteDisk, request_deserializer=glutton__pb2.WriteDiskRequest.FromString, @@ -209,6 +228,33 @@ def WriteRAM(request, metadata, _registered_method=True) + @staticmethod + def ReadRAM(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/glutton.Glutton/ReadRAM', + glutton__pb2.ReadRAMRequest.SerializeToString, + glutton__pb2.ReadRAMResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def WriteDisk(request, target, diff --git a/benchmarking/locust/common/memload_config.py b/benchmarking/locust/common/memload_config.py index 52a7b1b0ba..b1077058f9 100644 --- a/benchmarking/locust/common/memload_config.py +++ b/benchmarking/locust/common/memload_config.py @@ -35,9 +35,21 @@ def add_memload_arguments(parser: LocustArgumentParser) -> None: "--mem-churn", type=str, default="", - help="How much of the working set each GluttonUser re-randomizes in " - "place every cycle (WriteRAM overwrite), with an optional unit " - "suffix (e.g. '64Mi'), so repeated suspends snapshot changing " - "memory like a live application's (default: empty = disabled). " + help="How much of the working set each GluttonUser re-randomizes " + "every cycle (WriteRAM rotate: the dirty window advances through " + "the array, wrapping at the end), with an optional unit suffix " + "(e.g. '64Mi'), so repeated suspends snapshot changing memory " + "like a live application's (default: empty = disabled). " "Requires --mem-target.", ) + group.add_argument( + "--mem-read", + type=str, + default="", + help="How much of the working set each GluttonUser walks (ReadRAM, " + "one byte per page) right after every resume, with an optional " + "unit suffix (e.g. '1Gi') or 'all' for the whole array. Under a " + "demand-paged restore the walk's latency is the cost of paging " + "the previous snapshot's memory back in (default: empty = " + "disabled). Requires --mem-target.", + ) diff --git a/cmd/benchmarking/boomer-glutton/main.go b/cmd/benchmarking/boomer-glutton/main.go index 8f4ddefa40..86cb487286 100644 --- a/cmd/benchmarking/boomer-glutton/main.go +++ b/cmd/benchmarking/boomer-glutton/main.go @@ -42,7 +42,7 @@ func main() { routerURL = flag.String("router-url", "http://atenet-router.ate-system.svc.cluster.local", "atenet HTTP router base URL (no trailing slash).") atespace = flag.String("atespace", "benchmark", "Atespace every actor this worker creates lives in. Ensured (CreateAtespace, AlreadyExists is ok) at startup.") promAddr = flag.String("prometheus-addr", ":8001", "Address for the Prometheus /metrics endpoint.") - configJSON = flag.String("config-json", "", "Initial dynconfig as a JSON object (keys: trace_probability, min_wait_time, max_wait_time in seconds, durdir_file_size_bytes, resume_mode, durdir_read_mode, durdir_template, mem_target, mem_churn). Unset fields keep their built-in defaults.") + configJSON = flag.String("config-json", "", "Initial dynconfig as a JSON object (keys: trace_probability, min_wait_time, max_wait_time in seconds, durdir_file_size_bytes, resume_mode, durdir_read_mode, durdir_template, mem_target, mem_churn, mem_read). Unset fields keep their built-in defaults.") masterWebPort = flag.Int("master-web-port", 0, "If non-zero, fetch dynconfig from http://{master-host}:{master-web-port}/boomer-config on each spawn message and fail fatally on error. {master-host} comes from boomer's existing --master-host flag.") configPollInterval = flag.Duration("config-poll-interval", 10*time.Second, "With --master-web-port, also fetch dynconfig on this interval. A spawn message comes only when the number of users or the spawn rate changes, thus a load shape that changes the sample rate alone needs this. Zero stops the polling.") userClass = flag.String("user-class", "glutton", fmt.Sprintf("Locust user class to run, lowercase; one of %s.", strings.Join(userclass.Names(), "|"))) diff --git a/cmd/benchmarking/glutton/main.go b/cmd/benchmarking/glutton/main.go index 7153f0c9c0..5488402745 100644 --- a/cmd/benchmarking/glutton/main.go +++ b/cmd/benchmarking/glutton/main.go @@ -186,6 +186,7 @@ func newMux(svc *gluttonService) *http.ServeMux { mux.HandleFunc("/writedisk", protoRoute("WriteDisk", svc.WriteDisk)) mux.HandleFunc("/readdisk", protoRoute("ReadDisk", svc.ReadDisk)) mux.HandleFunc("/writeram", protoRoute("WriteRAM", svc.WriteRAM)) + mux.HandleFunc("/readram", protoRoute("ReadRAM", svc.ReadRAM)) return mux } @@ -256,12 +257,16 @@ type gluttonService struct { // TODO: split this into per-resource locks (ram, fds, peers). A single // global mutex serializes unrelated operations across all three. - mu sync.Mutex - ram map[string][]byte - fds []*os.File - peers map[string]*peerGossip + mu sync.Mutex + ram map[string][]byte + // ramCursor is each array's next WRITE_MODE_OVERWRITE_ROTATE offset. + // Absent means 0; invalidated whenever the array is reallocated. + ramCursor map[string]int + fds []*os.File + peers map[string]*peerGossip ramWriteBytes metric.Int64Counter + ramReadBytes metric.Int64Counter diskWriteBytes metric.Int64Counter diskReadBytes metric.Int64Counter pingsReceived metric.Int64Counter @@ -278,9 +283,10 @@ type peerGossip struct { func newGluttonService(dir string) (*gluttonService, error) { s := &gluttonService{ - dataDir: dir, - ram: make(map[string][]byte), - peers: make(map[string]*peerGossip), + dataDir: dir, + ram: make(map[string][]byte), + ramCursor: make(map[string]int), + peers: make(map[string]*peerGossip), } m := otel.Meter(meterName) @@ -294,6 +300,14 @@ func newGluttonService(dir string) (*gluttonService, error) { if err != nil { return nil, fmt.Errorf("create glutton.ram.write.bytes counter: %w", err) } + s.ramReadBytes, err = m.Int64Counter( + "glutton.ram.read.bytes", + metric.WithUnit("By"), + metric.WithDescription("Total bytes walked by ReadRAM over the process lifetime."), + ) + if err != nil { + return nil, fmt.Errorf("create glutton.ram.read.bytes counter: %w", err) + } s.diskWriteBytes, err = m.Int64Counter( "glutton.disk.write.bytes", metric.WithUnit("By"), @@ -402,6 +416,7 @@ func (s *gluttonService) WriteRAM(ctx context.Context, req *glutton.WriteRAMRequ } s.mu.Lock() s.ram[req.GetKey()] = buf + delete(s.ramCursor, req.GetKey()) s.mu.Unlock() case glutton.WriteMode_WRITE_MODE_OVERWRITE: s.mu.Lock() @@ -409,12 +424,17 @@ func (s *gluttonService) WriteRAM(ctx context.Context, req *glutton.WriteRAMRequ if size > len(existing) { existing = make([]byte, size) s.ram[req.GetKey()] = existing + delete(s.ramCursor, req.GetKey()) } if _, err := rand.Read(existing[:size]); err != nil { s.mu.Unlock() return nil, status.Errorf(codes.Internal, "generate random bytes: %v", err) } s.mu.Unlock() + case glutton.WriteMode_WRITE_MODE_OVERWRITE_ROTATE: + if err := s.rotateRAM(req.GetKey(), size); err != nil { + return nil, err + } default: return nil, status.Errorf(codes.InvalidArgument, "unknown write_mode %v", req.GetWriteMode()) } @@ -423,6 +443,72 @@ func (s *gluttonService) WriteRAM(ctx context.Context, req *glutton.WriteRAMRequ return &glutton.WriteRAMResponse{}, nil } +// rotateRAM re-randomizes size bytes starting at the key's cursor, wrapping +// at the end of the array, then advances the cursor past the write. Repeated +// rotates therefore walk the whole array instead of re-dirtying the same +// prefix. The cursor lives in process memory, so it rides along in snapshots +// and the walk keeps advancing across suspend/resume cycles. +func (s *gluttonService) rotateRAM(key string, size int) error { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.ram[key] + if len(existing) == 0 { + return status.Errorf(codes.NotFound, "rotate needs an existing array %q; fill with TRUNCATE first", key) + } + if size > len(existing) { + size = len(existing) + } + start := s.ramCursor[key] + head := existing[start:min(start+size, len(existing))] + if _, err := rand.Read(head); err != nil { + return status.Errorf(codes.Internal, "generate random bytes: %v", err) + } + if wrapped := size - len(head); wrapped > 0 { + if _, err := rand.Read(existing[:wrapped]); err != nil { + return status.Errorf(codes.Internal, "generate random bytes: %v", err) + } + } + s.ramCursor[key] = (start + size) % len(existing) + return nil +} + +// pageSize is the stride of the ReadRAM walk: one byte per 4KiB page is +// enough to force every page resident without the cost of reading them all. +const pageSize = 4096 + +// Walk RAM previously written by WriteRAM, reading one byte per page so +// every touched page must be resident before the response returns. After a +// demand-paged restore this converts restore-time laziness into measurable +// read latency. +func (s *gluttonService) ReadRAM(ctx context.Context, req *glutton.ReadRAMRequest) (*glutton.ReadRAMResponse, error) { + if req.GetKey() == "" { + return nil, status.Error(codes.InvalidArgument, "key is required") + } + s.mu.Lock() + defer s.mu.Unlock() + arr, ok := s.ram[req.GetKey()] + if !ok { + return nil, status.Errorf(codes.NotFound, "no RAM array %q", req.GetKey()) + } + walk := int64(len(arr)) + if req.GetSize() != "" { + n, err := parseBytes(req.GetSize()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "size: %v", err) + } + if n < 0 { + return nil, status.Error(codes.InvalidArgument, "size must be non-negative") + } + walk = min(n, walk) + } + var sum uint32 + for i := int64(0); i < walk; i += pageSize { + sum ^= uint32(arr[i]) + } + s.ramReadBytes.Add(ctx, walk) + return &glutton.ReadRAMResponse{Size: walk, Checksum: sum}, nil +} + // Write to disk using the specified mode. Data written will be random bytes. func (s *gluttonService) WriteDisk(ctx context.Context, req *glutton.WriteDiskRequest) (*glutton.WriteDiskResponse, error) { if !diskKeyRE.MatchString(req.GetKey()) { diff --git a/cmd/benchmarking/glutton/ram_test.go b/cmd/benchmarking/glutton/ram_test.go new file mode 100644 index 0000000000..569a24b500 --- /dev/null +++ b/cmd/benchmarking/glutton/ram_test.go @@ -0,0 +1,220 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "testing" + + "github.com/agent-substrate/substrate/internal/proto/glutton" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newRAMTestService(t *testing.T) *gluttonService { + t.Helper() + svc, err := newGluttonService(t.TempDir()) + if err != nil { + t.Fatalf("failed to create glutton service: %v", err) + } + t.Cleanup(svc.Close) + return svc +} + +func fillRAM(t *testing.T, svc *gluttonService, key, size string) { + t.Helper() + _, err := svc.WriteRAM(context.Background(), &glutton.WriteRAMRequest{ + Key: key, Size: size, WriteMode: glutton.WriteMode_WRITE_MODE_TRUNCATE, + }) + if err != nil { + t.Fatalf("WriteRAM truncate %s (%s): %v", key, size, err) + } +} + +func rotateRAM(t *testing.T, svc *gluttonService, key, size string) { + t.Helper() + _, err := svc.WriteRAM(context.Background(), &glutton.WriteRAMRequest{ + Key: key, Size: size, WriteMode: glutton.WriteMode_WRITE_MODE_OVERWRITE_ROTATE, + }) + if err != nil { + t.Fatalf("WriteRAM rotate %s (%s): %v", key, size, err) + } +} + +// ramCopy snapshots the current bytes of a RAM array for change comparison. +func ramCopy(svc *gluttonService, key string) []byte { + svc.mu.Lock() + defer svc.mu.Unlock() + return append([]byte(nil), svc.ram[key]...) +} + +func TestReadRAMWalksArray(t *testing.T) { + svc := newRAMTestService(t) + ctx := context.Background() + fillRAM(t, svc, "m", "64Ki") + + whole, err := svc.ReadRAM(ctx, &glutton.ReadRAMRequest{Key: "m"}) + if err != nil { + t.Fatalf("ReadRAM whole: %v", err) + } + if whole.GetSize() != 64<<10 { + t.Errorf("whole walk size = %d, want %d", whole.GetSize(), 64<<10) + } + + again, err := svc.ReadRAM(ctx, &glutton.ReadRAMRequest{Key: "m"}) + if err != nil { + t.Fatalf("ReadRAM repeat: %v", err) + } + if again.GetChecksum() != whole.GetChecksum() { + t.Errorf("repeat checksum = %d, want %d (walk must be deterministic)", again.GetChecksum(), whole.GetChecksum()) + } + + partial, err := svc.ReadRAM(ctx, &glutton.ReadRAMRequest{Key: "m", Size: "4Ki"}) + if err != nil { + t.Fatalf("ReadRAM partial: %v", err) + } + if partial.GetSize() != 4<<10 { + t.Errorf("partial walk size = %d, want %d", partial.GetSize(), 4<<10) + } + + clamped, err := svc.ReadRAM(ctx, &glutton.ReadRAMRequest{Key: "m", Size: "1Gi"}) + if err != nil { + t.Fatalf("ReadRAM oversized: %v", err) + } + if clamped.GetSize() != 64<<10 { + t.Errorf("oversized walk size = %d, want %d (clamped to the array)", clamped.GetSize(), 64<<10) + } +} + +func TestReadRAMErrors(t *testing.T) { + svc := newRAMTestService(t) + ctx := context.Background() + fillRAM(t, svc, "m", "4Ki") + + tests := []struct { + name string + req *glutton.ReadRAMRequest + code codes.Code + }{ + {name: "empty key", req: &glutton.ReadRAMRequest{}, code: codes.InvalidArgument}, + {name: "missing key", req: &glutton.ReadRAMRequest{Key: "nope"}, code: codes.NotFound}, + {name: "bad size", req: &glutton.ReadRAMRequest{Key: "m", Size: "lots"}, code: codes.InvalidArgument}, + {name: "negative size", req: &glutton.ReadRAMRequest{Key: "m", Size: "-1"}, code: codes.InvalidArgument}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := svc.ReadRAM(ctx, tt.req) + if status.Code(err) != tt.code { + t.Errorf("ReadRAM error = %v, want code %v", err, tt.code) + } + }) + } +} + +// TestWriteRAMRotateAdvances verifies consecutive rotates dirty a moving +// window: first [0,4Ki), then [4Ki,8Ki), never the untouched remainder. A +// 4Ki random block matching its previous contents is astronomically +// unlikely, so byte comparison is a reliable change detector. +func TestWriteRAMRotateAdvances(t *testing.T) { + svc := newRAMTestService(t) + fillRAM(t, svc, "m", "8Ki") + before := ramCopy(svc, "m") + + rotateRAM(t, svc, "m", "4Ki") + afterFirst := ramCopy(svc, "m") + if bytes.Equal(before[:4<<10], afterFirst[:4<<10]) { + t.Error("first rotate left [0,4Ki) unchanged") + } + if !bytes.Equal(before[4<<10:], afterFirst[4<<10:]) { + t.Error("first rotate touched [4Ki,8Ki)") + } + + rotateRAM(t, svc, "m", "4Ki") + afterSecond := ramCopy(svc, "m") + if !bytes.Equal(afterFirst[:4<<10], afterSecond[:4<<10]) { + t.Error("second rotate touched [0,4Ki); want the window to advance") + } + if bytes.Equal(afterFirst[4<<10:], afterSecond[4<<10:]) { + t.Error("second rotate left [4Ki,8Ki) unchanged") + } +} + +// TestWriteRAMRotateWraps drives the cursor past the end of the array: an +// 8Ki array rotated by 6Ki twice writes [6Ki,8Ki) plus the wrapped [0,4Ki) +// on the second call, leaving [4Ki,6Ki) untouched. +func TestWriteRAMRotateWraps(t *testing.T) { + svc := newRAMTestService(t) + fillRAM(t, svc, "m", "8Ki") + + rotateRAM(t, svc, "m", "6Ki") + afterFirst := ramCopy(svc, "m") + + rotateRAM(t, svc, "m", "6Ki") + afterSecond := ramCopy(svc, "m") + if bytes.Equal(afterFirst[6<<10:], afterSecond[6<<10:]) { + t.Error("wrapping rotate left [6Ki,8Ki) unchanged") + } + if bytes.Equal(afterFirst[:4<<10], afterSecond[:4<<10]) { + t.Error("wrapping rotate left the wrapped [0,4Ki) unchanged") + } + if !bytes.Equal(afterFirst[4<<10:6<<10], afterSecond[4<<10:6<<10]) { + t.Error("wrapping rotate touched [4Ki,6Ki)") + } +} + +func TestWriteRAMRotateClampsAndKeepsRotating(t *testing.T) { + svc := newRAMTestService(t) + fillRAM(t, svc, "m", "4Ki") + before := ramCopy(svc, "m") + + // Oversized rotate rewrites the whole array and must not corrupt the + // cursor: the follow-up rotate still succeeds. + rotateRAM(t, svc, "m", "1Mi") + if bytes.Equal(before, ramCopy(svc, "m")) { + t.Error("oversized rotate left the array unchanged") + } + rotateRAM(t, svc, "m", "1Ki") +} + +func TestWriteRAMRotateNeedsExistingArray(t *testing.T) { + svc := newRAMTestService(t) + _, err := svc.WriteRAM(context.Background(), &glutton.WriteRAMRequest{ + Key: "nope", Size: "4Ki", WriteMode: glutton.WriteMode_WRITE_MODE_OVERWRITE_ROTATE, + }) + if status.Code(err) != codes.NotFound { + t.Errorf("rotate on missing array = %v, want NotFound", err) + } +} + +// TestWriteRAMTruncateResetsRotateCursor re-fills after rotating and checks +// the next rotate starts back at the head of the new array. +func TestWriteRAMTruncateResetsRotateCursor(t *testing.T) { + svc := newRAMTestService(t) + fillRAM(t, svc, "m", "8Ki") + rotateRAM(t, svc, "m", "4Ki") // cursor now 4Ki + + fillRAM(t, svc, "m", "8Ki") // reallocates; cursor must reset + before := ramCopy(svc, "m") + + rotateRAM(t, svc, "m", "4Ki") + after := ramCopy(svc, "m") + if bytes.Equal(before[:4<<10], after[:4<<10]) { + t.Error("rotate after truncate left [0,4Ki) unchanged; cursor was not reset") + } + if !bytes.Equal(before[4<<10:], after[4<<10:]) { + t.Error("rotate after truncate touched [4Ki,8Ki)") + } +} diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig.go b/internal/benchmarking/boomer/dynconfig/dynconfig.go index bb01dd98fc..fae92eea73 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig.go @@ -55,7 +55,8 @@ type Config struct { DurDirReadMode string // ReadModeData | ReadModeDigest DurDirTemplate string // ActorTemplate name MemTarget string // resident RAM the GluttonUser fills via WriteRAM, suffixed (e.g. "2Gi"); "" disables - MemChurn string // RAM re-randomized in place each cycle via WriteRAM overwrite, suffixed (e.g. "64Mi"); "" disables + MemChurn string // RAM re-randomized in place each cycle via WriteRAM rotate, suffixed (e.g. "64Mi"); "" disables + MemRead string // RAM walked (one byte per page) via ReadRAM after each resume, suffixed (e.g. "1Gi") or "all"; "" disables } // Holder lets readers Load() the current Config and writers Store() a new @@ -95,6 +96,7 @@ type payload struct { DurDirTemplate *string `json:"durdir_template"` MemTarget *string `json:"mem_target"` MemChurn *string `json:"mem_churn"` + MemRead *string `json:"mem_read"` } // Parse decodes a JSON blob (typically from a CLI flag) and merges its @@ -167,9 +169,10 @@ func (c Config) Validate() error { if c.DurDirReadMode != "" && c.DurDirReadMode != ReadModeData && c.DurDirReadMode != ReadModeDigest { return fmt.Errorf("invalid durdir_read_mode %q: must be %q or %q", c.DurDirReadMode, ReadModeData, ReadModeDigest) } - // MemTarget and MemChurn are passed to glutton verbatim, which owns the - // parse; invalid values fail loudly there as GluttonFillRAM / - // GluttonChurnRAM errors. + // MemTarget, MemChurn, and MemRead are passed to glutton verbatim + // (MemRead's "all" excepted, which the driver maps to an empty + // whole-array walk), which owns the parse; invalid values fail loudly + // there as GluttonFillRAM / GluttonChurnRAM / GluttonReadRAM errors. return nil } @@ -205,6 +208,9 @@ func (p payload) merge(current Config) Config { if p.MemChurn != nil { out.MemChurn = *p.MemChurn } + if p.MemRead != nil { + out.MemRead = *p.MemRead + } return out } @@ -272,6 +278,7 @@ func StartPoll( slog.String("durdir_template", next.DurDirTemplate), slog.String("mem_target", next.MemTarget), slog.String("mem_churn", next.MemChurn), + slog.String("mem_read", next.MemRead), ) } } @@ -307,6 +314,7 @@ func SubscribeSpawn(url string, holder *Holder, sampler ProbabilityUpdater, fetc slog.String("durdir_template", next.DurDirTemplate), slog.String("mem_target", next.MemTarget), slog.String("mem_churn", next.MemChurn), + slog.String("mem_read", next.MemRead), ) }) } diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 9e36bf9df9..29438bdac3 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -57,6 +57,9 @@ const ( actorDomain = "actors.resources.substrate.ate.dev" pingPath = "/ping" writeRAMPath = "/writeram" + readRAMPath = "/readram" + memLoadKey = "memload" + memReadAll = "all" sourceClient = "client" sourceServer = "server" @@ -113,8 +116,15 @@ func (r *taskRuntime) iterate() { // carries the full working set; glutton keeps the allocations across // suspend/resume, so this runs once per actor (retried if it fails). user.ensureRAMFilled(ctx) + // Walk the working set right after resume, before churn dirties it: + // under a demand-paged restore every touched page must be paged back + // in before the walk returns, so its latency measures the true cost + // of reaching the previous snapshot's memory. + user.readRAM(ctx) // Re-dirty part of the working set each cycle so repeated suspends // snapshot an actor whose memory is changing, like a live application's. + // Rotate mode advances through the array cycle over cycle, so the dirty + // window moves instead of re-dirtying the same prefix. user.churnRAM(ctx) user.ping(ctx) user.suspend(ctx) @@ -358,7 +368,7 @@ func (u *gluttonUser) ensureRAMFilled(ctx context.Context) { defer span.End() start := time.Now() - err := u.writeRAM(ctx, "memload", target, gluttonpb.WriteMode_WRITE_MODE_TRUNCATE) + err := u.writeRAM(ctx, memLoadKey, target, gluttonpb.WriteMode_WRITE_MODE_TRUNCATE) clientLatency := time.Since(start) logSampledTrace(span, "GluttonFillRAM", clientLatency, sourceClient, err) if err != nil { @@ -369,13 +379,15 @@ func (u *gluttonUser) ensureRAMFilled(ctx context.Context) { bmetrics.RecordSuccess("http", "GluttonFillRAM", userClass, clientLatency, 0) } -// churnRAM re-randomizes the first mem_churn bytes of the working set in -// place (WriteRAM overwrite on the fill's key), so pages arrive dirty at -// every suspend instead of only the first: a fill-once set is static, and -// any future incremental snapshotting would make cycles two onward -// unrepresentative of a live application. Runs once per iteration, only -// after the fill has succeeded, and reports as its own GluttonChurnRAM -// stats row. +// churnRAM re-randomizes mem_churn bytes of the working set in place +// (WriteRAM rotate on the fill's key), so pages arrive dirty at every +// suspend instead of only the first: a fill-once set is static, and any +// future incremental snapshotting would make cycles two onward +// unrepresentative of a live application. Rotate mode advances glutton's +// per-key cursor past each write, wrapping at the end, so consecutive +// cycles dirty a moving window rather than the same prefix. Runs once per +// iteration, only after the fill has succeeded, and reports as its own +// GluttonChurnRAM stats row. func (u *gluttonUser) churnRAM(ctx context.Context) { churn := u.cfg.Dyn.Load().MemChurn if churn == "" || !u.ramFilled { @@ -386,7 +398,7 @@ func (u *gluttonUser) churnRAM(ctx context.Context) { defer span.End() start := time.Now() - err := u.writeRAM(ctx, "memload", churn, gluttonpb.WriteMode_WRITE_MODE_OVERWRITE) + err := u.writeRAM(ctx, memLoadKey, churn, gluttonpb.WriteMode_WRITE_MODE_OVERWRITE_ROTATE) clientLatency := time.Since(start) logSampledTrace(span, "GluttonChurnRAM", clientLatency, sourceClient, err) if err != nil { @@ -396,19 +408,60 @@ func (u *gluttonUser) churnRAM(ctx context.Context) { bmetrics.RecordSuccess("http", "GluttonChurnRAM", userClass, clientLatency, 0) } +// readRAM walks mem_read bytes of the working set (memReadAll walks all of +// it) through the glutton ReadRAM API, one byte per page, and reports the +// walk as its own GluttonReadRAM stats row. Placed right after resume, the +// row's latency is the demand-paging cost of the previous snapshot's +// memory; on an eagerly-restored actor it degenerates to a fast in-memory +// scan, so the two restore modes are directly comparable. +func (u *gluttonUser) readRAM(ctx context.Context) { + read := u.cfg.Dyn.Load().MemRead + if read == "" || !u.ramFilled { + return + } + size := read + if read == memReadAll { + size = "" // ReadRAM walks the whole array on empty size + } + + ctx, span := u.cfg.Tracer.Start(ctx, "GluttonReadRAM") + defer span.End() + start := time.Now() + + resp := &gluttonpb.ReadRAMResponse{} + err := u.postProto(ctx, readRAMPath, &gluttonpb.ReadRAMRequest{Key: memLoadKey, Size: size}, resp) + clientLatency := time.Since(start) + logSampledTrace(span, "GluttonReadRAM", clientLatency, sourceClient, err) + if err != nil { + bmetrics.RecordFailure("http", "GluttonReadRAM", userClass, clientLatency, err.Error()) + return + } + bmetrics.RecordSuccess("http", "GluttonReadRAM", userClass, clientLatency, resp.GetSize()) +} + // writeRAM POSTs one WriteRAM request to the actor through the router, // mirroring ping's wire format (protobuf over HTTP). size is a suffixed // string (e.g. "2Gi") passed through verbatim; glutton parses it. func (u *gluttonUser) writeRAM(ctx context.Context, key, size string, mode gluttonpb.WriteMode) error { - body, err := proto.Marshal(&gluttonpb.WriteRAMRequest{ + err := u.postProto(ctx, writeRAMPath, &gluttonpb.WriteRAMRequest{ Key: key, Size: size, WriteMode: mode, - }) + }, &gluttonpb.WriteRAMResponse{}) + if err != nil { + return fmt.Errorf("WriteRAM %s (%s): %w", key, size, err) + } + return nil +} + +// postProto POSTs one protobuf request to the actor through the router and +// unmarshals the protobuf response into resp. +func (u *gluttonUser) postProto(ctx context.Context, path string, req, resp proto.Message) error { + body, err := proto.Marshal(req) if err != nil { return err } - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.cfg.RouterURL+writeRAMPath, bytes.NewReader(body)) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.cfg.RouterURL+path, bytes.NewReader(body)) if err != nil { return err } @@ -416,19 +469,19 @@ func (u *gluttonUser) writeRAM(ctx context.Context, key, size string, mode glutt httpReq.Header.Set("Content-Type", "application/x-protobuf") otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) - resp, err := u.cfg.HTTPClient.Do(httpReq) + httpResp, err := u.cfg.HTTPClient.Do(httpReq) if err != nil { return err } - defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) + defer httpResp.Body.Close() + respBody, err := io.ReadAll(httpResp.Body) if err != nil { return err } - if resp.StatusCode >= 400 { - return fmt.Errorf("WriteRAM %s (%s): HTTP %d: %s", key, size, resp.StatusCode, strings.TrimSpace(string(respBody))) + if httpResp.StatusCode >= 400 { + return fmt.Errorf("%s: HTTP %d: %s", path, httpResp.StatusCode, strings.TrimSpace(string(respBody))) } - return nil + return proto.Unmarshal(respBody, resp) } // logSampledTrace emits a single structured line per sampled span. Operators diff --git a/internal/benchmarking/boomer/glutton/memfill_test.go b/internal/benchmarking/boomer/glutton/memfill_test.go index 26c0cf5667..d979bfa48d 100644 --- a/internal/benchmarking/boomer/glutton/memfill_test.go +++ b/internal/benchmarking/boomer/glutton/memfill_test.go @@ -90,8 +90,8 @@ func TestChurnRAMOverwritesEachCycle(t *testing.T) { wantSizes := []string{"1Gi", "64Mi", "64Mi"} wantModes := []gluttonpb.WriteMode{ gluttonpb.WriteMode_WRITE_MODE_TRUNCATE, - gluttonpb.WriteMode_WRITE_MODE_OVERWRITE, - gluttonpb.WriteMode_WRITE_MODE_OVERWRITE, + gluttonpb.WriteMode_WRITE_MODE_OVERWRITE_ROTATE, + gluttonpb.WriteMode_WRITE_MODE_OVERWRITE_ROTATE, } if len(sizes) != len(wantSizes) { t.Fatalf("WriteRAM calls = %d (%v), want %d", len(sizes), sizes, len(wantSizes)) @@ -115,6 +115,60 @@ func TestChurnRAMDisabledByDefault(t *testing.T) { } } +func TestReadRAMWalksAfterFill(t *testing.T) { + srv := &fake.Server{} + u := newTestGluttonUser(t, srv, dynconfig.Config{MemTarget: "1Gi", MemRead: "all"}) + ctx := context.Background() + + // Read before fill is a no-op: there is nothing to walk yet. + u.readRAM(ctx) + if got := len(srv.RecordedRAMReadSizes()); got != 0 { + t.Fatalf("ReadRAM calls before fill = %d, want 0", got) + } + + u.ensureRAMFilled(ctx) + u.readRAM(ctx) + u.readRAM(ctx) + + sizes := srv.RecordedRAMReadSizes() + // "all" maps to an empty size: ReadRAM walks the whole array. + want := []string{"", ""} + if len(sizes) != len(want) { + t.Fatalf("ReadRAM calls = %d (%v), want %d", len(sizes), sizes, len(want)) + } + for i := range want { + if sizes[i] != want[i] { + t.Errorf("call %d size = %q, want %q", i, sizes[i], want[i]) + } + } +} + +func TestReadRAMPassesSizeVerbatim(t *testing.T) { + srv := &fake.Server{} + u := newTestGluttonUser(t, srv, dynconfig.Config{MemTarget: "1Gi", MemRead: "512Mi"}) + ctx := context.Background() + + u.ensureRAMFilled(ctx) + u.readRAM(ctx) + + sizes := srv.RecordedRAMReadSizes() + if len(sizes) != 1 || sizes[0] != "512Mi" { + t.Fatalf("ReadRAM sizes = %v, want [512Mi] (passed through verbatim)", sizes) + } +} + +func TestReadRAMDisabledByDefault(t *testing.T) { + srv := &fake.Server{} + u := newTestGluttonUser(t, srv, dynconfig.Config{MemTarget: "1Gi"}) + ctx := context.Background() + + u.ensureRAMFilled(ctx) + u.readRAM(ctx) + if got := len(srv.RecordedRAMReadSizes()); got != 0 { + t.Errorf("ReadRAM calls with mem_read unset = %d, want 0", got) + } +} + func TestEnsureRAMFilledRetriesAfterFailure(t *testing.T) { srv := &fake.Server{Status: 503} u := newTestGluttonUser(t, srv, dynconfig.Config{MemTarget: "1Mi"}) diff --git a/internal/benchmarking/glutton/fake/server.go b/internal/benchmarking/glutton/fake/server.go index c1c473f560..d72f2b1a67 100644 --- a/internal/benchmarking/glutton/fake/server.go +++ b/internal/benchmarking/glutton/fake/server.go @@ -35,6 +35,7 @@ const ( WriteDiskRoute = "/writedisk" ReadDiskRoute = "/readdisk" WriteRAMRoute = "/writeram" + ReadRAMRoute = "/readram" ) // Server is an httptest-backed stand-in for a glutton actor holding one file. @@ -62,6 +63,7 @@ type Server struct { readModes []gluttonpb.ReadMode ramWriteSizes []string ramWriteModes []gluttonpb.WriteMode + ramReadSizes []string } func (s *Server) reportedDigest() []byte { @@ -118,6 +120,13 @@ func (s *Server) RecordedRAMWriteModes() []gluttonpb.WriteMode { return append([]gluttonpb.WriteMode(nil), s.ramWriteModes...) } +// RecordedRAMReadSizes returns each /readram request's size string. +func (s *Server) RecordedRAMReadSizes() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.ramReadSizes...) +} + func (s *Server) Start(t *testing.T) *httptest.Server { t.Helper() ts := httptest.NewServer(http.HandlerFunc(s.serve)) @@ -203,6 +212,24 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request) { resp, _ := proto.Marshal(&gluttonpb.WriteRAMResponse{}) _, _ = w.Write(resp) + case ReadRAMRoute: + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + var req gluttonpb.ReadRAMRequest + if err := proto.Unmarshal(body, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + s.mu.Lock() + s.ramReadSizes = append(s.ramReadSizes, req.GetSize()) + s.mu.Unlock() + + resp, _ := proto.Marshal(&gluttonpb.ReadRAMResponse{Size: int64(len(s.Data))}) + _, _ = w.Write(resp) + default: http.NotFound(w, r) } diff --git a/internal/proto/glutton/glutton.pb.go b/internal/proto/glutton/glutton.pb.go index bb5139fa25..f9e02c7f5e 100644 --- a/internal/proto/glutton/glutton.pb.go +++ b/internal/proto/glutton/glutton.pb.go @@ -42,6 +42,10 @@ const ( WriteMode_WRITE_MODE_TRUNCATE WriteMode = 0 // Write on top of the existing data WriteMode_WRITE_MODE_OVERWRITE WriteMode = 1 + // Write on top of the existing data at a per-key cursor that advances + // past each write and wraps at the end, so repeated writes walk the + // whole array instead of re-dirtying the same prefix. RAM only. + WriteMode_WRITE_MODE_OVERWRITE_ROTATE WriteMode = 2 ) // Enum value maps for WriteMode. @@ -49,10 +53,12 @@ var ( WriteMode_name = map[int32]string{ 0: "WRITE_MODE_TRUNCATE", 1: "WRITE_MODE_OVERWRITE", + 2: "WRITE_MODE_OVERWRITE_ROTATE", } WriteMode_value = map[string]int32{ - "WRITE_MODE_TRUNCATE": 0, - "WRITE_MODE_OVERWRITE": 1, + "WRITE_MODE_TRUNCATE": 0, + "WRITE_MODE_OVERWRITE": 1, + "WRITE_MODE_OVERWRITE_ROTATE": 2, } ) @@ -231,6 +237,115 @@ func (*WriteRAMResponse) Descriptor() ([]byte, []int) { return file_glutton_proto_rawDescGZIP(), []int{1} } +type ReadRAMRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // name of the array to be read + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // size of bytes to walk, with an optional unit suffix (e.g. "2Gi"); + // empty walks the whole array + Size string `protobuf:"bytes,2,opt,name=size,proto3" json:"size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadRAMRequest) Reset() { + *x = ReadRAMRequest{} + mi := &file_glutton_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadRAMRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadRAMRequest) ProtoMessage() {} + +func (x *ReadRAMRequest) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadRAMRequest.ProtoReflect.Descriptor instead. +func (*ReadRAMRequest) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{2} +} + +func (x *ReadRAMRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ReadRAMRequest) GetSize() string { + if x != nil { + return x.Size + } + return "" +} + +type ReadRAMResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // total size of bytes walked + Size int64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` + // XOR of the sampled bytes, so the reads are observable + Checksum uint32 `protobuf:"varint,2,opt,name=checksum,proto3" json:"checksum,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadRAMResponse) Reset() { + *x = ReadRAMResponse{} + mi := &file_glutton_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadRAMResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadRAMResponse) ProtoMessage() {} + +func (x *ReadRAMResponse) ProtoReflect() protoreflect.Message { + mi := &file_glutton_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadRAMResponse.ProtoReflect.Descriptor instead. +func (*ReadRAMResponse) Descriptor() ([]byte, []int) { + return file_glutton_proto_rawDescGZIP(), []int{3} +} + +func (x *ReadRAMResponse) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ReadRAMResponse) GetChecksum() uint32 { + if x != nil { + return x.Checksum + } + return 0 +} + type WriteDiskRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // name of the file to be written to @@ -244,7 +359,7 @@ type WriteDiskRequest struct { func (x *WriteDiskRequest) Reset() { *x = WriteDiskRequest{} - mi := &file_glutton_proto_msgTypes[2] + mi := &file_glutton_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -256,7 +371,7 @@ func (x *WriteDiskRequest) String() string { func (*WriteDiskRequest) ProtoMessage() {} func (x *WriteDiskRequest) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[2] + mi := &file_glutton_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -269,7 +384,7 @@ func (x *WriteDiskRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WriteDiskRequest.ProtoReflect.Descriptor instead. func (*WriteDiskRequest) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{2} + return file_glutton_proto_rawDescGZIP(), []int{4} } func (x *WriteDiskRequest) GetKey() string { @@ -305,7 +420,7 @@ type WriteDiskResponse struct { func (x *WriteDiskResponse) Reset() { *x = WriteDiskResponse{} - mi := &file_glutton_proto_msgTypes[3] + mi := &file_glutton_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -317,7 +432,7 @@ func (x *WriteDiskResponse) String() string { func (*WriteDiskResponse) ProtoMessage() {} func (x *WriteDiskResponse) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[3] + mi := &file_glutton_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -330,7 +445,7 @@ func (x *WriteDiskResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WriteDiskResponse.ProtoReflect.Descriptor instead. func (*WriteDiskResponse) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{3} + return file_glutton_proto_rawDescGZIP(), []int{5} } func (x *WriteDiskResponse) GetSize() int64 { @@ -358,7 +473,7 @@ type ReadDiskRequest struct { func (x *ReadDiskRequest) Reset() { *x = ReadDiskRequest{} - mi := &file_glutton_proto_msgTypes[4] + mi := &file_glutton_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -370,7 +485,7 @@ func (x *ReadDiskRequest) String() string { func (*ReadDiskRequest) ProtoMessage() {} func (x *ReadDiskRequest) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[4] + mi := &file_glutton_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -383,7 +498,7 @@ func (x *ReadDiskRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadDiskRequest.ProtoReflect.Descriptor instead. func (*ReadDiskRequest) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{4} + return file_glutton_proto_rawDescGZIP(), []int{6} } func (x *ReadDiskRequest) GetKey() string { @@ -414,7 +529,7 @@ type ReadDiskResponse struct { func (x *ReadDiskResponse) Reset() { *x = ReadDiskResponse{} - mi := &file_glutton_proto_msgTypes[5] + mi := &file_glutton_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -426,7 +541,7 @@ func (x *ReadDiskResponse) String() string { func (*ReadDiskResponse) ProtoMessage() {} func (x *ReadDiskResponse) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[5] + mi := &file_glutton_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -439,7 +554,7 @@ func (x *ReadDiskResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadDiskResponse.ProtoReflect.Descriptor instead. func (*ReadDiskResponse) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{5} + return file_glutton_proto_rawDescGZIP(), []int{7} } func (x *ReadDiskResponse) GetSize() int64 { @@ -473,7 +588,7 @@ type OpenFDRequest struct { func (x *OpenFDRequest) Reset() { *x = OpenFDRequest{} - mi := &file_glutton_proto_msgTypes[6] + mi := &file_glutton_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +600,7 @@ func (x *OpenFDRequest) String() string { func (*OpenFDRequest) ProtoMessage() {} func (x *OpenFDRequest) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[6] + mi := &file_glutton_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +613,7 @@ func (x *OpenFDRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use OpenFDRequest.ProtoReflect.Descriptor instead. func (*OpenFDRequest) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{6} + return file_glutton_proto_rawDescGZIP(), []int{8} } func (x *OpenFDRequest) GetCount() int32 { @@ -516,7 +631,7 @@ type OpenFDResponse struct { func (x *OpenFDResponse) Reset() { *x = OpenFDResponse{} - mi := &file_glutton_proto_msgTypes[7] + mi := &file_glutton_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -528,7 +643,7 @@ func (x *OpenFDResponse) String() string { func (*OpenFDResponse) ProtoMessage() {} func (x *OpenFDResponse) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[7] + mi := &file_glutton_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -541,7 +656,7 @@ func (x *OpenFDResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use OpenFDResponse.ProtoReflect.Descriptor instead. func (*OpenFDResponse) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{7} + return file_glutton_proto_rawDescGZIP(), []int{9} } type PingRequest struct { @@ -554,7 +669,7 @@ type PingRequest struct { func (x *PingRequest) Reset() { *x = PingRequest{} - mi := &file_glutton_proto_msgTypes[8] + mi := &file_glutton_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -566,7 +681,7 @@ func (x *PingRequest) String() string { func (*PingRequest) ProtoMessage() {} func (x *PingRequest) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[8] + mi := &file_glutton_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -579,7 +694,7 @@ func (x *PingRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PingRequest.ProtoReflect.Descriptor instead. func (*PingRequest) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{8} + return file_glutton_proto_rawDescGZIP(), []int{10} } func (x *PingRequest) GetMessage() string { @@ -599,7 +714,7 @@ type PingResponse struct { func (x *PingResponse) Reset() { *x = PingResponse{} - mi := &file_glutton_proto_msgTypes[9] + mi := &file_glutton_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -611,7 +726,7 @@ func (x *PingResponse) String() string { func (*PingResponse) ProtoMessage() {} func (x *PingResponse) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[9] + mi := &file_glutton_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -624,7 +739,7 @@ func (x *PingResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PingResponse.ProtoReflect.Descriptor instead. func (*PingResponse) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{9} + return file_glutton_proto_rawDescGZIP(), []int{11} } func (x *PingResponse) GetMessage() string { @@ -643,7 +758,7 @@ type GossipRequest struct { func (x *GossipRequest) Reset() { *x = GossipRequest{} - mi := &file_glutton_proto_msgTypes[10] + mi := &file_glutton_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -655,7 +770,7 @@ func (x *GossipRequest) String() string { func (*GossipRequest) ProtoMessage() {} func (x *GossipRequest) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[10] + mi := &file_glutton_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -668,7 +783,7 @@ func (x *GossipRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GossipRequest.ProtoReflect.Descriptor instead. func (*GossipRequest) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{10} + return file_glutton_proto_rawDescGZIP(), []int{12} } func (x *GossipRequest) GetPeers() []*Peer { @@ -686,7 +801,7 @@ type GossipResponse struct { func (x *GossipResponse) Reset() { *x = GossipResponse{} - mi := &file_glutton_proto_msgTypes[11] + mi := &file_glutton_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -698,7 +813,7 @@ func (x *GossipResponse) String() string { func (*GossipResponse) ProtoMessage() {} func (x *GossipResponse) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[11] + mi := &file_glutton_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -711,7 +826,7 @@ func (x *GossipResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GossipResponse.ProtoReflect.Descriptor instead. func (*GossipResponse) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{11} + return file_glutton_proto_rawDescGZIP(), []int{13} } type Peer struct { @@ -724,7 +839,7 @@ type Peer struct { func (x *Peer) Reset() { *x = Peer{} - mi := &file_glutton_proto_msgTypes[12] + mi := &file_glutton_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -736,7 +851,7 @@ func (x *Peer) String() string { func (*Peer) ProtoMessage() {} func (x *Peer) ProtoReflect() protoreflect.Message { - mi := &file_glutton_proto_msgTypes[12] + mi := &file_glutton_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -749,7 +864,7 @@ func (x *Peer) ProtoReflect() protoreflect.Message { // Deprecated: Use Peer.ProtoReflect.Descriptor instead. func (*Peer) Descriptor() ([]byte, []int) { - return file_glutton_proto_rawDescGZIP(), []int{12} + return file_glutton_proto_rawDescGZIP(), []int{14} } func (x *Peer) GetHost() string { @@ -776,7 +891,13 @@ const file_glutton_proto_rawDesc = "" + "\x04size\x18\x02 \x01(\tR\x04size\x121\n" + "\n" + "write_mode\x18\x03 \x01(\x0e2\x12.glutton.WriteModeR\twriteMode\"\x12\n" + - "\x10WriteRAMResponse\"k\n" + + "\x10WriteRAMResponse\"6\n" + + "\x0eReadRAMRequest\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x12\n" + + "\x04size\x18\x02 \x01(\tR\x04size\"A\n" + + "\x0fReadRAMResponse\x12\x12\n" + + "\x04size\x18\x01 \x01(\x03R\x04size\x12\x1a\n" + + "\bchecksum\x18\x02 \x01(\rR\bchecksum\"k\n" + "\x10WriteDiskRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x12\n" + "\x04size\x18\x02 \x01(\x05R\x04size\x121\n" + @@ -804,15 +925,17 @@ const file_glutton_proto_rawDesc = "" + "\x0eGossipResponse\"5\n" + "\x04Peer\x12\x12\n" + "\x04host\x18\x01 \x01(\tR\x04host\x12\x19\n" + - "\bdelay_ms\x18\x02 \x01(\x05R\adelayMs*>\n" + + "\bdelay_ms\x18\x02 \x01(\x05R\adelayMs*_\n" + "\tWriteMode\x12\x17\n" + "\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n" + - "\x14WRITE_MODE_OVERWRITE\x10\x01*9\n" + + "\x14WRITE_MODE_OVERWRITE\x10\x01\x12\x1f\n" + + "\x1bWRITE_MODE_OVERWRITE_ROTATE\x10\x02*9\n" + "\bReadMode\x12\x12\n" + "\x0eREAD_MODE_DATA\x10\x00\x12\x19\n" + - "\x15READ_MODE_DIGEST_ONLY\x10\x012\x86\x03\n" + + "\x15READ_MODE_DIGEST_ONLY\x10\x012\xc6\x03\n" + "\aGlutton\x12A\n" + - "\bWriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12D\n" + + "\bWriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12>\n" + + "\aReadRAM\x12\x17.glutton.ReadRAMRequest\x1a\x18.glutton.ReadRAMResponse\"\x00\x12D\n" + "\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12A\n" + "\bReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n" + "\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x125\n" + @@ -832,43 +955,47 @@ func file_glutton_proto_rawDescGZIP() []byte { } var file_glutton_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_glutton_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_glutton_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_glutton_proto_goTypes = []any{ (WriteMode)(0), // 0: glutton.WriteMode (ReadMode)(0), // 1: glutton.ReadMode (*WriteRAMRequest)(nil), // 2: glutton.WriteRAMRequest (*WriteRAMResponse)(nil), // 3: glutton.WriteRAMResponse - (*WriteDiskRequest)(nil), // 4: glutton.WriteDiskRequest - (*WriteDiskResponse)(nil), // 5: glutton.WriteDiskResponse - (*ReadDiskRequest)(nil), // 6: glutton.ReadDiskRequest - (*ReadDiskResponse)(nil), // 7: glutton.ReadDiskResponse - (*OpenFDRequest)(nil), // 8: glutton.OpenFDRequest - (*OpenFDResponse)(nil), // 9: glutton.OpenFDResponse - (*PingRequest)(nil), // 10: glutton.PingRequest - (*PingResponse)(nil), // 11: glutton.PingResponse - (*GossipRequest)(nil), // 12: glutton.GossipRequest - (*GossipResponse)(nil), // 13: glutton.GossipResponse - (*Peer)(nil), // 14: glutton.Peer + (*ReadRAMRequest)(nil), // 4: glutton.ReadRAMRequest + (*ReadRAMResponse)(nil), // 5: glutton.ReadRAMResponse + (*WriteDiskRequest)(nil), // 6: glutton.WriteDiskRequest + (*WriteDiskResponse)(nil), // 7: glutton.WriteDiskResponse + (*ReadDiskRequest)(nil), // 8: glutton.ReadDiskRequest + (*ReadDiskResponse)(nil), // 9: glutton.ReadDiskResponse + (*OpenFDRequest)(nil), // 10: glutton.OpenFDRequest + (*OpenFDResponse)(nil), // 11: glutton.OpenFDResponse + (*PingRequest)(nil), // 12: glutton.PingRequest + (*PingResponse)(nil), // 13: glutton.PingResponse + (*GossipRequest)(nil), // 14: glutton.GossipRequest + (*GossipResponse)(nil), // 15: glutton.GossipResponse + (*Peer)(nil), // 16: glutton.Peer } var file_glutton_proto_depIdxs = []int32{ 0, // 0: glutton.WriteRAMRequest.write_mode:type_name -> glutton.WriteMode 0, // 1: glutton.WriteDiskRequest.write_mode:type_name -> glutton.WriteMode 1, // 2: glutton.ReadDiskRequest.read_mode:type_name -> glutton.ReadMode - 14, // 3: glutton.GossipRequest.peers:type_name -> glutton.Peer + 16, // 3: glutton.GossipRequest.peers:type_name -> glutton.Peer 2, // 4: glutton.Glutton.WriteRAM:input_type -> glutton.WriteRAMRequest - 4, // 5: glutton.Glutton.WriteDisk:input_type -> glutton.WriteDiskRequest - 6, // 6: glutton.Glutton.ReadDisk:input_type -> glutton.ReadDiskRequest - 8, // 7: glutton.Glutton.OpenFD:input_type -> glutton.OpenFDRequest - 10, // 8: glutton.Glutton.Ping:input_type -> glutton.PingRequest - 12, // 9: glutton.Glutton.Gossip:input_type -> glutton.GossipRequest - 3, // 10: glutton.Glutton.WriteRAM:output_type -> glutton.WriteRAMResponse - 5, // 11: glutton.Glutton.WriteDisk:output_type -> glutton.WriteDiskResponse - 7, // 12: glutton.Glutton.ReadDisk:output_type -> glutton.ReadDiskResponse - 9, // 13: glutton.Glutton.OpenFD:output_type -> glutton.OpenFDResponse - 11, // 14: glutton.Glutton.Ping:output_type -> glutton.PingResponse - 13, // 15: glutton.Glutton.Gossip:output_type -> glutton.GossipResponse - 10, // [10:16] is the sub-list for method output_type - 4, // [4:10] is the sub-list for method input_type + 4, // 5: glutton.Glutton.ReadRAM:input_type -> glutton.ReadRAMRequest + 6, // 6: glutton.Glutton.WriteDisk:input_type -> glutton.WriteDiskRequest + 8, // 7: glutton.Glutton.ReadDisk:input_type -> glutton.ReadDiskRequest + 10, // 8: glutton.Glutton.OpenFD:input_type -> glutton.OpenFDRequest + 12, // 9: glutton.Glutton.Ping:input_type -> glutton.PingRequest + 14, // 10: glutton.Glutton.Gossip:input_type -> glutton.GossipRequest + 3, // 11: glutton.Glutton.WriteRAM:output_type -> glutton.WriteRAMResponse + 5, // 12: glutton.Glutton.ReadRAM:output_type -> glutton.ReadRAMResponse + 7, // 13: glutton.Glutton.WriteDisk:output_type -> glutton.WriteDiskResponse + 9, // 14: glutton.Glutton.ReadDisk:output_type -> glutton.ReadDiskResponse + 11, // 15: glutton.Glutton.OpenFD:output_type -> glutton.OpenFDResponse + 13, // 16: glutton.Glutton.Ping:output_type -> glutton.PingResponse + 15, // 17: glutton.Glutton.Gossip:output_type -> glutton.GossipResponse + 11, // [11:18] is the sub-list for method output_type + 4, // [4:11] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name 4, // [4:4] is the sub-list for extension extendee 0, // [0:4] is the sub-list for field type_name @@ -885,7 +1012,7 @@ func file_glutton_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_glutton_proto_rawDesc), len(file_glutton_proto_rawDesc)), NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/glutton/glutton.proto b/internal/proto/glutton/glutton.proto index 7f2b005fc8..5e2048c385 100644 --- a/internal/proto/glutton/glutton.proto +++ b/internal/proto/glutton/glutton.proto @@ -26,6 +26,11 @@ service Glutton { // will be random bytes. rpc WriteRAM(WriteRAMRequest) returns (WriteRAMResponse) {} + // Walks RAM previously allocated by WriteRAM: reads one byte per 4KiB + // page across the requested size, so every touched page must be resident + // before the response returns. + rpc ReadRAM(ReadRAMRequest) returns (ReadRAMResponse) {} + // Tells glutton to write to disk using the specified mode. Data // written will be random bytes. rpc WriteDisk(WriteDiskRequest) returns (WriteDiskResponse) {} @@ -53,6 +58,11 @@ enum WriteMode { // Write on top of the existing data WRITE_MODE_OVERWRITE = 1; + + // Write on top of the existing data at a per-key cursor that advances + // past each write and wraps at the end, so repeated writes walk the + // whole array instead of re-dirtying the same prefix. RAM only. + WRITE_MODE_OVERWRITE_ROTATE = 2; } // ReadMode selects how much of the file ReadDisk sends back. @@ -78,6 +88,23 @@ message WriteRAMRequest { message WriteRAMResponse { } +message ReadRAMRequest { + // name of the array to be read + string key = 1; + + // size of bytes to walk, with an optional unit suffix (e.g. "2Gi"); + // empty walks the whole array + string size = 2; +} + +message ReadRAMResponse { + // total size of bytes walked + int64 size = 1; + + // XOR of the sampled bytes, so the reads are observable + uint32 checksum = 2; +} + message WriteDiskRequest { // name of the file to be written to string key = 1; diff --git a/internal/proto/glutton/glutton_grpc.pb.go b/internal/proto/glutton/glutton_grpc.pb.go index 6d50635704..703e4d2c52 100644 --- a/internal/proto/glutton/glutton_grpc.pb.go +++ b/internal/proto/glutton/glutton_grpc.pb.go @@ -34,6 +34,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( Glutton_WriteRAM_FullMethodName = "/glutton.Glutton/WriteRAM" + Glutton_ReadRAM_FullMethodName = "/glutton.Glutton/ReadRAM" Glutton_WriteDisk_FullMethodName = "/glutton.Glutton/WriteDisk" Glutton_ReadDisk_FullMethodName = "/glutton.Glutton/ReadDisk" Glutton_OpenFD_FullMethodName = "/glutton.Glutton/OpenFD" @@ -52,6 +53,10 @@ type GluttonClient interface { // RAM or allocating additional RAM per request instructions. Data written // will be random bytes. WriteRAM(ctx context.Context, in *WriteRAMRequest, opts ...grpc.CallOption) (*WriteRAMResponse, error) + // Walks RAM previously allocated by WriteRAM: reads one byte per 4KiB + // page across the requested size, so every touched page must be resident + // before the response returns. + ReadRAM(ctx context.Context, in *ReadRAMRequest, opts ...grpc.CallOption) (*ReadRAMResponse, error) // Tells glutton to write to disk using the specified mode. Data // written will be random bytes. WriteDisk(ctx context.Context, in *WriteDiskRequest, opts ...grpc.CallOption) (*WriteDiskResponse, error) @@ -87,6 +92,16 @@ func (c *gluttonClient) WriteRAM(ctx context.Context, in *WriteRAMRequest, opts return out, nil } +func (c *gluttonClient) ReadRAM(ctx context.Context, in *ReadRAMRequest, opts ...grpc.CallOption) (*ReadRAMResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReadRAMResponse) + err := c.cc.Invoke(ctx, Glutton_ReadRAM_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *gluttonClient) WriteDisk(ctx context.Context, in *WriteDiskRequest, opts ...grpc.CallOption) (*WriteDiskResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(WriteDiskResponse) @@ -148,6 +163,10 @@ type GluttonServer interface { // RAM or allocating additional RAM per request instructions. Data written // will be random bytes. WriteRAM(context.Context, *WriteRAMRequest) (*WriteRAMResponse, error) + // Walks RAM previously allocated by WriteRAM: reads one byte per 4KiB + // page across the requested size, so every touched page must be resident + // before the response returns. + ReadRAM(context.Context, *ReadRAMRequest) (*ReadRAMResponse, error) // Tells glutton to write to disk using the specified mode. Data // written will be random bytes. WriteDisk(context.Context, *WriteDiskRequest) (*WriteDiskResponse, error) @@ -176,6 +195,9 @@ type UnimplementedGluttonServer struct{} func (UnimplementedGluttonServer) WriteRAM(context.Context, *WriteRAMRequest) (*WriteRAMResponse, error) { return nil, status.Error(codes.Unimplemented, "method WriteRAM not implemented") } +func (UnimplementedGluttonServer) ReadRAM(context.Context, *ReadRAMRequest) (*ReadRAMResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ReadRAM not implemented") +} func (UnimplementedGluttonServer) WriteDisk(context.Context, *WriteDiskRequest) (*WriteDiskResponse, error) { return nil, status.Error(codes.Unimplemented, "method WriteDisk not implemented") } @@ -230,6 +252,24 @@ func _Glutton_WriteRAM_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _Glutton_ReadRAM_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReadRAMRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GluttonServer).ReadRAM(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Glutton_ReadRAM_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GluttonServer).ReadRAM(ctx, req.(*ReadRAMRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Glutton_WriteDisk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(WriteDiskRequest) if err := dec(in); err != nil { @@ -331,6 +371,10 @@ var Glutton_ServiceDesc = grpc.ServiceDesc{ MethodName: "WriteRAM", Handler: _Glutton_WriteRAM_Handler, }, + { + MethodName: "ReadRAM", + Handler: _Glutton_ReadRAM_Handler, + }, { MethodName: "WriteDisk", Handler: _Glutton_WriteDisk_Handler,