From 14ddcd6682228d89455e2cd5465e014b712059fa Mon Sep 17 00:00:00 2001 From: Tetsuro Aoki Date: Thu, 5 Mar 2020 14:59:41 +0900 Subject: [PATCH 1/2] Update go.mod --- go.mod | 11 ++++++++--- go.sum | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 690dc57..867ba8e 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,12 @@ module github.com/qedus/nds +go 1.11 + require ( - golang.org/x/net v0.0.0-20181107093936-a544f70c90f1 - golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect - google.golang.org/appengine v1.3.0 + github.com/golang/protobuf v1.3.4 // indirect + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a + golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect + golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb // indirect + google.golang.org/appengine v1.6.5 ) diff --git a/go.sum b/go.sum index 3316e9c..36d9bfa 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,41 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181107093936-a544f70c90f1 h1:SwqD3GJ9PsSBBVv1HlDeSwjjPSeZjUZQcAgGnwsWpyc= golang.org/x/net v0.0.0-20181107093936-a544f70c90f1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= From 2bf4eedddbbd11e942e46df6d6513d0744927de9 Mon Sep 17 00:00:00 2001 From: Tetsuro Aoki Date: Fri, 6 Mar 2020 09:24:11 +0900 Subject: [PATCH 2/2] Fix mishandling of memcache error --- put.go | 39 ++++++++++++++++++++++++++++----------- put_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/put.go b/put.go index ddaff04..e44be0e 100644 --- a/put.go +++ b/put.go @@ -121,17 +121,20 @@ func putMulti(c context.Context, lockMemcacheKeys := make([]string, 0, len(keys)) lockMemcacheItems := make([]*memcache.Item, 0, len(keys)) - for _, key := range keys { - if !key.Incomplete() { - item := &memcache.Item{ - Key: createMemcacheKey(key), - Flags: lockItem, - Value: itemLock(), - Expiration: memcacheLockTime, - } - lockMemcacheItems = append(lockMemcacheItems, item) - lockMemcacheKeys = append(lockMemcacheKeys, item.Key) + incompleteIndexes := make(map[int]struct{}, len(keys)) + for i, key := range keys { + if key.Incomplete() { + incompleteIndexes[i] = struct{}{} + continue + } + item := &memcache.Item{ + Key: createMemcacheKey(key), + Flags: lockItem, + Value: itemLock(), + Expiration: memcacheLockTime, } + lockMemcacheItems = append(lockMemcacheItems, item) + lockMemcacheKeys = append(lockMemcacheKeys, item.Key) } memcacheCtx, err := memcacheContext(c) @@ -156,7 +159,21 @@ func putMulti(c context.Context, tx.Unlock() } else if err := memcacheSetMulti(memcacheCtx, lockMemcacheItems); err != nil { - return nil, err + me, ok := err.(appengine.MultiError) + if !ok { + return nil, err + } + retErr := make(appengine.MultiError, len(keys)) + meIdx := 0 + for i := range keys { + if _, ok := incompleteIndexes[i]; ok { + retErr[i] = nil + continue + } + retErr[i] = me[meIdx] + meIdx++ + } + return keys, retErr } // Save to the datastore. diff --git a/put_test.go b/put_test.go index 9cdaac4..4f898a3 100644 --- a/put_test.go +++ b/put_test.go @@ -162,6 +162,56 @@ func TestPutMultiLockFailure(t *testing.T) { } } +func TestPutMultiLockFailureWithMultiError(t *testing.T) { + c, closeFunc := NewContext(t) + defer closeFunc() + + type testEntity struct { + IntVal int + } + + nds.SetMemcacheSetMulti(func(c context.Context, + items []*memcache.Item) error { + errs := appengine.MultiError{ + errors.New("unexpected error"), + nil, + errors.New("unexpected error"), + } + return errs + }) + + defer func() { + nds.SetMemcacheSetMulti(memcache.SetMulti) + }() + + keys := []*datastore.Key{ + datastore.NewKey(c, "Test", "", 1, nil), + datastore.NewKey(c, "Test", "", 0, nil), // incomplete key + datastore.NewKey(c, "Test", "", 2, nil), + datastore.NewKey(c, "Test", "", 3, nil), + datastore.NewKey(c, "Test", "", 0, nil), // incomplete key + } + vals := []testEntity{ + {42}, + {43}, + {44}, + {45}, + {46}, + } + + _, err := nds.PutMulti(c, keys, vals) + if err == nil { + t.Fatal("expected nds.PutMulti error") + } + me, ok := err.(appengine.MultiError) + if !ok { + t.Fatal("expected appengine.MultiError") + } + if len(me) != len(keys) { + t.Fatal("expected multi error length equals keys length") + } +} + // Make sure PutMulti still works if we have a memcache unlock failure. func TestPutMultiUnlockMemcacheSuccess(t *testing.T) { c, closeFunc := NewContext(t)