Merge branch 'master' of https://github.com/gluster/glusterd2 into rebalance

Author: rishubhjain

Gopkg.lock

@@ -89,3 +89,7 @@
 [[constraint]]
   name = "github.com/asaskevich/govalidator"
   version = "v8"
+
+[[constraint]]
+  name = "go.opencensus.io"
+  version = "0.11.0"

Gopkg.toml

@@ -7,5 +7,4 @@ peeraddress = "127.0.0.1:24008"
 clientaddress = "127.0.0.1:24007"
 etcdcurls = "http://127.0.0.1:2479"
 etcdpurls = "http://127.0.0.1:2480"
-#restauth should be set to false to disable REST authentication in glusterd2
-#restauth = false
+restauth = true

e2e/config/1.toml

@@ -7,5 +7,4 @@ peeraddress = "127.0.0.1:23008"
 clientaddress = "127.0.0.1:23007"
 etcdcurls = "http://127.0.0.1:2379"
 etcdpurls = "http://127.0.0.1:2380"
-#restauth should be set to false to disable REST authentication in glusterd2
-restauth = true
+restauth = true

e2e/config/2.toml

@@ -7,5 +7,4 @@ peeraddress = "127.0.0.1:22008"
 clientaddress = "127.0.0.1:22007"
 etcdcurls = "http://127.0.0.1:2279"
 etcdpurls = "http://127.0.0.1:2280"
-#restauth should be set to false to disable REST authentication in glusterd2
-restauth = true
+restauth = true

e2e/config/3.toml

@@ -7,5 +7,4 @@ peeraddress = "127.0.0.1:21008"
 clientaddress = "127.0.0.1:21007"
 etcdcurls = "http://127.0.0.1:2179"
 etcdpurls = "http://127.0.0.1:2180"
-#restauth should be set to false to disable REST authentication in glusterd2
 restauth = true

e2e/config/4.toml

@@ -6,7 +6,7 @@ import (
 	"syscall"
 	"testing"
 
-	smartvolapi "github.com/gluster/glusterd2/plugins/smartvol/api"
+	"github.com/gluster/glusterd2/pkg/api"
 
 	"github.com/stretchr/testify/require"
 )
@@ -32,12 +32,12 @@ func testSmartVolumeDistribute(t *testing.T) {
 	r := require.New(t)
 	smartvolname := formatVolName(t.Name())
 	// create Distribute 3 Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:            smartvolname,
 		Size:            60,
 		DistributeCount: 3,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 3)
@@ -57,12 +57,12 @@ func testSmartVolumeReplicate2(t *testing.T) {
 	r := require.New(t)
 	smartvolname := formatVolName(t.Name())
 	// create Replica 2 Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:         smartvolname,
 		Size:         20,
 		ReplicaCount: 2,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 1)
@@ -80,12 +80,12 @@ func testSmartVolumeReplicate3(t *testing.T) {
 
 	smartvolname := formatVolName(t.Name())
 	// create Replica 3 Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:         smartvolname,
 		Size:         20,
 		ReplicaCount: 3,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 1)
@@ -103,13 +103,13 @@ func testSmartVolumeArbiter(t *testing.T) {
 
 	smartvolname := formatVolName(t.Name())
 	// create Replica 3 Arbiter Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:         smartvolname,
 		Size:         20,
 		ReplicaCount: 3,
 		ArbiterCount: 1,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 1)
@@ -132,12 +132,12 @@ func testSmartVolumeDisperse(t *testing.T) {
 	smartvolname := formatVolName(t.Name())
 
 	// create Disperse Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:          smartvolname,
 		Size:          40,
 		DisperseCount: 3,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 1)
@@ -157,14 +157,14 @@ func testSmartVolumeDistributeReplicate(t *testing.T) {
 	smartvolname := formatVolName(t.Name())
 
 	// create Distribute Replicate(2x3) Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:               smartvolname,
 		Size:               40,
 		DistributeCount:    2,
 		ReplicaCount:       3,
 		SubvolZonesOverlap: true,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 2)
@@ -188,14 +188,14 @@ func testSmartVolumeDistributeDisperse(t *testing.T) {
 	smartvolname := formatVolName(t.Name())
 
 	// create Distribute Disperse(2x3) Volume
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Name:               smartvolname,
 		Size:               80,
 		DistributeCount:    2,
 		DisperseCount:      3,
 		SubvolZonesOverlap: true,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Len(volinfo.Subvols, 2)
@@ -216,10 +216,10 @@ func testSmartVolumeDistributeDisperse(t *testing.T) {
 func testSmartVolumeWithoutName(t *testing.T) {
 	r := require.New(t)
 
-	createReq := smartvolapi.VolCreateReq{
+	createReq := api.VolCreateReq{
 		Size: 20,
 	}
-	volinfo, err := client.SmartVolumeCreate(createReq)
+	volinfo, err := client.VolumeCreate(createReq)
 	r.Nil(err)
 
 	r.Nil(client.VolumeDelete(volinfo.Name))

e2e/smartvol_ops_test.go

@@ -82,7 +82,7 @@ func TestSnapshot(t *testing.T) {
 
 	/*
 		TODO:
-		test snapshot on a volume that is expanded or shrinked
+		test snapshot on a volume that is expanded or shrunk
 	*/
 
 }

e2e/snapshot_ops_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/gluster/glusterd2/pkg/restclient"
 )
@@ -20,6 +21,7 @@ var (
 
 func initRESTClient(hostname, user, secret, cacert string, insecure bool) {
 	client = restclient.New(hostname, user, secret, cacert, insecure)
+	client.SetTimeout(time.Duration(flagTimeout) * time.Second)
 }
 
 func isConnectionRefusedErr(err error) bool {

glustercli/cmd/common.go

@@ -11,6 +11,21 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var (
+	// variables set by LDFLAGS during build time
+	defaultAuthPath = ""
+)
+
+var (
+	// global variables set during runtime
+	secret string
+)
+
+const (
+	defaultLogLevel = "INFO"
+	defaultTimeout  = 30 // in seconds
+)
+
 // RootCmd represents main command
 var RootCmd = &cobra.Command{
 	Use:   "glustercli",
@@ -21,53 +36,64 @@ var RootCmd = &cobra.Command{
 			fmt.Println("Error initializing log file ", err)
 		}
 
-		if flagAuthFile == "" && flagSecret == "" {
-			data, err := ioutil.ReadFile(defaultAuthPath)
-			if err != nil && !os.IsNotExist(err) {
-				if verbose {
-					log.WithError(err).Error("failed to read secret")
-				}
+		// Secret is taken in following order of precedence (highest to lowest):
+		// --secret
+		// --secret-file
+		// GLUSTERD2_AUTH_SECRET (environment variable)
+		// --secret-file (default path)
+
+		// NOTE: For simplicity, we don't distinguish between an empty
+		// value and an unset value.
+
+		// --secret
+		if flagSecret != "" {
+			secret = flagSecret
+		}
+
+		// --secret-file
+		if flagSecretFile != "" && secret == "" {
+			data, err := ioutil.ReadFile(flagSecretFile)
+			if err != nil {
+				failure(fmt.Sprintf("failed to read secret file %s", flagSecretFile),
+					err, 1)
 			}
 			secret = string(data)
 		}
 
-		if flagAuthFile != "" {
-			data, err := ioutil.ReadFile(flagAuthFile)
-			if err != nil {
+		// GLUSTERD2_AUTH_SECRET
+		if secret == "" {
+			secret = os.Getenv("GLUSTERD2_AUTH_SECRET")
+		}
+
+		// --secret-file (default path)
+		if flagSecretFile == "" && secret == "" {
+			data, err := ioutil.ReadFile(defaultAuthPath)
+			if err != nil && !os.IsNotExist(err) {
 				if verbose {
-					log.WithError(err).Error("failed to read secret")
+					log.WithError(err).Error(
+						fmt.Sprintf("failed to read default secret file %s", defaultAuthPath))
 				}
-				failure("failed to read secret", err, 1)
 			}
 			secret = string(data)
 		}
 
-		if flagSecret != "" {
-			secret = flagSecret
-		}
-
 		initRESTClient(flagEndpoints[0], flagUser, secret, flagCacert, flagInsecure)
 	},
 }
 
 var (
+	// set by command line flags
 	flagXMLOutput  bool
 	flagJSONOutput bool
-	flagEndpoints  []string
-	flagCacert     string
 	flagInsecure   bool
-	flagLogLevel   string
 	verbose        bool
+	flagCacert     string
+	flagLogLevel   string
 	flagUser       string
 	flagSecret     string
-	flagAuthFile   string
-	secret         string
-	//defaultAuthPath is set by LDFLAGS
-	defaultAuthPath = ""
-)
-
-const (
-	defaultLogLevel = "INFO"
+	flagSecretFile string
+	flagEndpoints  []string
+	flagTimeout    uint
 )
 
 func init() {
@@ -76,11 +102,13 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&flagJSONOutput, "json", "", false, "JSON Output")
 	RootCmd.PersistentFlags().StringSliceVar(&flagEndpoints, "endpoints", []string{"http://127.0.0.1:24007"}, "glusterd2 endpoints")
 	RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose output")
+	RootCmd.PersistentFlags().UintVar(&flagTimeout, "timeout", defaultTimeout,
+		"overall client timeout (in seconds) which includes time taken to read the response body")
 
 	//user and secret for token authentication
 	RootCmd.PersistentFlags().StringVar(&flagUser, "user", "glustercli", "Username for authentication")
 	RootCmd.PersistentFlags().StringVar(&flagSecret, "secret", "", "Password for authentication")
-	RootCmd.PersistentFlags().StringVar(&flagAuthFile, "authfile", "", "Auth file path, which contains secret for authentication")
+	RootCmd.PersistentFlags().StringVar(&flagSecretFile, "secret-file", "", "Path to file which contains the secret for authentication")
 
 	// Log options
 	RootCmd.PersistentFlags().StringVarP(&flagLogLevel, logging.LevelFlag, "", defaultLogLevel, logging.LevelHelp)