Add pagination, configurable via yaml, when showing results i.e from dynamo/S3 etc

cmd/loader.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/cmd-tools/aws-commander/constants"
+	"github.com/cmd-tools/aws-commander/helpers"
 	"log"
 	"os"
 	"path/filepath"
@@ -22,19 +24,24 @@ const VariablePlaceHolderPrefix = "$"
 var Resources = map[string]Resource{}
 
 type Command struct {
-	Name           string   `yaml:"name"`
-	ResourceName   string   `yaml:"resourceName"`
-	DefaultCommand string   `yaml:"defaultCommand"`
-	Arguments      []string `yaml:"arguments"`
-	View           string   `yaml:"view"`
-	Parse          Parse    `yaml:"parse"`
+	Name           string    `yaml:"name"`
+	ResourceName   string    `yaml:"resourceName"`
+	DefaultCommand string    `yaml:"defaultCommand"`
+	Arguments      []string  `yaml:"arguments"`
+	View           string    `yaml:"view"`
+	Parse          Parse     `yaml:"parse"`
+	Paginate       *Paginate `yaml:"paginate"`
 }
 
 type Parse struct {
 	Type          string `yaml:"type"`
 	AttributeName string `yaml:"attributeName"`
 }
 
+type Paginate struct {
+	Max *uint8 `yaml:"max"`
+}
+
 type Resource struct {
 	Name           string    `yaml:"name"`
 	DefaultCommand string    `yaml:"defaultCommand"`
@@ -91,12 +98,14 @@ func (resource *Resource) GetCommand(name string) Command {
 	panic(fmt.Sprintf("Requested command %s does not exists in %s resouce", name, resource.Name))
 }
 
-func (command *Command) Run(resource string, profile string) string {
+func (command *Command) Run(resource string, profile string, nextToken string) string {
 	binaryName := "aws"
 	var argumentsCopy = make([]string, len(command.Arguments))
 	copy(argumentsCopy, command.Arguments)
 	args := []string{resource, command.Name, "--profile", profile}
 	args = append(args, replaceVariablesOnCommandArguments(command.Arguments)...)
+	args = addPaginationArgs(args, nextToken, *command)
+
 	logger.Logger.Debug().Msg(fmt.Sprintf("Running: %s %s", binaryName, strings.Join(args, " ")))
 	start := time.Now()
 	output := executor.ExecCommand(binaryName, args)
@@ -136,3 +145,22 @@ func replaceVariablesOnCommandArguments(arguments []string) []string {
 	}
 	return arguments
 }
+
+func addPaginationArgs(args []string, nextToken string, command Command) []string {
+	// TODO: sanitize args by removing existing --max-items and the value if any
+	helpers.RemoveItem(args, "--no-pagination")
+	var defaultPaginationValue = constants.DefaultPaginationValue
+	if command.Paginate != nil && command.Paginate.Max != nil && *command.Paginate.Max != 0 {
+		defaultPaginationValue = int(*command.Paginate.Max)
+		logger.Logger.Debug().Msg(fmt.Sprintf("[Worker] Found pagination override to: %d", defaultPaginationValue))
+	}
+
+	logger.Logger.Debug().Msg(fmt.Sprintf("[Worker] Using pagination value: %d", defaultPaginationValue))
+	args = append(args, "--max-items", fmt.Sprintf("%d", defaultPaginationValue))
+
+	if !helpers.IsStringEmpty(nextToken) {
+		args = append(args, "--starting-token", nextToken)
+	}
+
+	return args
+}

cmd/uiState.go

@@ -7,6 +7,8 @@ type UIState struct {
 	SelectedItems     map[string]string `yaml:"selectedItems"`
 	CommandBarVisible bool              `yaml:"commandBarVisible"`
 	Breadcrumbs       []string          `yaml:"breadcrumbs"`
+	PageNumber        uint64            `yaml:"pageNumber"`
+	NextToken         string            `yaml:"nextToken"`
 }
 
 var UiState UIState = UIState{SelectedItems: make(map[string]string), Breadcrumbs: []string{}}

configurations/dynamodb.yaml

@@ -3,7 +3,6 @@ commands:
   - name: "list-tables"
     resourceName: tableName
     arguments:
-      - "--no-paginate"
       - "--output"
       - "json"
       - "--cli-read-timeout"
@@ -25,6 +24,8 @@ commands:
       - "--cli-connect-timeout"
       - "5"
     view: tableView
+    paginate:
+      max: 50
     parse:
       type: "object"
       attributeName: "Items"

constants/constants.go

@@ -1,7 +1,8 @@
 package constants
 
 const (
-	EmptyString = ""
+	EmptyString            = ""
+	DefaultPaginationValue = 100
 )
 
 var EmptyRune rune

go.mod

@@ -5,9 +5,9 @@ go 1.22.2
 require (
 	github.com/gdamore/tcell/v2 v2.7.4
 	github.com/iancoleman/orderedmap v0.3.0
-	github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803
+	github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f
 	github.com/rs/zerolog v1.32.0
-	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -18,7 +18,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/term v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 )

go.sum

@@ -20,6 +20,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803 h1:26AUJmGMCp5z8kiJcB3m5vGyj4Pni8aIjy6Ua6Ind7c=
 github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss=
+github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f h1:DAbaKhyPcZQp/TqlSdUd6Z445PkJb3bI0VccXg22oeg=
+github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -32,6 +34,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
 golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -53,18 +57,24 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
 golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

helpers/string_helper.go

@@ -15,3 +15,17 @@ func RemoveEmptyStrings(strings []string) []string {
 	}
 	return result
 }
+
+func RemoveItem(slice []string, item string) []string {
+	index := -1
+	for i, s := range slice {
+		if s == item {
+			index = i
+			break
+		}
+	}
+	if index != -1 {
+		return append(slice[:index], slice[index+1:]...)
+	}
+	return slice
+}

main.go

@@ -262,7 +262,7 @@ func createResources(resources []string) tview.Primitive {
 			} else {
 				cmd.UiState.Command = cmd.UiState.Resource.GetCommand(cmd.UiState.Resource.DefaultCommand)
 				cmd.UiState.Breadcrumbs = []string{constants.Profiles, cmd.UiState.Profile, cmd.UiState.Resource.Name, cmd.UiState.Command.Name}
-				var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile))
+				var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString))
 				Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler)
 			}
 
@@ -283,7 +283,7 @@ func createExecuteCommandView(selectedCommandName string) {
 	cmd.UiState.Command = cmd.UiState.Resource.GetCommand(selectedCommandName)
 	AutoCompletionWordList = append(cmd.UiState.Resource.GetCommandNames(), constants.Profiles)
 
-	var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile))
+	var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString))
 	Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler)
 
 	cmd.UiState.Breadcrumbs = append(cmd.UiState.Breadcrumbs, cmd.UiState.Command.Name)
@@ -312,7 +312,7 @@ func itemHandler(selectedItemName string) {
 	} else {
 		cmd.UiState.Command = cmd.UiState.Resource.GetCommand(cmd.UiState.Command.DefaultCommand)
 		cmd.UiState.Breadcrumbs = append(cmd.UiState.Breadcrumbs, cmd.UiState.Command.Name)
-		var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile))
+		var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString))
 		Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler)
 	}
 

parser/command.go

@@ -3,7 +3,6 @@ package parser
 import (
 	"encoding/json"
 	"fmt"
-
 	"github.com/cmd-tools/aws-commander/cmd"
 	"github.com/cmd-tools/aws-commander/logger"
 	"github.com/cmd-tools/aws-commander/ui"
@@ -12,9 +11,11 @@ import (
 )
 
 type ParseCommandResult struct {
-	Command string
-	Header  []string
-	Values  [][]string
+	Command   string
+	Header    []string
+	Values    [][]string
+	Count     uint64
+	NextToken string
 }
 
 func ParseCommand(command cmd.Command, commandOutput string) ParseCommandResult {
@@ -26,37 +27,51 @@ func ParseCommand(command cmd.Command, commandOutput string) ParseCommandResult
 
 	var parseCommandResult = ParseCommandResult{Command: command.Name}
 	baseAttribute, _ := jsonResult.Get(command.Parse.AttributeName)
+
+	if count, ok := jsonResult.Get("Count"); ok && count != nil {
+		parseCommandResult.Count = uint64(int(count.(float64)))
+	}
+
+	if nextToken, isPaginating := jsonResult.Get("NextToken"); isPaginating && nextToken != nil {
+		parseCommandResult.NextToken = nextToken.(string)
+	}
+
 	switch baseAttribute.(type) {
 	case []interface{}:
 		logger.Logger.Debug().Msg("Parse command list")
 		for i, s := range baseAttribute.([]interface{}) {
 			var values []string
-			if command.Parse.Type == "object" {
+			switch command.Parse.Type {
+			case "object":
 				item := s.(orderedmap.OrderedMap)
 				for _, key := range item.Keys() {
 					if i == 0 {
 						parseCommandResult.Header = append(parseCommandResult.Header, key)
 					}
 					value, exists := item.Get(key)
+
 					if exists {
 						switch value.(type) {
 						case string:
 							values = append(values, fmt.Sprintf("%v", value))
+							break
 						default:
 							bytes, _ := json.Marshal(value)
 							values = append(values, fmt.Sprintf("%v", string(bytes)))
 						}
 					}
 				}
 				parseCommandResult.Values = append(parseCommandResult.Values, values)
-			} else if command.Parse.Type == "list" {
+				break
+			case "list":
 				if i == 0 {
 					parseCommandResult.Header = append(parseCommandResult.Header, "Item")
 				}
 				if s != nil {
 					parseCommandResult.Values = append(parseCommandResult.Values, append(values, s.(string)))
 				}
-			} else {
+				break
+			default:
 				logger.Logger.Debug().Msg("Wrong type. Accepted types [Object, List]")
 			}
 		}