From 272f3f49c257f87da4ec61ef9e5610501b864625 Mon Sep 17 00:00:00 2001 From: MrJack <36191829+biagiopietro@users.noreply.github.com> Date: Sun, 12 May 2024 14:01:23 +0200 Subject: [PATCH] Add pagination, configurable via yaml, when showing results i.e from dynamo/S3 etc --- cmd/loader.go | 42 ++++++++++++++++++++++++++++++------ cmd/uiState.go | 2 ++ configurations/dynamodb.yaml | 3 ++- constants/constants.go | 3 ++- go.mod | 10 ++++----- go.sum | 10 +++++++++ helpers/string_helper.go | 14 ++++++++++++ main.go | 6 +++--- parser/command.go | 29 +++++++++++++++++++------ 9 files changed, 95 insertions(+), 24 deletions(-) diff --git a/cmd/loader.go b/cmd/loader.go index 9821e4b..e9ae94d 100644 --- a/cmd/loader.go +++ b/cmd/loader.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + "github.com/cmd-tools/aws-commander/constants" + "github.com/cmd-tools/aws-commander/helpers" "log" "os" "path/filepath" @@ -22,12 +24,13 @@ const VariablePlaceHolderPrefix = "$" var Resources = map[string]Resource{} type Command struct { - Name string `yaml:"name"` - ResourceName string `yaml:"resourceName"` - DefaultCommand string `yaml:"defaultCommand"` - Arguments []string `yaml:"arguments"` - View string `yaml:"view"` - Parse Parse `yaml:"parse"` + Name string `yaml:"name"` + ResourceName string `yaml:"resourceName"` + DefaultCommand string `yaml:"defaultCommand"` + Arguments []string `yaml:"arguments"` + View string `yaml:"view"` + Parse Parse `yaml:"parse"` + Paginate *Paginate `yaml:"paginate"` } type Parse struct { @@ -35,6 +38,10 @@ type Parse struct { AttributeName string `yaml:"attributeName"` } +type Paginate struct { + Max *uint8 `yaml:"max"` +} + type Resource struct { Name string `yaml:"name"` DefaultCommand string `yaml:"defaultCommand"` @@ -91,12 +98,14 @@ func (resource *Resource) GetCommand(name string) Command { panic(fmt.Sprintf("Requested command %s does not exists in %s resouce", name, resource.Name)) } -func (command *Command) Run(resource string, profile string) string { +func (command *Command) Run(resource string, profile string, nextToken string) string { binaryName := "aws" var argumentsCopy = make([]string, len(command.Arguments)) copy(argumentsCopy, command.Arguments) args := []string{resource, command.Name, "--profile", profile} args = append(args, replaceVariablesOnCommandArguments(command.Arguments)...) + args = addPaginationArgs(args, nextToken, *command) + logger.Logger.Debug().Msg(fmt.Sprintf("Running: %s %s", binaryName, strings.Join(args, " "))) start := time.Now() output := executor.ExecCommand(binaryName, args) @@ -136,3 +145,22 @@ func replaceVariablesOnCommandArguments(arguments []string) []string { } return arguments } + +func addPaginationArgs(args []string, nextToken string, command Command) []string { + // TODO: sanitize args by removing existing --max-items and the value if any + helpers.RemoveItem(args, "--no-pagination") + var defaultPaginationValue = constants.DefaultPaginationValue + if command.Paginate != nil && command.Paginate.Max != nil && *command.Paginate.Max != 0 { + defaultPaginationValue = int(*command.Paginate.Max) + logger.Logger.Debug().Msg(fmt.Sprintf("[Worker] Found pagination override to: %d", defaultPaginationValue)) + } + + logger.Logger.Debug().Msg(fmt.Sprintf("[Worker] Using pagination value: %d", defaultPaginationValue)) + args = append(args, "--max-items", fmt.Sprintf("%d", defaultPaginationValue)) + + if !helpers.IsStringEmpty(nextToken) { + args = append(args, "--starting-token", nextToken) + } + + return args +} diff --git a/cmd/uiState.go b/cmd/uiState.go index 8402209..ee9131f 100644 --- a/cmd/uiState.go +++ b/cmd/uiState.go @@ -7,6 +7,8 @@ type UIState struct { SelectedItems map[string]string `yaml:"selectedItems"` CommandBarVisible bool `yaml:"commandBarVisible"` Breadcrumbs []string `yaml:"breadcrumbs"` + PageNumber uint64 `yaml:"pageNumber"` + NextToken string `yaml:"nextToken"` } var UiState UIState = UIState{SelectedItems: make(map[string]string), Breadcrumbs: []string{}} diff --git a/configurations/dynamodb.yaml b/configurations/dynamodb.yaml index a2d975d..2c32e92 100644 --- a/configurations/dynamodb.yaml +++ b/configurations/dynamodb.yaml @@ -3,7 +3,6 @@ commands: - name: "list-tables" resourceName: tableName arguments: - - "--no-paginate" - "--output" - "json" - "--cli-read-timeout" @@ -25,6 +24,8 @@ commands: - "--cli-connect-timeout" - "5" view: tableView + paginate: + max: 50 parse: type: "object" attributeName: "Items" \ No newline at end of file diff --git a/constants/constants.go b/constants/constants.go index e11f4ce..1e67b72 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -1,7 +1,8 @@ package constants const ( - EmptyString = "" + EmptyString = "" + DefaultPaginationValue = 100 ) var EmptyRune rune diff --git a/go.mod b/go.mod index 0deb7f0..13e7f2d 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.22.2 require ( github.com/gdamore/tcell/v2 v2.7.4 github.com/iancoleman/orderedmap v0.3.0 - github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803 + github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f github.com/rs/zerolog v1.32.0 - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 gopkg.in/yaml.v2 v2.4.0 ) @@ -18,7 +18,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/rivo/uniseg v0.4.7 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index dfd30cd..e9ebf15 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803 h1:26AUJmGMCp5z8kiJcB3m5vGyj4Pni8aIjy6Ua6Ind7c= github.com/rivo/tview v0.0.0-20240426173458-c766eefb3803/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= +github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f h1:DAbaKhyPcZQp/TqlSdUd6Z445PkJb3bI0VccXg22oeg= +github.com/rivo/tview v0.0.0-20240505185119-ed116790de0f/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -32,6 +34,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -53,18 +57,24 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/helpers/string_helper.go b/helpers/string_helper.go index 4e560ee..268c053 100644 --- a/helpers/string_helper.go +++ b/helpers/string_helper.go @@ -15,3 +15,17 @@ func RemoveEmptyStrings(strings []string) []string { } return result } + +func RemoveItem(slice []string, item string) []string { + index := -1 + for i, s := range slice { + if s == item { + index = i + break + } + } + if index != -1 { + return append(slice[:index], slice[index+1:]...) + } + return slice +} diff --git a/main.go b/main.go index 0c47f01..b37ebb9 100644 --- a/main.go +++ b/main.go @@ -262,7 +262,7 @@ func createResources(resources []string) tview.Primitive { } else { cmd.UiState.Command = cmd.UiState.Resource.GetCommand(cmd.UiState.Resource.DefaultCommand) cmd.UiState.Breadcrumbs = []string{constants.Profiles, cmd.UiState.Profile, cmd.UiState.Resource.Name, cmd.UiState.Command.Name} - var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile)) + var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString)) Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler) } @@ -283,7 +283,7 @@ func createExecuteCommandView(selectedCommandName string) { cmd.UiState.Command = cmd.UiState.Resource.GetCommand(selectedCommandName) AutoCompletionWordList = append(cmd.UiState.Resource.GetCommandNames(), constants.Profiles) - var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile)) + var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString)) Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler) cmd.UiState.Breadcrumbs = append(cmd.UiState.Breadcrumbs, cmd.UiState.Command.Name) @@ -312,7 +312,7 @@ func itemHandler(selectedItemName string) { } else { cmd.UiState.Command = cmd.UiState.Resource.GetCommand(cmd.UiState.Command.DefaultCommand) cmd.UiState.Breadcrumbs = append(cmd.UiState.Breadcrumbs, cmd.UiState.Command.Name) - var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile)) + var commandParsed = commandParser.ParseCommand(cmd.UiState.Command, cmd.UiState.Command.Run(cmd.UiState.Resource.Name, cmd.UiState.Profile, constants.EmptyString)) Body = commandParser.ParseToObject(cmd.UiState.Command.View, commandParsed, itemHandler) } diff --git a/parser/command.go b/parser/command.go index 40f44d4..73cb4cc 100644 --- a/parser/command.go +++ b/parser/command.go @@ -3,7 +3,6 @@ package parser import ( "encoding/json" "fmt" - "github.com/cmd-tools/aws-commander/cmd" "github.com/cmd-tools/aws-commander/logger" "github.com/cmd-tools/aws-commander/ui" @@ -12,9 +11,11 @@ import ( ) type ParseCommandResult struct { - Command string - Header []string - Values [][]string + Command string + Header []string + Values [][]string + Count uint64 + NextToken string } func ParseCommand(command cmd.Command, commandOutput string) ParseCommandResult { @@ -26,22 +27,34 @@ func ParseCommand(command cmd.Command, commandOutput string) ParseCommandResult var parseCommandResult = ParseCommandResult{Command: command.Name} baseAttribute, _ := jsonResult.Get(command.Parse.AttributeName) + + if count, ok := jsonResult.Get("Count"); ok && count != nil { + parseCommandResult.Count = uint64(int(count.(float64))) + } + + if nextToken, isPaginating := jsonResult.Get("NextToken"); isPaginating && nextToken != nil { + parseCommandResult.NextToken = nextToken.(string) + } + switch baseAttribute.(type) { case []interface{}: logger.Logger.Debug().Msg("Parse command list") for i, s := range baseAttribute.([]interface{}) { var values []string - if command.Parse.Type == "object" { + switch command.Parse.Type { + case "object": item := s.(orderedmap.OrderedMap) for _, key := range item.Keys() { if i == 0 { parseCommandResult.Header = append(parseCommandResult.Header, key) } value, exists := item.Get(key) + if exists { switch value.(type) { case string: values = append(values, fmt.Sprintf("%v", value)) + break default: bytes, _ := json.Marshal(value) values = append(values, fmt.Sprintf("%v", string(bytes))) @@ -49,14 +62,16 @@ func ParseCommand(command cmd.Command, commandOutput string) ParseCommandResult } } parseCommandResult.Values = append(parseCommandResult.Values, values) - } else if command.Parse.Type == "list" { + break + case "list": if i == 0 { parseCommandResult.Header = append(parseCommandResult.Header, "Item") } if s != nil { parseCommandResult.Values = append(parseCommandResult.Values, append(values, s.(string))) } - } else { + break + default: logger.Logger.Debug().Msg("Wrong type. Accepted types [Object, List]") } }