Added support of field list, output format, etc

.gitignore

@@ -0,0 +1 @@
+.idea

README.md

@@ -32,6 +32,24 @@ This uses Graylog's search mechanism, so you can use `field: value` syntax.
 
 `gtail --stream nile --query crocodile`
 
+### Select fields
+
+`gtail --fields comma-separated-fields-here`
+
+This will display only selected fields.
+
+### Display format
+
+`gtail --format [text or json]`
+
+Choose between comma separated values map and JSON output.
+
+### Initial range
+
+`gtail --range [interval]`
+
+Time range for initial fetch. Use "h" for hours, "m" for minutes, "s" or no identifier for seconds.
+
 ### Full usage instructions
 
 `gtail --help`

gtail/gtail.py

@@ -12,10 +12,30 @@
 import sys
 import time
 import urllib
+from json import dumps
 
 MAX_DELAY = 10
+DEFAULT_RANGE="5m"
 DEFAULT_CONFIG_PATHS = [".gtail", os.path.expanduser("~/.gtail")]
 
+# converts human readable time interval into seconds
+def convert_time_interval(value):
+    value = value.lower()
+    value_int = 0
+    if "h" in value:
+        value_parts = value.split("h", 1)
+        value_int+=int(value_parts[0])*3600
+        value=value_parts[1]
+    if "m" in value:
+        value_parts = value.split("m", 1)
+        value_int += int(value_parts[0]) * 60
+        value = value_parts[1]
+    value_parts = value.split("s", 1)
+    if value_parts[0] == "":
+        value_parts[0] = "0"
+    value_int += int(value_parts[0])
+    return value_int
+
 # returns a bold version of text using ansi characters
 def bold(text):
    make_bold = "\033[1m"
@@ -60,17 +80,32 @@ def list_streams(streams):
 def fetch_messages(server_config,
         query = None,
         stream_ids = None,
-        last_message_id = None):
+        last_message_id = None,
+        fields = None,
+        delay = MAX_DELAY,
+        initial_range = None):
     url = []
     url.append(server_config.uri)
-    url.append("/search/universal/relative?range=7200&limit=100")
+    if last_message_id:
+        limit = "&limit={0}".format(1000)
+        range = max(delay * 5, 300)
+    else:
+        range=initial_range
+        limit=""
+    url.append("/search/universal/relative?range={range}{limit}".format(range=range, limit=limit))
 
     # query terms
     if query:
         url.append("&query=" + urllib.quote_plus(query))
     else:
         url.append("&query=*")
 
+    # fields list
+    if fields:
+        if "_id" not in fields:
+            fields.append("_id")
+        url.append("&fields=" + "%2C".join(fields))
+
     # stream ID
     if stream_ids:
         quoted = map(urllib.quote_plus, stream_ids)
@@ -109,37 +144,46 @@ def fetch_messages(server_config,
 # pretty prints a message
 # streams, if provided, is the full list of streams; it is used for pretty
 # printing of the stream name
-def print_message(message, streams=None):
-    s = []
-    if "timestamp" in message:
-        timestamp = message["timestamp"]
-        s.append(timestamp)
-    if streams and "streams" in message:
-        stream_ids = message["streams"]
-        stream_names = []
-        for sid in stream_ids:
-            stream_names.append(streams[sid]["title"])
-        s.append("[" + ", ".join(stream_names) + "]")
-    if "facility" in message:
-        facility = message["facility"]
-        s.append(facility)
-    if "level" in message:
-        level = message["level"]
-        s.append(level)
-    if "source" in message:
-        source = message["source"]
-        s.append(source)
-    if "loggerName" in message:
-        logger_name = message["loggerName"]
-        s.append(logger_name)
-
-    if "full_message" in message:
-        text = message["full_message"]
+def print_message(message, streams=None, fields=None, format="json"):
+    s = dict()
+    text = None
+    if fields:
+        count = 0
+        for field in fields:
+            if field != "_id" and field in message:
+                count += 1
+                s[field] = str(message[field])
     else:
-        text = message["message"]
+        if "timestamp" in message:
+            s["timestamp"] = str(message["timestamp"])
+        if streams and "streams" in message:
+            stream_ids = message["streams"]
+            stream_names = []
+            for sid in stream_ids:
+                stream_names.append(streams[sid]["title"])
+            s["streams"] = "[" + ", ".join(stream_names) + "]"
+        if "facility" in message:
+            s["facility"] = message["facility"]
+        if "level" in message:
+            s["level"] = message["level"]
+        if "source" in message:
+            s["source"] = message["source"]
+        if "loggerName" in message:
+            s["loggerName"] = message["loggerName"]
+
+        if "full_message" in message:
+            text = message["full_message"]
+        elif "message" in message:
+            text = message["message"]
+
+    if format == "text":
+        out = map(str, s.values())
+    else:
+        out = dumps(s)
+    print bold(out)
 
-    print bold(" ".join(map(str, s)))
-    print text
+    if text:
+        print text
 
 # config object and config parsing
 Config = namedtuple("Config", "server_config")
@@ -234,6 +278,18 @@ def main():
     parser.add_argument("--query", dest="query",
             nargs="+",
             help="Query terms to search on")
+    parser.add_argument("--fields", dest="fields",
+                        nargs="+",
+                        help="Fields to display")
+    parser.add_argument("--format", dest="format",
+                        choices=["text", "json"], default="json",
+                        help="Display format")
+    parser.add_argument("--delay", dest="delay",
+                        type=int, default=MAX_DELAY,
+                        help="Delay between Rest API calls (seconds)")
+    parser.add_argument("--range", dest="range",
+                        type=str, default=DEFAULT_RANGE,
+                        help="Time range for initial fetch")
     parser.add_argument("--config", dest="config_paths",
             nargs="+",
             help="Config files. Default: " + ", ".join(DEFAULT_CONFIG_PATHS))
@@ -280,37 +336,48 @@ def main():
     # print log messages
     #
 
-    last_message_id = None
-    while True:
-        # time-forward messages
-        query = None
-        if args.query:
-            query = ' '.join(args.query)
-        try:
-            messages = fetch_messages(
-                    server_config = server_config,
-                    query = query,
-                    stream_ids = stream_ids,
-                    last_message_id = last_message_id)
-        except Exception as e:
-            print e
-            time.sleep(MAX_DELAY)
-            continue
-
-        # print new messages
-        last_timestamp = None
-        for m in messages:
-            print_message(m, streams)
-            last_message_id = m["_id"]
-            last_timestamp = m["timestamp"]
-
-        if last_timestamp:
-            seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds())
-            delay = min(seconds_since_last_message, MAX_DELAY)
-            if delay > 2:
-                time.sleep(delay)
-        else:
-            time.sleep(MAX_DELAY)
+    try:
+        last_message_id = None
+        while True:
+            # time-forward messages
+            query = None
+            fields = None
+            if args.query:
+                query = ' '.join(args.query)
+            if args.fields:
+                fields = []
+                for field in args.fields:
+                  fields.extend(field.split(","))
+            try:
+                messages = fetch_messages(
+                        server_config = server_config,
+                        query = query,
+                        stream_ids = stream_ids,
+                        last_message_id = last_message_id,
+                        fields=fields,
+                        delay=args.delay,
+                        initial_range=convert_time_interval(args.range))
+            except Exception as e:
+                print e
+                time.sleep(args.delay)
+                continue
+
+            # print new messages
+            last_timestamp = None
+            for m in messages:
+                print_message(m, streams, fields=fields, format=args.format)
+                last_message_id = m["_id"]
+                last_timestamp = m["timestamp"]
+
+            if last_timestamp:
+                seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds())
+                delay = min(seconds_since_last_message, args.delay)
+                if delay > 2:
+                    time.sleep(delay)
+            else:
+                time.sleep(args.delay)
+    except KeyboardInterrupt:
+        os._exit(0)
 
 if __name__ == "__main__":
     rc = main()