diff --git a/src/main/java/ru/naumen/sd40/log/parser/ActionDoneParser.java b/src/main/java/ru/naumen/sd40/log/parser/ActionDoneParser.java index d3bb85c..29c8b97 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/ActionDoneParser.java +++ b/src/main/java/ru/naumen/sd40/log/parser/ActionDoneParser.java @@ -12,8 +12,7 @@ /** * Created by doki on 22.10.16. */ -public class ActionDoneParser -{ +public class ActionDoneParser implements IDataParser{ private static Set EXCLUDED_ACTIONS = new HashSet<>(); static diff --git a/src/main/java/ru/naumen/sd40/log/parser/App.java b/src/main/java/ru/naumen/sd40/log/parser/App.java index cbb518f..708bbdb 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/App.java +++ b/src/main/java/ru/naumen/sd40/log/parser/App.java @@ -10,7 +10,6 @@ import org.springframework.web.multipart.MultipartFile; import ru.naumen.perfhouse.influx.InfluxDAO; -import ru.naumen.sd40.log.parser.GCParser.GCTimeParser; /** * Created by doki on 22.10.16. @@ -36,97 +35,75 @@ public static void parse(MultipartFile logs, String nameInfluxDB, String timeZon BatchPoints points = influxDAO.startBatchPoints(influxDb); HashMap data = new HashMap<>(); - TimeParser timeParser = new TimeParser(timeZone); - GCTimeParser gcTime = new GCTimeParser(timeZone); - - switch (parseMode) - { + IDataParser parser; + ITimeParser timeParser; + switch (parseMode){ case "sdng": //Parse sdng - try (BufferedReader br = new BufferedReader(new InputStreamReader(logs.getInputStream()))) - { - String line; - while ((line = br.readLine()) != null) - { - long time = timeParser.parseLine(line); - - if (time == 0) - { - continue; - } - - int min5 = 5 * 60 * 1000; - long count = time / min5; - long key = count * min5; - - data.computeIfAbsent(key, k -> new DataSet()).parseLine(line); - } - } + parser = new SDNGParser(); + timeParser = new SDNGTimeParser(); break; case "gc": //Parse gc log - try (BufferedReader br = new BufferedReader(new InputStreamReader(logs.getInputStream()))) - { - String line; - while ((line = br.readLine()) != null) - { - long time = gcTime.parseTime(line); - - if (time == 0) - { - continue; - } - - int min5 = 5 * 60 * 1000; - long count = time / min5; - long key = count * min5; - data.computeIfAbsent(key, k -> new DataSet()).parseGcLine(line); - } - } + parser = new GCParser(); + timeParser = new GCTimeParser(); break; case "top": - TopParser topParser = new TopParser(logs, data); - topParser.configureTimeZone(timeZone); - //Parse top - topParser.parse(); + //Parse top + parser = new TopParser(); + timeParser = new TopTimeParser(logs, data, timeZone); + break; default: throw new IllegalArgumentException( "Unknown parse mode! Availiable modes: sdng, gc, top. Requested mode: " + parseMode); } + parse(data, timeZone, logs, parser, timeParser); - if (traceCheck) - { + if (traceCheck){ System.out.print("Timestamp;Actions;Min;Mean;Stddev;50%%;95%%;99%%;99.9%%;Max;Errors\n"); } - data.forEach((k, set) -> - { + data.forEach((k, set) ->{ ActionDoneParser dones = set.getActionsDone(); dones.calculate(); ErrorParser erros = set.getErrors(); - if (traceCheck) - { + if (traceCheck){ System.out.print(String.format("%d;%d;%f;%f;%f;%f;%f;%f;%f;%f;%d\n", k, dones.getCount(), dones.getMin(), dones.getMean(), dones.getStddev(), dones.getPercent50(), dones.getPercent95(), dones.getPercent99(), dones.getPercent999(), dones.getMax(), erros.getErrorCount())); } - if (!dones.isNan()) - { + if (!dones.isNan()){ influxDAO.storeActionsFromLog(points, influxDb, k, dones, erros); } GCParser gc = set.getGc(); - if (!gc.isNan()) - { + if (!gc.isNan()){ influxDAO.storeGc(points, influxDb, k, gc); } TopData cpuData = set.cpuData(); - if (!cpuData.isNan()) - { + if (!cpuData.isNan()){ influxDAO.storeTop(points, influxDb, k, cpuData); } }); influxDAO.writeBatch(points); } + + private static void parse(HashMap data, String timeZone, + MultipartFile logs, IDataParser dataParser, ITimeParser timeParser) + throws ParseException, IOException { + try (BufferedReader br = new BufferedReader(new InputStreamReader(logs.getInputStream()))){ + String line; + while ((line = br.readLine()) != null){ + long time = timeParser.parseTime(line); + + if (time == 0){ + continue; + } + + data.computeIfAbsent(time, k -> new DataSet()); + dataParser.parseLine(line); + } + } + } } diff --git a/src/main/java/ru/naumen/sd40/log/parser/DataSet.java b/src/main/java/ru/naumen/sd40/log/parser/DataSet.java index bdcd410..b02f8b9 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/DataSet.java +++ b/src/main/java/ru/naumen/sd40/log/parser/DataSet.java @@ -10,41 +10,25 @@ public class DataSet private GCParser gc; private TopData cpuData = new TopData(); - public DataSet() - { + public DataSet(){ actionsDone = new ActionDoneParser(); errors = new ErrorParser(); gc = new GCParser(); } - public void parseLine(String line) - { - errors.parseLine(line); - actionsDone.parseLine(line); - } - - public void parseGcLine(String line) - { - gc.parseLine(line); - } - - public ActionDoneParser getActionsDone() - { + public ActionDoneParser getActionsDone(){ return actionsDone; } - public ErrorParser getErrors() - { + public ErrorParser getErrors(){ return errors; } - public GCParser getGc() - { + public GCParser getGc(){ return gc; } - public TopData cpuData() - { + public TopData cpuData(){ return cpuData; } } diff --git a/src/main/java/ru/naumen/sd40/log/parser/ErrorParser.java b/src/main/java/ru/naumen/sd40/log/parser/ErrorParser.java index 32bc357..3325f36 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/ErrorParser.java +++ b/src/main/java/ru/naumen/sd40/log/parser/ErrorParser.java @@ -5,8 +5,7 @@ /** * Created by doki on 22.10.16. */ -public class ErrorParser -{ +public class ErrorParser implements IDataParser{ long warnCount; long errorCount; long fatalCount; @@ -15,7 +14,7 @@ public class ErrorParser Pattern errorRegEx = Pattern.compile("^\\d+ \\[.+?\\] \\(.+?\\) ERROR"); Pattern fatalRegEx = Pattern.compile("^\\d+ \\[.+?\\] \\(.+?\\) FATAL"); - void parseLine(String line) + public void parseLine(String line) { if (warnRegEx.matcher(line).find()) { diff --git a/src/main/java/ru/naumen/sd40/log/parser/GCParser.java b/src/main/java/ru/naumen/sd40/log/parser/GCParser.java index 6b80531..405bff8 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/GCParser.java +++ b/src/main/java/ru/naumen/sd40/log/parser/GCParser.java @@ -3,77 +3,36 @@ import static ru.naumen.sd40.log.parser.NumberUtils.getSafeDouble; import static ru.naumen.sd40.log.parser.NumberUtils.roundToTwoPlaces; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; -import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; -public class GCParser -{ - public final static class GCTimeParser - { - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", - new Locale("ru", "RU")); - - private static final Pattern PATTERN = Pattern - .compile("^(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\+\\d{4}).*"); - - public GCTimeParser() - { - DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); - } - - public GCTimeParser(String timeZone) - { - DATE_FORMAT.setTimeZone(TimeZone.getTimeZone(timeZone)); - } - - public long parseTime(String line) throws ParseException - { - Matcher matcher = PATTERN.matcher(line); - if (matcher.find()) - { - Date parse = DATE_FORMAT.parse(matcher.group(1)); - return parse.getTime(); - } - return 0L; - } - } +public class GCParser implements IDataParser{ private DescriptiveStatistics ds = new DescriptiveStatistics(); private Pattern gcExecutionTime = Pattern.compile(".*real=(.*)secs.*"); - public double getCalculatedAvg() - { + public double getCalculatedAvg(){ return roundToTwoPlaces(getSafeDouble(ds.getMean())); } - public long getGcTimes() - { + public long getGcTimes() { return ds.getN(); } - public double getMaxGcTime() - { + public double getMaxGcTime(){ return roundToTwoPlaces(getSafeDouble(ds.getMax())); } - public boolean isNan() - { + public boolean isNan(){ return getGcTimes() == 0; } - public void parseLine(String line) - { + public void parseLine(String line){ Matcher matcher = gcExecutionTime.matcher(line); - if (matcher.find()) - { + if (matcher.find()){ ds.addValue(Double.parseDouble(matcher.group(1).trim().replace(',', '.'))); } } diff --git a/src/main/java/ru/naumen/sd40/log/parser/GCTimeParser.java b/src/main/java/ru/naumen/sd40/log/parser/GCTimeParser.java new file mode 100644 index 0000000..5b6b116 --- /dev/null +++ b/src/main/java/ru/naumen/sd40/log/parser/GCTimeParser.java @@ -0,0 +1,34 @@ +package ru.naumen.sd40.log.parser; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class GCTimeParser implements ITimeParser{ + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", + new Locale("ru", "RU")); + + private static final Pattern PATTERN = Pattern + .compile("^(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}\\+\\d{4}).*"); + + public GCTimeParser(){ + DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + public GCTimeParser(String timeZone){ + DATE_FORMAT.setTimeZone(TimeZone.getTimeZone(timeZone)); + } + + public long parseTime(String line) throws ParseException{ + Matcher matcher = PATTERN.matcher(line); + if (matcher.find()){ + Date parse = DATE_FORMAT.parse(matcher.group(1)); + return parse.getTime(); + } + return 0L; + } +} \ No newline at end of file diff --git a/src/main/java/ru/naumen/sd40/log/parser/IDataParser.java b/src/main/java/ru/naumen/sd40/log/parser/IDataParser.java new file mode 100644 index 0000000..790ea42 --- /dev/null +++ b/src/main/java/ru/naumen/sd40/log/parser/IDataParser.java @@ -0,0 +1,7 @@ +package ru.naumen.sd40.log.parser; + +import java.text.ParseException; + +public interface IDataParser { + public void parseLine(String line) throws ParseException; +} diff --git a/src/main/java/ru/naumen/sd40/log/parser/ITimeParser.java b/src/main/java/ru/naumen/sd40/log/parser/ITimeParser.java new file mode 100644 index 0000000..a3b8fd0 --- /dev/null +++ b/src/main/java/ru/naumen/sd40/log/parser/ITimeParser.java @@ -0,0 +1,7 @@ +package ru.naumen.sd40.log.parser; + +import java.text.ParseException; + +public interface ITimeParser { + public long parseTime(String line) throws ParseException; +} diff --git a/src/main/java/ru/naumen/sd40/log/parser/SDNGParser.java b/src/main/java/ru/naumen/sd40/log/parser/SDNGParser.java new file mode 100644 index 0000000..99a6e31 --- /dev/null +++ b/src/main/java/ru/naumen/sd40/log/parser/SDNGParser.java @@ -0,0 +1,17 @@ +package ru.naumen.sd40.log.parser; + + +/** + * Created by doki on 22.10.16. + */ +public class SDNGParser implements IDataParser{ + + private DataSet currentSet; + + public void parseLine(String line){ + ErrorParser errors = currentSet.getErrors(); + ActionDoneParser actionsDone = currentSet.getActionsDone(); + errors.parseLine(line); + actionsDone.parseLine(line); + } +} diff --git a/src/main/java/ru/naumen/sd40/log/parser/TimeParser.java b/src/main/java/ru/naumen/sd40/log/parser/SDNGTimeParser.java similarity index 78% rename from src/main/java/ru/naumen/sd40/log/parser/TimeParser.java rename to src/main/java/ru/naumen/sd40/log/parser/SDNGTimeParser.java index adbd9d0..3e971ef 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/TimeParser.java +++ b/src/main/java/ru/naumen/sd40/log/parser/SDNGTimeParser.java @@ -8,36 +8,28 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - * Created by doki on 22.10.16. - */ -public class TimeParser -{ +public class SDNGTimeParser implements ITimeParser{ private static final Pattern TIME_PATTERN = Pattern .compile("^\\d+ \\[.*?\\] \\((\\d{2} .{3} \\d{4} \\d{2}:\\d{2}:\\d{2},\\d{3})\\)"); private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("dd MMM yyyy HH:mm:ss,SSS", new Locale("ru", "RU")); - public TimeParser() - { + public SDNGTimeParser(){ DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); } - public TimeParser(String zoneId) - { + public SDNGTimeParser(String zoneId){ DATE_FORMAT.setTimeZone(TimeZone.getTimeZone(zoneId)); } - public long parseLine(String line) throws ParseException - { + public long parseTime(String line) throws ParseException { Matcher matcher = TIME_PATTERN.matcher(line); - if (matcher.find()) - { + if (matcher.find()){ String timeString = matcher.group(1); Date recDate = DATE_FORMAT.parse(timeString); return recDate.getTime(); } return 0L; - } -} + } +} \ No newline at end of file diff --git a/src/main/java/ru/naumen/sd40/log/parser/TopParser.java b/src/main/java/ru/naumen/sd40/log/parser/TopParser.java index 1f6ae81..c8f73c4 100644 --- a/src/main/java/ru/naumen/sd40/log/parser/TopParser.java +++ b/src/main/java/ru/naumen/sd40/log/parser/TopParser.java @@ -1,13 +1,6 @@ package ru.naumen.sd40.log.parser; -import org.springframework.web.multipart.MultipartFile; -import java.io.InputStreamReader; -import java.io.BufferedReader; -import java.io.IOException; import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Map; -import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -16,66 +9,15 @@ * @author dkolmogortsev * */ -public class TopParser -{ - - private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH:mm"); - - private String dataDate; - private MultipartFile logs; - - private Map existing; - - private Pattern timeRegex = Pattern.compile("^_+ (\\S+)"); +public class TopParser implements IDataParser{ + + private DataSet currentSet; private Pattern cpuAndMemPattren = Pattern .compile("^ *\\d+ \\S+ +\\S+ +\\S+ +\\S+ +\\S+ +\\S+ +\\S+ \\S+ +(\\S+) +(\\S+) +\\S+ java"); - private DataSet currentSet; - - public TopParser(MultipartFile logs, Map existingDataSet) throws IllegalArgumentException - { - //Supports these masks in file name: YYYYmmdd, YYY-mm-dd i.e. 20161101, 2016-11-01 - Matcher matcher = Pattern.compile("\\d{8}|\\d{4}-\\d{2}-\\d{2}").matcher(logs.getOriginalFilename()); - if (!matcher.find()) - { - throw new IllegalArgumentException(); - } - this.dataDate = matcher.group(0).replaceAll("-", ""); - this.logs = logs; - this.existing = existingDataSet; - sdf.setTimeZone(TimeZone.getTimeZone("GMT")); - - } - - public void configureTimeZone(String timeZone) - { - sdf.setTimeZone(TimeZone.getTimeZone(timeZone)); - } - - public void parse() throws IOException, ParseException - { - try (BufferedReader br = new BufferedReader(new InputStreamReader(logs.getInputStream()))) - { - String line; - while ((line = br.readLine()) != null) - { - parseLine(line); - } - } - } - - private void parseLine(String line) throws IOException, ParseException - { + public void parseLine(String line) throws ParseException{ //check time - long time = 0; - Matcher matcher = timeRegex.matcher(line); - if (matcher.find()) - { - time = prepareDate(sdf.parse(dataDate + matcher.group(1)).getTime()); - currentSet = existing.computeIfAbsent(time, k -> new DataSet()); - return; - } if (currentSet != null) { //get la @@ -96,12 +38,4 @@ private void parseLine(String line) throws IOException, ParseException } } } - - private long prepareDate(long parsedDate) - { - int min5 = 5 * 60 * 1000; - long count = parsedDate / min5; - return count * min5; - } - } diff --git a/src/main/java/ru/naumen/sd40/log/parser/TopTimeParser.java b/src/main/java/ru/naumen/sd40/log/parser/TopTimeParser.java new file mode 100644 index 0000000..8115dda --- /dev/null +++ b/src/main/java/ru/naumen/sd40/log/parser/TopTimeParser.java @@ -0,0 +1,49 @@ +package ru.naumen.sd40.log.parser; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Map; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.web.multipart.MultipartFile; + +public class TopTimeParser implements ITimeParser{ + private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHH:mm"); + private String dataDate; + private MultipartFile logs; +// private Map existing; +// private Pattern timeRegex = Pattern.compile("^_+ (\\S+)"); + + public TopTimeParser(MultipartFile logs, Map existingDataSet, String timeZone) throws IllegalArgumentException{ + //Supports these masks in file name: YYYYmmdd, YYY-mm-dd i.e. 20161101, 2016-11-01 + Matcher matcher = Pattern.compile("\\d{8}|\\d{4}-\\d{2}-\\d{2}").matcher(logs.getOriginalFilename()); + if (!matcher.find()){ + throw new IllegalArgumentException(); + } + this.dataDate = matcher.group(0).replaceAll("-", ""); + this.logs = logs; + this.existing = existingDataSet; + configureTimeZone(timeZone); + } + public void configureTimeZone(String timeZone){ + sdf.setTimeZone(TimeZone.getTimeZone(timeZone)); + } + + public long parseTime(String line) throws ParseException { + long time; + Matcher matcher = Pattern.compile("\\d{8}|\\d{4}-\\d{2}-\\d{2}").matcher(logs.getOriginalFilename()); + if (matcher.find()){ + time = prepareDate(sdf.parse(dataDate + matcher.group(1)).getTime()); + return time; + } + return 0L; + } + + private long prepareDate(long parsedDate){ + int min5 = 5 * 60 * 1000; + long count = parsedDate / min5; + return count * min5; + } +} \ No newline at end of file