OwlCyberSecurity - MANAGER
Edit File: monitoranalysis.py
#!/usr/bin/env python ## # Copyright (c) 2010-2017 Apple Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ## from __future__ import print_function import matplotlib.pyplot as plt import getopt import sys import os import datetime dataset = [] initialDate = None def analyze(fpath, noweekends, startDate=None, endDate=None, title=None): print("Analyzing data for %s" % (fpath,)) data = [] firstDate = None global initialDate with open(fpath) as f: for line in f: try: if line.startswith("2010/0"): date = line[:10] if startDate and date < startDate or endDate and date > endDate: continue if noweekends: dt = datetime.date(int(date[0:4]), int(date[5:7]), int(date[8:10])) if dt.weekday() > 4: continue digits = line[11:13] if digits in ("05", "06"): for _ignore in range(3): f.next() continue dtstamp = line[:19] if firstDate is None: firstDate = date.replace("/", "") if initialDate is None: initialDate = firstDate if "Listenq" in line: lqnon = line[len("2010/05/12 22:27:24 Listenq (ssl+non): "):].split("+", 1)[1] else: lqnon = line[len("2010/01/05 19:47:23 Listen queue: "):] lqnon = int(lqnon.split(" ", 1)[0]) line = f.next() cpu = int(line[len("CPU idle %: "):].split(" ", 1)[0]) line = f.next() if line.startswith("Memory"): line = f.next() reqs = int(float(line.split(" ", 1)[0])) line = f.next() resp = line[len("Response time: average "):].split(" ", 1)[0] resp = int(float(resp) / 10.0) * 10 if reqs <= 80: data.append((dtstamp, reqs, resp, lqnon, cpu)) # print("%s %d %d %d %d" % (dtstamp, reqs, resp, lqnon, cpu)) except StopIteration: break if not title: if startDate and endDate: title = "Between %s and %s" % (startDate, endDate,) elif startDate: title = "Since %s" % (startDate,) elif endDate: title = "Up to %s" % (endDate,) else: title = "Start at %s" % (firstDate,) dataset.append((title, data,)) print("Stored %d data points" % (len(data),)) def plotListenQBands(data, first, last, xlim, ylim): x1 = [] y1 = [] x2 = [] y2 = [] x3 = [] y3 = [] for _ignore_dt, reqs, resp, lq, _ignore_cpu in data: if lq == 0: x1.append(reqs) y1.append(resp) elif lq < 50: x2.append(reqs) y2.append(resp) else: x3.append(reqs) y3.append(resp) plt.plot(x1, y1, "b+", x2, y2, "g+", x3, y3, "y+") if first: plt.legend( ('ListenQ at zero', 'ListenQ < 50', 'ListenQ >= 50'), 'upper left', shadow=True, fancybox=True ) if last: plt.xlabel("Requests/second") plt.ylabel("Av. Response Time (ms)") plt.xlim(0, xlim) plt.ylim(0, ylim) def plotCPUBands(data, first, last, xlim, ylim): x = [[], [], [], []] y = [[], [], [], []] for _ignore_dt, reqs, resp, _ignore_lq, cpu in data: if cpu > 75: x[0].append(reqs) y[0].append(resp) elif cpu > 50: x[1].append(reqs) y[1].append(resp) elif cpu > 25: x[2].append(reqs) y[2].append(resp) else: x[3].append(reqs) y[3].append(resp) plt.plot( x[0], y[0], "b+", x[1], y[1], "g+", x[2], y[2], "y+", x[3], y[3], "m+", ) if first: plt.legend( ('CPU < 1/4', 'CPU < 1/2', 'CPU < 3/4', "CPU High"), 'upper left', shadow=True, fancybox=True ) if last: plt.xlabel("Requests/second") plt.ylabel("Av. Response Time (ms)") plt.xlim(0, xlim) plt.ylim(0, ylim) def plot(figure, noshow, nosave, pngDir, xlim, ylim): print("Plotting data") plt.figure(figure, figsize=(16, 5 * len(dataset))) nplots = len(dataset) subplot = nplots * 100 + 20 for ctr, item in enumerate(dataset): title, data = item if not title: title = "#%d" % (ctr + 1,) plt.subplot(subplot + 2 * ctr + 1) plotListenQBands(data, first=(ctr == 0), last=(ctr + 1 == len(dataset)), xlim=xlim, ylim=ylim) plt.title("ListenQ %s" % (title,)) plt.subplot(subplot + 2 * ctr + 2) plotCPUBands(data, first=(ctr == 0), last=(ctr + 1 == len(dataset)), xlim=xlim, ylim=ylim) plt.title("CPU %s" % (title,)) def argPath(path): fpath = os.path.expanduser(path) if not fpath.startswith("/"): fpath = os.path.join(pwd, fpath) return fpath def expandDate(date): return "%s/%s/%s" % (date[0:4], date[4:6], date[6:8],) def usage(error_msg=None): if error_msg: print(error_msg) print("""Usage: monitoranalysis [options] [FILE+] Options: -h Print this help and exit -d Directory to save PNGs to -s Directory to scan for data instead of FILEs --no-weekends Ignore data for Saturday and Sunday --no-show Do not show plots on screen --no-save Do not save plots to file --xlim x-axis limit [80] --ylim y-axim limit [4000] Arguments: FILE File names for the requests.log to analyze. A date range can be specified by append a comma, then a dash seperated pair of YYYYMMDD dates, e.g.: ~/request.log,20100614-20100619. Multiple ranges can be specified for multiple plots. Description: This utility will analyze the output of the request monitor tool and generate some pretty plots of data. """) if error_msg: raise ValueError(error_msg) else: sys.exit(0) if __name__ == "__main__": pngDir = None scanDir = None noweekends = False noshow = False nosave = False xlim = 80 ylim = 4000 options, args = getopt.getopt(sys.argv[1:], "hd:s:", ["no-weekends", "no-show", "no-save", "xlim=", "ylim="]) for option, value in options: if option == "-h": usage() elif option == "-d": pngDir = os.path.expanduser(value) elif option == "-s": scanDir = os.path.expanduser(value) elif option == "--no-show": noshow = True elif option == "--no-save": nosave = True elif option == "--no-weekends": noweekends = True elif option == "--xlim": xlim = int(value) elif option == "--ylim": ylim = int(value) else: usage("Unrecognized option: %s" % (option,)) if pngDir is None and scanDir: pngDir = scanDir if not nosave and not os.path.isdir(pngDir): usage("Must have a valid -d path for saving images") # Process arguments if len(args) == 0 and scanDir is None: usage("Must have arguments") elif scanDir and len(args) != 0: usage("No arguments allowed when scanning a directory") pwd = os.getcwd() if scanDir: fnames = os.listdir(scanDir) count = 1 for name in fnames: if name.startswith("request.log"): print("Found file: %s" % (os.path.join(scanDir, name),)) trailer = name[len("request.log"):] if trailer.startswith("."): trailer = trailer[1:] initialDate = None dataset = [] analyze(os.path.join(scanDir, name), noweekends) plot(count, noshow, nosave, pngDir, xlim, ylim) if not nosave: plt.savefig(os.path.expanduser(os.path.join(pngDir, "Monitor-%s" % (trailer,)))) count += 1 if not noshow: plt.show() else: for arg in args: if "," in arg: items = arg.split(",") arg = items[0] start = [] end = [] for daterange in items[1:]: splits = daterange.split("-") if len(splits) == 1: start.append(expandDate(splits[0])) end.append(None) elif len(splits) == 2: start.append(expandDate(splits[0])) end.append(expandDate(splits[1])) else: start.append(None) end.append(None) else: start = (None,) end = (None,) for i in range(len(start)): analyze(argPath(arg), noweekends, start[i], end[i]) plot(1, noshow, nosave, pngDir, xlim, ylim) if not nosave: plt.savefig(os.path.expanduser(os.path.join(pngDir, "Monitor-%s" % (initialDate,)))) if not noshow: plt.show()