Python cmdlinetool.LogFileTool类代码示例

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=True, stdin_allowed=True)

        self.argparser.description='A script to plot various information from logfiles. ' \
            'Clicking on any of the plot points will print the corresponding log line to stdout.'

        # import all plot type classes in plottypes module
        self.plot_types = [c[1] for c in inspect.getmembers(plottypes, inspect.isclass)]
        self.plot_types = dict((pt.plot_type_str, pt) for pt in self.plot_types)
        self.plot_instances = []

        # main parser arguments
        self.argparser.add_argument('--exclude-ns', action='store', nargs='*', metavar='NS', help='(deprecated) use a prior mlogfilter instead.')
        self.argparser.add_argument('--ns', action='store', nargs='*', metavar='NS', help='(deprecated) use a prior mlogfilter instead. ')
        self.argparser.add_argument('--logscale', action='store_true', help='plot y-axis in logarithmic scale (default=off)')
        self.argparser.add_argument('--overlay', action='store', nargs='?', default=None, const='add', choices=['add', 'list', 'reset'], help="create combinations of several plots. Use '--overlay' to create an overlay (this will not plot anything). The first call without '--overlay' will additionally plot all existing overlays. Use '--overlay reset' to clear all overlays.")
        self.argparser.add_argument('--type', action='store', default='scatter', choices=self.plot_types.keys(), help='type of plot (default=scatter with --yaxis duration).')        
        self.argparser.add_argument('--group', help="specify value to group on. Possible values depend on type of plot. All basic plot types can group on 'namespace', 'operation', 'thread', range and histogram plots can additionally group on 'log2code'.")

        # mutex = self.argparser.add_mutually_exclusive_group()
        # mutex.add_argument('--label', help="instead of specifying a group, a label can be specified. Grouping is then disabled, and the single group for all data points is named LABEL.")

        self.legend = None

        # store logfile ranges
        self.logfile_ranges = []

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=True, stdin_allowed=True)

        self.argparser.description='A script to plot various information from logfiles. ' \
            'Clicking on any of the plot points will print the corresponding log line to stdout.'

        # disable some default shortcuts
        plt.rcParams['keymap.xscale'] = ''
        plt.rcParams['keymap.yscale'] = ''

        # import all plot type classes in plottypes module
        self.plot_types = [c[1] for c in inspect.getmembers(plottypes, inspect.isclass)]
        self.plot_types = dict((pt.plot_type_str, pt) for pt in self.plot_types)
        self.plot_instances = []
        self.plot_instance = None

        # main parser arguments
        self.argparser.add_argument('--logscale', action='store_true', help='plot y-axis in logarithmic scale (default=off)')
        self.argparser.add_argument('--overlay', action='store', nargs='?', default=None, const='add', choices=['add', 'list', 'reset'], help="create combinations of several plots. Use '--overlay' to create an overlay (this will not plot anything). The first call without '--overlay' will additionally plot all existing overlays. Use '--overlay reset' to clear all overlays.")
        self.argparser.add_argument('--type', action='store', default='scatter', choices=self.plot_types.keys(), help='type of plot (default=scatter with --yaxis duration).')        
        self.argparser.add_argument('--title', action='store', default=None, help='change the title of the plot (default=filename(s))')        
        self.argparser.add_argument('--group', help="specify value to group on. Possible values depend on type of plot. All basic plot types can group on 'namespace', 'operation', 'thread', 'pattern', range and histogram plots can additionally group on 'log2code'. The group can also be a regular expression.")
        self.argparser.add_argument('--group-limit', metavar='N', type=int, default=None, help="specify an upper limit of the number of groups. Groups are sorted by number of data points. If limit is specified, only the top N will be listed separately, the rest are grouped together in an 'others' group")
        self.argparser.add_argument('--no-others', action='store_true', default=False, help="if this flag is used, the 'others' group (see --group-limit) will be discarded.")
        self.argparser.add_argument('--optime-start', action='store_true', default=False, help="plot operations with a duration when they started instead (by subtracting the duration). The default is to plot them when they finish (at the time they are logged).")
        self.argparser.add_argument('--ylimits', action='store', default=None, type=int, nargs=2, metavar='VAL', help="if set, limits the y-axis view to [min, max], requires exactly 2 values.")
        self.argparser.add_argument('--output-file', metavar='FILE', action='store', default=None, help="Save the plot to a file instead of displaying it in a window")

        self.legend = None

        # progress bar
        self.progress_bar_enabled = not self.is_stdin

    def run(self, arguments=None):
        LogFileTool.run(self, arguments, get_unknowns=True)

        self.parse_loglines()
        self.group()

        if self.args['overlay'] == 'reset':
            self.remove_overlays()

        # if --overlay is set, save groups in a file, else load groups and plot
        if self.args['overlay'] == "list":
            self.list_overlays()
            raise SystemExit

        plot_specified = not sys.stdin.isatty() or len(self.args['logfile']) > 0

        # if no plot is specified (either pipe or filename(s)) and reset, quit now
        if not plot_specified and self.args['overlay'] == 'reset':
            raise SystemExit
        
        if self.args['overlay'] == "" or self.args['overlay'] == "add":
            if plot_specified:
                self.save_overlay()
            else:
                print "Nothing to plot."
            raise SystemExit

        # else plot (with potential overlays) if there is something to plot
        overlay_loaded = self.load_overlays()
        
        if plot_specified or overlay_loaded:
            self.plot()
        else:
            print "Nothing to plot."
            raise SystemExit

    def run(self, arguments=None):
        """Print useful information about the log file."""
        LogFileTool.run(self, arguments)

        for i, self.logfile in enumerate(self.args['logfile']):
            if i > 0:
                print("\n ------------------------------------------\n")

            if self.logfile.datetime_format == 'ctime-pre2.4':
                # no milliseconds when datetime format doesn't support it
                start_time = (self.logfile.start.strftime("%Y %b %d %H:%M:%S")
                              if self.logfile.start else "unknown")
                end_time = (self.logfile.end.strftime("%Y %b %d %H:%M:%S")
                            if self.logfile.start else "unknown")
            else:
                # include milliseconds
                start_time = (self.logfile.start.strftime("%Y %b %d "
                                                          "%H:%M:%S.%f")[:-3]
                              if self.logfile.start else "unknown")
                end_time = (self.logfile.end.strftime("%Y %b %d "
                                                      "%H:%M:%S.%f")[:-3]
                            if self.logfile.start else "unknown")

            print("     source: %s" % self.logfile.name)
            print("       host: %s"
                  % (self.logfile.hostname + ':' + str(self.logfile.port)
                     if self.logfile.hostname else "unknown"))
            print("      start: %s" % (start_time))
            print("        end: %s" % (end_time))

            # TODO: add timezone if iso8601 format
            print("date format: %s" % self.logfile.datetime_format)
            print("     length: %s" % len(self.logfile))
            print("     binary: %s" % (self.logfile.binary or "unknown"))

            version = (' -> '.join(self.logfile.versions) or "unknown")

            # if version is unknown, go by date
            if version == 'unknown':
                if self.logfile.datetime_format == 'ctime-pre2.4':
                    version = '< 2.4 (no milliseconds)'
                elif self.logfile.datetime_format == 'ctime':
                    version = '>= 2.4.x ctime (milliseconds present)'
                elif (self.logfile.datetime_format == "iso8601-utc" or
                      self.logfile.datetime_format == "iso8601-local"):
                    if self.logfile.has_level:
                        version = '>= 3.0 (iso8601 format, level, component)'
                    else:
                        version = '= 2.6.x (iso8601 format)'

            print("    version: %s" % version)
            print("    storage: %s"
                  % (self.logfile.storage_engine or 'unknown'))

            # now run all sections
            for section in self.sections:
                if section.active:
                    print("\n%s" % section.name.upper())
                    section.run()

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)

        self.argparser.description = 'mongod/mongos log file visualizer (browser edition). Extracts \
            information from each line of the log file and outputs a html file that can be viewed in \
            a browser. Automatically opens a browser tab and shows the file.'
        self.argparser.add_argument('--no-browser', action='store_true', help='only creates .html file, but does not open the browser.')
        self.argparser.add_argument('--out', '-o', action='store', default=None, help='filename to output. Default is <original logfile>.html')

    def run(self, arguments=None):
        """ Go through each line, convert string to LogLine object, then print
            JSON representation of the line. 
        """
        LogFileTool.run(self, arguments)

        for line in self.args['logfile']:
            print LogLine(line).to_json()

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=True, stdin_allowed=True)

        # add all filter classes from the filters module
        self.filters = [c[1] for c in inspect.getmembers(filters, inspect.isclass)]

        self.argparser.description = "mongod/mongos log file parser. Use parameters to enable filters. A line only gets printed if it passes all enabled filters. If several log files are provided, their lines are merged by timestamp."
        self.argparser.add_argument(
            "--verbose", action="store_true", help="outputs information about the parser and arguments."
        )
        self.argparser.add_argument(
            "--shorten",
            action="store",
            type=int,
            default=False,
            nargs="?",
            metavar="LENGTH",
            help="shortens long lines by cutting characters out of the middle until the length is <= LENGTH (default 200)",
        )
        self.argparser.add_argument(
            "--exclude",
            action="store_true",
            default=False,
            help="if set, excludes the matching lines rather than includes them.",
        )
        self.argparser.add_argument(
            "--human",
            action="store_true",
            help="outputs large numbers formatted with commas and print milliseconds as hr,min,sec,ms for easier readability.",
        )
        self.argparser.add_argument(
            "--json",
            action="store_true",
            help="outputs all matching lines in json format rather than the native log line.",
        )
        self.argparser.add_argument(
            "--markers",
            action="store",
            nargs="*",
            default=["filename"],
            help="use markers when merging several files to distinguish them. Choose from none, enum, alpha, filename (default), or provide list.",
        )
        self.argparser.add_argument(
            "--timezone",
            action="store",
            nargs="*",
            default=[],
            type=int,
            metavar="N",
            help="timezone adjustments: add N hours to corresponding log file, single value for global adjustment.",
        )
        self.argparser.add_argument(
            "--timestamp-format",
            action="store",
            default="none",
            choices=["none", "ctime-pre2.4", "ctime", "iso8601-utc", "iso8601-local"],
            help="choose datetime format for log output",
        )

    def __init__(self):
        """ Constructor: add description to argparser. """
        LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)
        
        self.argparser.description = 'Groups all log messages in the logfile together \
            and only displays a distinct set of messages with count'

        self.argparser.add_argument('--verbose', action='store_true', default=False, 
            help="outputs lines that couldn't be matched.")

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)
        
        self.filters = [] 

        self.argparser.description = 'mongod/mongos log file parser. Use parameters to enable filters. A line only gets printed if it passes all enabled filters.'
        self.argparser.add_argument('--verbose', action='store_true', help='outputs information about the parser and arguments.')
        self.argparser.add_argument('--shorten', action='store', type=int, default=False, nargs='?', metavar='LENGTH', help='shortens long lines by cutting characters out of the middle until the length is <= LENGTH (default 200)')
        self.argparser.add_argument('--exclude', action='store_true', default=False, help='if set, excludes the matching lines rather than includes them.')

    def run(self):
        """ parses the logfile and asks each filter if it accepts the line.
            it will only be printed if all filters accept the line.
        """

        # add arguments from filter classes before calling superclass run
        for f in self.filters:
            for fa in f.filterArgs:
                self.argparser.add_argument(fa[0], **fa[1])

        # now parse arguments and post-process
        LogFileTool.run(self)
        self.args = dict((k, self._arrayToString(self.args[k])) for k in self.args)

        # create filter objects from classes and pass args
        self.filters = [f(self.args) for f in self.filters]

        # remove non-active filter objects
        self.filters = [f for f in self.filters if f.active]

        # call setup for each active filter
        for f in self.filters:
            f.setup()

        if self.args['shorten'] != False:
            if self.args['shorten'] == None:
                self.args['shorten'] = 200        

        if self.args['verbose']:
            print "mlogfilter> command line arguments"
            for a in self.args:
                print "mlogfilter> %8s: %s" % (a, self.args[a])

        # go through each line and ask each filter if it accepts
        if not 'logfile' in self.args or not self.args['logfile']:
            exit()


        for line in self.args['logfile']:
            logline = LogLine(line)

            if self.args['exclude']:
                # print line if any filter disagrees
                if any([not f.accept(logline) for f in self.filters]):
                    self._outputLine(logline.line_str, self.args['shorten'])

            else:
                # only print line if all filters agree
                if all([f.accept(logline) for f in self.filters]):
                    self._outputLine(logline.line_str, self.args['shorten'])

                # if at least one filter refuses to accept any remaining lines, stop
                if any([f.skipRemaining() for f in self.filters]):
                    # if input is not stdin
                    if sys.stdin.isatty():
                        break

    def run(self, arguments=None):
        LogFileTool.run(self, arguments)

        possible_versions = set(Log2CodeConverter.all_versions)

        re_versiond = re.compile(r'db version v(\d\.\d\.\d), pdfile version')
        re_versions = re.compile(r'MongoS version (\d\.\d\.\d) starting:')

        re_brackets = re.compile(r'\[\w+\]')

        for i, line in enumerate(self.args['logfile']): 
            match = re_brackets.search(line)
            if not match:
                continue

            start = match.end()

            # check for explicit version string
            match = re_versiond.search(line[start:]) or re_versions.search(line[start:])

            if match:
                version = match.group(1)
                print "%32s %s" % ("restart detected in log line %i:"%(i+1), line.rstrip())
                print "%32s %s" % ("previous possible versions:", ", ".join([pv[1:] for pv in sorted(possible_versions)]))
                print "%32s %s" % ("version after restart is:", version)
                print
                possible_versions = set(["r"+version])
                
            if len(possible_versions) == 1:
                # from here on, version is known, skip to next section
                continue

            ll = LogLine(line)
            if ll.operation != None:
                # if log line is a known command operation (query, update, command, ...) skip
                continue

            lcl = self.log2code(line[start:])
            if lcl:
                old_len = len(possible_versions)
                possible_versions = possible_versions & set(lcl.versions)
                if len(possible_versions) != old_len:
                    print "%32s %s" % ("log line %i:"%(i+1), line.rstrip())
                    print "%32s %s" % ("matched pattern:", " ... ".join(lcl.pattern))
                    print "%32s %s" % ("only present in:", ", ".join(sorted(lcl.versions)))
                    print "%32s %s" % ("possible versions now:", ", ".join(sorted(possible_versions)))
                    print

            if len(possible_versions) == 0:
                print "empty version set. exiting."
                raise SystemExit

        if len(possible_versions) > 1:
            print "possible versions at end of file:", ", ".join([pv[1:] for pv in sorted(possible_versions)])
        else:
            print "version at end of file: ", possible_versions.pop()[1:]

    def __init__(self):
        """ Constructor: add description to argparser. """
        LogFileTool.__init__(self, multiple_logfiles=True, stdin_allowed=False)

        self.argparser.description = 'Extracts general information from logfile and prints it to stdout.'
        self.argparser.add_argument('--verbose', action='store_true', help='show more verbose output (depends on info section)')
        self.argparser_sectiongroup = self.argparser.add_argument_group('info sections', 'Below commands activate additional info sections for the log file.')

        # add all filter classes from the filters module
        self.sections = [c[1](self) for c in inspect.getmembers(sections, inspect.isclass)]

    def run(self, arguments=None):
        """ Print out useful information about the log file. """
        LogFileTool.run(self, arguments)

        self.logfiles = self.args['logfile']

        for i, logfileOpen in enumerate(self.args['logfile']):
            if i > 0:
                print
                print ' ------------------------------------------'
                print

            self.logfileOpen = logfileOpen
            self.logfile = LogFile(logfileOpen)

            print "        filename: %s" % self.args['logfile'][i].name
            print "start of logfile: %s" % (self.logfile.start.strftime("%b %d %H:%M:%S") if self.logfile.start else "unknown")
            print "  end of logfile: %s" % (self.logfile.end.strftime("%b %d %H:%M:%S") if self.logfile.start else "unknown")

            # get one logline (within first 20 lines) for datetime format
            logline = None
            for i in range(20):
                try:
                    logline = LogLine(logfileOpen.next())
                except StopIteration as e:
                    raise SystemExit("no valid log lines found (datetime not available).")
                if logline.datetime:
                    break

            # TODO: add timezone if iso8601 format

            print "    line numbers: %s" % self.logfile.num_lines
            print "          binary: %s" % (self.logfile.binary or "unknown")

            version = (' -> '.join(self.logfile.versions) or "unknown")

            # if version is unknown, go by date
            if version == 'unknown' and logline:
                if logline.datetime_format == 'ctime-pre2.4':
                    version = '< 2.4 (no milliseconds)'
                elif logline.datetime_format == 'ctime':
                    version = '>= 2.4 (milliseconds present)'
                elif logline.datetime_format.startswith('iso8601-'):
                    version = '>= 2.6 (iso8601 format)'

            print "         version: %s" % version,
            print

            # now run all sections
            for section in self.sections:
                if section.active:
                    print
                    print section.name.upper()
                    section.run()

    def __init__(self):
        LogFileTool.__init__(self, multiple_logfiles=False, stdin_allowed=True)
        
        # add all filter classes from the filters module
        self.filters = [c[1] for c in inspect.getmembers(filters, inspect.isclass)]

        self.argparser.description = 'mongod/mongos log file parser. Use parameters to enable filters. A line only gets printed if it passes all enabled filters.'
        self.argparser.add_argument('--verbose', action='store_true', help='outputs information about the parser and arguments.')
        self.argparser.add_argument('--shorten', action='store', type=int, default=False, nargs='?', metavar='LENGTH', help='shortens long lines by cutting characters out of the middle until the length is <= LENGTH (default 200)')
        self.argparser.add_argument('--exclude', action='store_true', default=False, help='if set, excludes the matching lines rather than includes them.')
        self.argparser.add_argument('--human', action='store_true', help='outputs numbers formatted with commas and milliseconds as hr,min,sec,ms for easier readability.')

    def run(self, arguments=None):
        """ Print out useful information about the log file. """
        LogFileTool.run(self, arguments)

        for i, self.logfile in enumerate(self.args['logfile']):
            if i > 0:
                print
                print ' ------------------------------------------'
                print

            if self.logfile.datetime_format == 'ctime-pre2.4':
                # no milliseconds when datetime format doesn't support it
                start_time = self.logfile.start.strftime("%Y %b %d %H:%M:%S") if self.logfile.start else "unknown"
                end_time = self.logfile.end.strftime("%Y %b %d %H:%M:%S") if self.logfile.start else "unknown"
            else:
                # include milliseconds
                start_time = self.logfile.start.strftime("%Y %b %d %H:%M:%S.%f")[:-3] if self.logfile.start else "unknown"
                end_time = self.logfile.end.strftime("%Y %b %d %H:%M:%S.%f")[:-3] if self.logfile.start else "unknown"

            print "     source: %s" % self.logfile.name
            print "       host: %s" % (self.logfile.hostname + ':' + self.logfile.port if self.logfile.hostname else "unknown")
            print "      start: %s" % (start_time)
            print "        end: %s" % (end_time)

            # TODO: add timezone if iso8601 format
            print "date format: %s" % self.logfile.datetime_format
            print "     length: %s" % len(self.logfile)
            print "     binary: %s" % (self.logfile.binary or "unknown")
            

            version = (' -> '.join(self.logfile.versions) or "unknown")

            # if version is unknown, go by date
            if version == 'unknown':
                if self.logfile.datetime_format == 'ctime-pre2.4':
                    version = '< 2.4 (no milliseconds)'
                elif self.logfile.datetime_format == 'ctime':
                    version = '>= 2.4 (milliseconds present)'
                elif self.logfile.datetime_format == "iso8601-utc" or \
                     self.logfile.datetime_format == "iso8601-local":
                    version = '>= 2.6 (iso8601 format)'

            print "    version: %s" % version,
            print

            # now run all sections
            for section in self.sections:
                if section.active:
                    print
                    print section.name.upper()
                    section.run()