Python pygtail.Pygtail类代码示例

 def test_copytruncate_on_smaller(self):
     self.test_readlines()
     self.copytruncate()
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=True)
     self.assertEqual(pygtail.read(), new_lines)

    def __init__(self, filepath, groupby, groupname=None):
        self.groupmatch = re.compile(groupby)
        # write an offset file so that we start somewhat at the end of the file

        # either filepath is a path or a syslogd url
        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(filepath)
        if scheme == 'syslog':
            host, port = netloc.split(':')
            self.fin = QueueFile()
            self.server = SocketServer.UDPServer((host, int(port)), SyslogUDPHandler)
            self.server.queue = self.fin

            th = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.5))
            th.setDaemon(True)
            th.start()
        else:
            # Create a temporal file with offset info
            self.offsetpath = "/tmp/" + str(uuid.uuid4())
            try:
                inode = os.stat(filepath).st_ino
                offset = os.path.getsize(filepath) - 1024
            except OSError:
                pass
            else:
                if offset > 0:
                    foffset = open(self.offsetpath, "w")
                    foffset.write("%s\n%s" % (inode, offset))
                    foffset.close()

            self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)

        # List of matchings
        self.match_definitions = []
        # Regex group name for grouping
        self.groupbygroup = groupname

 def test_subsequent_read_with_new_data(self):
     pygtail = Pygtail(self.logfile.name)
     self.assertEqual(pygtail.read(), self.test_str)
     new_lines = "4\n5\n"
     self.append(new_lines)
     new_pygtail = Pygtail(self.logfile.name)
     self.assertEqual(new_pygtail.read(), new_lines)

 def _test_copytruncate_larger(self, onoff):
     self.test_readlines()
     self.copytruncate()
     self.append(self.test_str)
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=onoff)
     self.assertEqual(pygtail.read(), new_lines)

 def test_copytruncate_off_smaller(self):
     self.test_readlines()
     self.copytruncate()
     new_lines = "4\n5\n"
     self.append(new_lines)
     pygtail = Pygtail(self.logfile.name, copytruncate=False)
     self.assertEqual(pygtail.read(), None)
     self.assertRegexpMatches(sys.stderr.getvalue(), r".*?\bWARN\b.*?\bshrank\b.*")

 def test_logrotate_without_close(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     # note it doesn't matter what filename the file gets rotated to
     os.rename(self.logfile.name, "%s.somethingodd" % self.logfile.name)
     self.append(new_lines[1])
     self.assertEqual(pygtail.read(), ''.join(new_lines))

 def test_logrotate_with_delay_compress(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     os.rename(self.logfile.name, "%s.1" % self.logfile.name)
     self.append(new_lines[1])
     pygtail = Pygtail(self.logfile.name)
     self.assertEqual(pygtail.read(), ''.join(new_lines))

 def test_logrotate(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name)
     pygtail.read()
     self.append(new_lines[0])
     os.rename(self.logfile.name, "%s.1" % self.logfile.name)
     self.append(new_lines[1])
     pygtail = Pygtail(self.logfile.name)
     self.assertEquals(pygtail.read(), "".join(new_lines))

 def test_copytruncate_off_smaller_without_close(self):
     new_lines = ["4\n5\n", "6\n7\n"]
     pygtail = Pygtail(self.logfile.name, copytruncate=True)
     pygtail.read()
     self.append(new_lines[0])
     read1 = pygtail.read()
     self.copytruncate()
     self.append(new_lines[1])
     read2 = pygtail.read()
     self.assertEqual([read1,read2], new_lines)

    def test_copytruncate_off_smaller(self):
        self.test_readlines()
        self.copytruncate()
        new_lines = "4\n5\n"
        self.append(new_lines)

        sys.stderr = captured = io.BytesIO() if PY2 else io.StringIO()
        pygtail = Pygtail(self.logfile.name, copytruncate=False)
        captured_value = captured.getvalue()
        sys.stderr = sys.__stderr__

        assert_class = self.assertRegex if sys.version_info >= (3, 1) else self.assertRegexpMatches
        assert_class(captured_value, r".*?\bWARN\b.*?\bshrank\b.*")
        self.assertEqual(pygtail.read(), None)

    def __init__(self, filepath, groupby):

        self.groupmatch = re.compile(groupby)

        # write an offset file so that we start somewhat at the end of the file
       
        self.offsetpath = "/tmp/" + str(uuid.uuid4())
        #print self.offsetpath
        try:
            inode = os.stat(filepath).st_ino
            offset = os.path.getsize(filepath) - 1024
            #print inode
            #print offset
        except OSError:
            pass
        else:
            if offset > 0:
                #print 'write offset'
                foffset = open(self.offsetpath, "w")
                foffset.write ("%s\n%s" % (inode, offset))
                foffset.close()

        self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)
        #self.fin.readlines()

        self.match_definitions = []

class FileFollower():
    '''
    Use pygtail to keep track of EOF and rotated files, catch exceptions to
    make things more seamless
    '''
    def __init__(self, path):
        self.path = path
        self.pygtail = None
        self.last_inode = 0

    def next(self):
        line = ''
        curr_inode = 0
        if self.pygtail is None:
            try:
                # remove last offset file if the log file is different
                # PygTail's inode detection doesn't work in certain cases
                curr_inode = os.stat(self.path).st_ino
                if self.last_inode != curr_inode:
                    os.unlink(self.path + '.offset')
                    self.last_inode = curr_inode
                    log.debug('deleted offset file, inode difference')
            except Exception as e:
                log.info('inode checking failed (not terminal): %s' % e)
            self.pygtail = Pygtail(self.path)
        try:
            line = self.pygtail.next()
        except StopIteration as si:
            # Need to get a new instance of pygtail after this incase the inode
            # has changed
            self.pygtail = None
            return False
        return line

    def test_logrotate_without_delay_compress(self):
        new_lines = ["4\n5\n", "6\n7\n"]
        pygtail = Pygtail(self.logfile.name)
        pygtail.read()
        self.append(new_lines[0])

        # put content to gzip file
        gzip_handle = gzip.open("%s.1.gz" % self.logfile.name, 'wb')
        with open(self.logfile.name, 'rb') as logfile:
            gzip_handle.write(logfile.read())
        gzip_handle.close()

        with open(self.logfile.name, 'w'):
            # truncate file
            pass

        self.append(new_lines[1])
        pygtail = Pygtail(self.logfile.name)
        self.assertEqual(pygtail.read(), ''.join(new_lines))

class GroupingTail (object):

    def __init__(self, filepath, groupby):

        self.groupmatch = re.compile(groupby)

        # write an offset file so that we start somewhat at the end of the file
       
        self.offsetpath = "/tmp/" + str(uuid.uuid4())
        #print self.offsetpath
        try:
            inode = os.stat(filepath).st_ino
            offset = os.path.getsize(filepath) - 1024
            #print inode
            #print offset
        except OSError:
            pass
        else:
            if offset > 0:
                #print 'write offset'
                foffset = open(self.offsetpath, "w")
                foffset.write ("%s\n%s" % (inode, offset))
                foffset.close()

        self.fin = Pygtail(filepath, offset_file=self.offsetpath, copytruncate=True)
        #self.fin.readlines()

        self.match_definitions = []

    def update(self):
        for line in self.fin.readlines():
            #print 'line: %s' % line
            mo = self.groupmatch.match(line)
            if mo is not None and mo.groups():
                groupname = mo.groups()[0].replace(".", "_").replace("-", "_")
                for match in self.match_definitions:
                    instrument = match["instrument"]
                    instrument.write(groupname, line)

    def add_match(self,  instance_name, valuetype, instrument):
        self.match_definitions.append(dict(
            instance_name=instance_name,
            valuetype=valuetype,
            instrument=instrument
        ))

    def read_metrics(self):
        for match in self.match_definitions:
            instance_name = match["instance_name"]
            instrument = match["instrument"]
            valuetype = match["valuetype"]

            for groupname, value in instrument.read():
                metric_name = "%s.%s" % (groupname, instance_name)
                yield (metric_name, valuetype, value)

    def download_file(self, url):
        try:
            # Check that the log file exists.
            assert os.path.isfile(LOG_FILE)

            # Check that the offset directory exists.
            assert os.path.isdir(OFFSET_DIR)

            # Check that the offset directory is writeable.
            assert os.access(OFFSET_DIR, os.W_OK)

            # Check that the log file is writeable.
            assert os.access(LOG_FILE, os.R_OK)
        except AssertionError:
            sys.stderr.write('Error: One or more preconditions failed.\n')
            # Exit 13, don't restart tcollector.
            sys.exit(13)

        # If the offset file exists, it had better not be empty.
        if os.path.isfile(OFFSET_FILE):
            try:
                assert os.path.getsize(OFFSET_FILE) > 0
            except AssertionError:
                os.remove(OFFSET_FILE)

        # We're not using paranoid mode in Pygtail for performance.
        maillog = Pygtail(LOG_FILE, offset_file=OFFSET_FILE)

        for line in maillog:
            try:
                process_line(line)
            except LineProcessingError:
                pass

            if STOP:
               # Force the offset file to update before we shutdown.
               # If pygtail is not paranoid=True, this is necessary to ensure
               # that the offset gets written after SIGTERM.
               maillog._update_offset_file()
               break