Python WalFileInfo.from_xlogdb_line方法代码示例

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
 def test_recover_xlog(self, rsync_pg_mock, tmpdir):
     """
     Test the recovery of the xlogs of a backup
     :param rsync_pg_mock: Mock rsync object for the purpose if this test
     """
     # Build basic folders/files structure
     dest = tmpdir.mkdir('destination')
     wals = tmpdir.mkdir('wals')
     xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
     xlog_file = xlog_dir.join('000000000000000000000002')
     xlog_file.write('dummy content')
     server = testing_helpers.build_real_server(
         main_conf={'wals_directory': wals.strpath})
     # build executor
     executor = RecoveryExecutor(server.backup_manager)
     required_wals = (WalFileInfo.from_xlogdb_line(
         '000000000000000000000002\t42\t43\tNone\n'),)
     executor.xlog_copy(required_wals, dest.strpath, None)
     # check for a correct invocation of rsync using local paths
     rsync_pg_mock.from_file_list.assert_called_once(
         ['000000000000000000000002'],
         xlog_dir.strpath,
         dest.strpath)
     # reset mock calls
     rsync_pg_mock.reset_mock()
     required_wals = (WalFileInfo.from_xlogdb_line(
         '000000000000000000000002\t42\t43\tNone\n'),)
     executor.backup_manager.compression_manager = Mock()
     executor.xlog_copy(required_wals, dest.strpath, 'remote_command')
     # check for the invocation of rsync on a remote call
     rsync_pg_mock.assert_called_once(network_compression=False,
                                      bwlimit=None,
                                      ssh='remote_command')

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
 def test_get_wal_info(self, get_wal_mock, tmpdir):
     """
     Basic test for get_wal_info method
     Test the wals per second and total time in seconds values.
     :return:
     """
     # Build a test server with a test path
     server = build_real_server(global_conf={
         'barman_home': tmpdir.strpath
     })
     # Mock method get_wal_until_next_backup for returning a list of
     # 3 fake WAL. the first one is the start and stop WAL of the backup
     wal_list = [
         WalFileInfo.from_xlogdb_line(
             "000000010000000000000002\t16777216\t1434450086.53\tNone\n"),
         WalFileInfo.from_xlogdb_line(
             "000000010000000000000003\t16777216\t1434450087.54\tNone\n"),
         WalFileInfo.from_xlogdb_line(
             "000000010000000000000004\t16777216\t1434450088.55\tNone\n")]
     get_wal_mock.return_value = wal_list
     backup_info = build_test_backup_info(
         server=server,
         begin_wal=wal_list[0].name,
         end_wal=wal_list[0].name)
     backup_info.save()
     # Evaluate total time in seconds:
     # last_wal_timestamp - first_wal_timestamp
     wal_total_seconds = wal_list[-1].time - wal_list[0].time
     # Evaluate the wals_per_second value:
     # wals_in_backup + wals_until_next_backup / total_time_in_seconds
     wals_per_second = len(wal_list) / wal_total_seconds
     wal_info = server.get_wal_info(backup_info)
     assert wal_info
     assert wal_info['wal_total_seconds'] == wal_total_seconds
     assert wal_info['wals_per_second'] == wals_per_second

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
 def get_required_xlog_files(self, backup, target_tli=None, target_time=None,
                             target_xid=None):
     """
     Get the xlog files required for a recovery
     """
     begin = backup.begin_wal
     end = backup.end_wal
     # If timeline isn't specified, assume it is the same timeline
     # of the backup
     if not target_tli:
         target_tli, _, _ = xlog.decode_segment_name(end)
     with self.xlogdb() as fxlogdb:
         for line in fxlogdb:
             wal_info = WalFileInfo.from_xlogdb_line(line)
             # Handle .history files: add all of them to the output,
             # regardless of their age
             if xlog.is_history_file(wal_info.name):
                 yield wal_info
                 continue
             if wal_info.name < begin:
                 continue
             tli, _, _ = xlog.decode_segment_name(wal_info.name)
             if tli > target_tli:
                 continue
             yield wal_info
             if wal_info.name > end:
                 end = wal_info.name
                 if target_time and target_time < wal_info.time:
                     break
         # return all the remaining history files
         for line in fxlogdb:
             wal_info = WalFileInfo.from_xlogdb_line(line)
             if xlog.is_history_file(wal_info.name):
                 yield wal_info

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
        """
        Test the recovery of the xlogs of a backup
        :param rsync_pg_mock: Mock rsync object for the purpose if this test
        """
        # Build basic folders/files structure
        dest = tmpdir.mkdir("destination")
        wals = tmpdir.mkdir("wals")
        # Create 3 WAL files with different compressions
        xlog_dir = wals.mkdir(xlog.hash_dir("000000000000000000000002"))
        xlog_plain = xlog_dir.join("000000000000000000000001")
        xlog_gz = xlog_dir.join("000000000000000000000002")
        xlog_bz2 = xlog_dir.join("000000000000000000000003")
        xlog_plain.write("dummy content")
        xlog_gz.write("dummy content gz")
        xlog_bz2.write("dummy content bz2")
        server = testing_helpers.build_real_server(main_conf={"wals_directory": wals.strpath})
        # Prepare compressors mock
        c = {"gzip": mock.Mock(name="gzip"), "bzip2": mock.Mock(name="bzip2")}
        cm_mock.return_value.get_compressor = lambda compression=None, path=None: c[compression]
        # touch destination files to avoid errors on cleanup
        c["gzip"].decompress.side_effect = lambda src, dst: open(dst, "w")
        c["bzip2"].decompress.side_effect = lambda src, dst: open(dst, "w")
        # Build executor
        executor = RecoveryExecutor(server.backup_manager)

        # Test: local copy
        required_wals = (
            WalFileInfo.from_xlogdb_line("000000000000000000000001\t42\t43\tNone\n"),
            WalFileInfo.from_xlogdb_line("000000000000000000000002\t42\t43\tgzip\n"),
            WalFileInfo.from_xlogdb_line("000000000000000000000003\t42\t43\tbzip2\n"),
        )
        executor._xlog_copy(required_wals, dest.strpath, None)
        # Check for a correct invocation of rsync using local paths
        rsync_pg_mock.assert_called_once_with(network_compression=False, bwlimit=None, path=None, ssh=None)
        assert not rsync_pg_mock.return_value.from_file_list.called
        c["gzip"].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
        c["bzip2"].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY)

        # Reset mock calls
        rsync_pg_mock.reset_mock()
        c["gzip"].reset_mock()
        c["bzip2"].reset_mock()

        # Test: remote copy
        executor._xlog_copy(required_wals, dest.strpath, "remote_command")
        # Check for the invocation of rsync on a remote call
        rsync_pg_mock.assert_called_once_with(
            network_compression=False, bwlimit=None, path=mock.ANY, ssh="remote_command"
        )
        rsync_pg_mock.return_value.from_file_list.assert_called_once_with(
            ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003"], mock.ANY, mock.ANY
        )
        c["gzip"].decompress.assert_called_once_with(xlog_gz.strpath, mock.ANY)
        c["bzip2"].decompress.assert_called_once_with(xlog_bz2.strpath, mock.ANY)

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def remove_wal_before_backup(self, backup_info):
        """
        Remove WAL files which have been archived before the start of
        the provided backup.

        If no backup_info is provided delete all available WAL files

        :param BackupInfo|None backup_info: the backup information structure
        :return list: a list of removed WAL files
        """
        removed = []
        with self.server.xlogdb() as fxlogdb:
            xlogdb_new = fxlogdb.name + ".new"
            with open(xlogdb_new, 'w') as fxlogdb_new:
                for line in fxlogdb:
                    wal_info = WalFileInfo.from_xlogdb_line(line)
                    # Keeps the WAL segment if it is a history file or later
                    # than the given backup (the first available)
                    if (xlog.is_history_file(wal_info.name) or
                        (backup_info and wal_info.name >= backup_info.begin_wal)):
                            fxlogdb_new.write(wal_info.to_xlogdb_line())
                            continue
                    else:
                        self.delete_wal(wal_info)
                        removed.append(wal_info.name)
                fxlogdb_new.flush()
                os.fsync(fxlogdb_new.fileno())
            shutil.move(xlogdb_new, fxlogdb.name)
            fsync_dir(os.path.dirname(fxlogdb.name))
        return removed

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def get_wal_until_next_backup(self, backup, include_history=False):
        """
        Get the xlog files between backup and the next

        :param BackupInfo backup: a backup object, the starting point
            to retrieve WALs
        :param bool include_history: option for the inclusion of
            include_history files into the output
        """
        begin = backup.begin_wal
        next_end = None
        if self.get_next_backup(backup.backup_id):
            next_end = self.get_next_backup(backup.backup_id).end_wal
        backup_tli, _, _ = xlog.decode_segment_name(begin)

        with self.xlogdb() as fxlogdb:
            for line in fxlogdb:
                wal_info = WalFileInfo.from_xlogdb_line(line)
                # Handle .history files: add all of them to the output,
                # regardless of their age, if requested (the 'include_history'
                # parameter is True)
                if xlog.is_history_file(wal_info.name):
                    if include_history:
                        yield wal_info
                    continue
                if wal_info.name < begin:
                    continue
                tli, _, _ = xlog.decode_segment_name(wal_info.name)
                if tli > backup_tli:
                    continue
                if not xlog.is_wal_file(wal_info.name):
                    continue
                if next_end and wal_info.name > next_end:
                    break
                yield wal_info

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
def missing_wals(server, args):
    warn = args.warning
    crit = args.critical

    from barman.xlog import is_wal_file
    from barman.infofile import WalFileInfo

    wals_directory = server.config.wals_directory

    missing_wals = 0
    with server.xlogdb() as fxlogdb:
        for line in fxlogdb:
            #name, size, time, compression = server.xlogdb_parse_line(line)
            wal_info = WalFileInfo.from_xlogdb_line(line)
            name = wal_info.name

            directory = name[0:16]

            if is_wal_file(name):
                file_path = os.path.join(wals_directory, directory, name)
                if not os.path.exists(file_path):
                    missing_wals = missing_wals + 1

    exit_check(missing_wals, warn, crit,
               "There are %d missing wals for the last backup." % missing_wals,
               perfdata_key="missing", perfdata_min=0)

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
def last_wal_age(server, args):
    warn = args.warning
    crit = args.critical

    from barman.infofile import WalFileInfo

    with server.xlogdb() as fxlogdb:
        line = None
        for line in fxlogdb:
            pass

        if line is None:
            critical("No WAL received yet.")

    #name, size, time, compression = server.xlogdb_parse_line(line)
    wal_info = WalFileInfo.from_xlogdb_line(line)

    time = datetime.fromtimestamp(wal_info.time)
    now = datetime.now()

    age = now - time
    minutes = age.seconds / 60
    minutes = minutes + age.days * 60 * 24

    exit_check(minutes, warn, crit, "Last WAL is %s minutes old." % minutes,
               perfdata_key="minutes", perfdata_min=0)

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def remove_wal_before_backup(self, backup_info, timelines_to_protect=None):
        """
        Remove WAL files which have been archived before the start of
        the provided backup.

        If no backup_info is provided delete all available WAL files

        If timelines_to_protect list is passed, never remove a wal in one of
        these timelines.

        :param BackupInfo|None backup_info: the backup information structure
        :param set timelines_to_protect: optional list of timelines
            to protect
        :return list: a list of removed WAL files
        """
        removed = []
        with self.server.xlogdb() as fxlogdb:
            xlogdb_new = fxlogdb.name + ".new"
            with open(xlogdb_new, 'w') as fxlogdb_new:
                for line in fxlogdb:
                    wal_info = WalFileInfo.from_xlogdb_line(line)
                    if not xlog.is_any_xlog_file(wal_info.name):
                        output.error(
                            "invalid xlog segment name %r\n"
                            "HINT: Please run \"barman rebuild-xlogdb %s\" "
                            "to solve this issue",
                            wal_info.name, self.config.name)
                        continue

                    # Keeps the WAL segment if it is a history file
                    keep = xlog.is_history_file(wal_info.name)

                    # Keeps the WAL segment if its timeline is in
                    # `timelines_to_protect`
                    if timelines_to_protect:
                        tli, _, _ = xlog.decode_segment_name(wal_info.name)
                        keep |= tli in timelines_to_protect

                    # Keeps the WAL segment if it is a newer
                    # than the given backup (the first available)
                    if backup_info:
                        keep |= wal_info.name >= backup_info.begin_wal

                    # If the file has to be kept write it in the new xlogdb
                    # otherwise delete it  and record it in the removed list
                    if keep:
                        fxlogdb_new.write(wal_info.to_xlogdb_line())
                    else:
                        self.delete_wal(wal_info)
                        removed.append(wal_info.name)
                fxlogdb_new.flush()
                os.fsync(fxlogdb_new.fileno())
            shutil.move(xlogdb_new, fxlogdb.name)
            fsync_dir(os.path.dirname(fxlogdb.name))
        return removed

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def test_from_xlogdb_line(self):
        """
        Test the conversion from a string to a WalFileInfo file
        """
        # build a WalFileInfo object
        wfile_info = WalFileInfo()
        wfile_info.name = "000000000000000000000001"
        wfile_info.size = 42
        wfile_info.time = 43
        wfile_info.compression = None
        assert wfile_info.relpath() == ("0000000000000000/000000000000000000000001")

        # mock a server object
        server = mock.Mock(name="server")
        server.config.wals_directory = "/tmp/wals"

        # parse the string
        info_file = wfile_info.from_xlogdb_line("000000000000000000000001\t42\t43\tNone\n")

        assert list(wfile_info.items()) == list(info_file.items())

# 需要导入模块: from barman.infofile import WalFileInfo [as 别名]
# 或者: from barman.infofile.WalFileInfo import from_xlogdb_line [as 别名]
    def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
        """
        Test the recovery of the xlogs of a backup
        :param rsync_pg_mock: Mock rsync object for the purpose if this test
        """
        # Build basic folders/files structure
        dest = tmpdir.mkdir('destination')
        wals = tmpdir.mkdir('wals')
        # Create 3 WAL files with different compressions
        xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
        xlog_plain = xlog_dir.join('000000000000000000000001')
        xlog_gz = xlog_dir.join('000000000000000000000002')
        xlog_bz2 = xlog_dir.join('000000000000000000000003')
        xlog_plain.write('dummy content')
        xlog_gz.write('dummy content gz')
        xlog_bz2.write('dummy content bz2')
        server = testing_helpers.build_real_server(
            main_conf={'wals_directory': wals.strpath})
        # Prepare compressors mock
        c = {
            'gzip': Mock(name='gzip'),
            'bzip2': Mock(name='bzip2'),
        }
        cm_mock.return_value.get_compressor = \
            lambda compression=None, path=None: c[compression]
        # touch destination files to avoid errors on cleanup
        c['gzip'].decompress.side_effect = lambda src, dst: open(dst, 'w')
        c['bzip2'].decompress.side_effect = lambda src, dst: open(dst, 'w')
        # Build executor
        executor = RecoveryExecutor(server.backup_manager)

        # Test: local copy
        required_wals = (
            WalFileInfo.from_xlogdb_line(
                '000000000000000000000001\t42\t43\tNone\n'),
            WalFileInfo.from_xlogdb_line(
                '000000000000000000000002\t42\t43\tgzip\n'),
            WalFileInfo.from_xlogdb_line(
                '000000000000000000000003\t42\t43\tbzip2\n'),
        )
        executor.xlog_copy(required_wals, dest.strpath, None)
        # Check for a correct invocation of rsync using local paths
        rsync_pg_mock.assert_called_once_with(
            network_compression=False,
            bwlimit=None, path=None,
            ssh=None)
        assert not rsync_pg_mock.return_value.from_file_list.called
        c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, ANY)
        c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, ANY)

        # Reset mock calls
        rsync_pg_mock.reset_mock()
        c['gzip'].reset_mock()
        c['bzip2'].reset_mock()

        # Test: remote copy
        executor.xlog_copy(required_wals, dest.strpath, 'remote_command')
        # Check for the invocation of rsync on a remote call
        rsync_pg_mock.assert_called_once_with(
            network_compression=False,
            bwlimit=None, path=ANY,
            ssh='remote_command')
        rsync_pg_mock.return_value.from_file_list.assert_called_once_with(
            [
                '000000000000000000000001',
                '000000000000000000000002',
                '000000000000000000000003'],
            ANY,
            ANY)
        c['gzip'].decompress.assert_called_once_with(xlog_gz.strpath, ANY)
        c['bzip2'].decompress.assert_called_once_with(xlog_bz2.strpath, ANY)