Python backend.AccountBroker类代码示例

def print_info(db_type, db_file, swift_dir='/etc/swift'):
    if db_type not in ('account', 'container'):
        print "Unrecognized DB type: internal error"
        raise InfoSystemExit()
    if not os.path.exists(db_file) or not db_file.endswith('.db'):
        print "DB file doesn't exist"
        raise InfoSystemExit()
    if not db_file.startswith(('/', './')):
        db_file = './' + db_file  # don't break if the bare db file is given
    if db_type == 'account':
        broker = AccountBroker(db_file)
        datadir = ABDATADIR
    else:
        broker = ContainerBroker(db_file)
        datadir = CBDATADIR
    try:
        info = broker.get_info()
    except sqlite3.OperationalError as err:
        if 'no such table' in str(err):
            print "Does not appear to be a DB of type \"%s\": %s" % (
                db_type, db_file)
            raise InfoSystemExit()
        raise
    account = info['account']
    container = info['container'] if db_type == 'container' else None
    print_db_info_metadata(db_type, info, broker.metadata)
    try:
        ring = Ring(swift_dir, ring_name=db_type)
    except Exception:
        ring = None
    else:
        print_ring_locations(ring, datadir, account, container)

    def account_audit(self, path):
        """
        Audits the given account path

        :param path: the path to an account db
        """
        start_time = time.time()
        try:
            broker = AccountBroker(path)
            if not broker.is_deleted():
                self.validate_per_policy_counts(broker)
                self.logger.increment('passes')
                self.account_passes += 1
                self.logger.debug(_('Audit passed for %s'), broker)
        except InvalidAccountInfo as e:
            self.logger.increment('failures')
            self.account_failures += 1
            self.logger.error(
                _('Audit Failed for %(path)s: %(err)s'),
                {'path': path, 'err': str(e)})
        except (Exception, Timeout):
            self.logger.increment('failures')
            self.account_failures += 1
            self.logger.exception(_('ERROR Could not get account info %s'),
                                  path)
        self.logger.timing_since('timing', start_time)

def print_info(db_type, db_file, swift_dir="/etc/swift"):
    if db_type not in ("account", "container"):
        print "Unrecognized DB type: internal error"
        raise InfoSystemExit()
    if not os.path.exists(db_file) or not db_file.endswith(".db"):
        print "DB file doesn't exist"
        raise InfoSystemExit()
    if not db_file.startswith(("/", "./")):
        db_file = "./" + db_file  # don't break if the bare db file is given
    if db_type == "account":
        broker = AccountBroker(db_file)
        datadir = ABDATADIR
    else:
        broker = ContainerBroker(db_file)
        datadir = CBDATADIR
    try:
        info = broker.get_info()
    except sqlite3.OperationalError as err:
        if "no such table" in str(err):
            print 'Does not appear to be a DB of type "%s": %s' % (db_type, db_file)
            raise InfoSystemExit()
        raise
    account = info["account"]
    container = info["container"] if db_type == "container" else None
    print_db_info_metadata(db_type, info, broker.metadata)
    try:
        ring = Ring(swift_dir, ring_name=db_type)
    except Exception:
        ring = None
    else:
        print_ring_locations(ring, datadir, account, container)

 def tearDown(self):
     AccountBroker.create_account_stat_table = \
         self._imported_create_account_stat_table
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(normalize_timestamp('1'))
     with broker.get() as conn:
         conn.execute('SELECT metadata FROM account_stat')

 def tearDown(self):
     AccountBroker.create_container_table = \
         self._imported_create_container_table
     AccountBroker._initialize = self._imported_initialize
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(Timestamp('1').internal)
     with broker.get() as conn:
         conn.execute('SELECT storage_policy_index FROM container')

 def test_account_stat_get_data(self):
     stat = db_stats_collector.AccountStatsCollector(self.conf)
     account_db = AccountBroker("%s/acc.db" % self.accounts,
                                     account='test_acc')
     account_db.initialize()
     account_db.put_container('test_container', time.time(),
                                   None, 10, 1000)
     info = stat.get_data("%s/acc.db" % self.accounts)
     self.assertEquals('''"test_acc",1,10,1000\n''', info)

 def test_empty(self):
     # Test AccountBroker.empty
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(normalize_timestamp('1'))
     self.assert_(broker.empty())
     broker.put_container('o', normalize_timestamp(time()), 0, 0, 0)
     self.assert_(not broker.empty())
     sleep(.00001)
     broker.put_container('o', 0, normalize_timestamp(time()), 0, 0)
     self.assert_(broker.empty())

    def test_policy_stats_tracking(self):
        ts = (Timestamp(t).internal for t in itertools.count(int(time())))
        broker = AccountBroker(':memory:', account='a')
        broker.initialize(ts.next())

        # policy 0
        broker.put_container('con1', ts.next(), 0, 12, 2798641, 0)
        broker.put_container('con1', ts.next(), 0, 13, 8156441, 0)
        # policy 1
        broker.put_container('con2', ts.next(), 0, 7, 5751991, 1)
        broker.put_container('con2', ts.next(), 0, 8, 6085379, 1)

        stats = broker.get_policy_stats()
        self.assertEqual(len(stats), 2)
        self.assertEqual(stats[0]['object_count'], 13)
        self.assertEqual(stats[0]['bytes_used'], 8156441)
        self.assertEqual(stats[1]['object_count'], 8)
        self.assertEqual(stats[1]['bytes_used'], 6085379)

        # Break encapsulation here to make sure that there's only 2 rows in
        # the stats table. It's possible that there could be 4 rows (one per
        # put_container) but that they came out in the right order so that
        # get_policy_stats() collapsed them down to the right number. To prove
        # that's not so, we have to go peek at the broker's internals.
        with broker.get() as conn:
            nrows = conn.execute(
                "SELECT COUNT(*) FROM policy_stat").fetchall()[0][0]
        self.assertEqual(nrows, 2)

    def reap_device(self, device):
        """
        Called once per pass for each device on the server. This will scan the
        accounts directory for the device, looking for partitions this device
        is the primary for, then looking for account databases that are marked
        status=DELETED and still have containers and calling
        :func:`reap_account`. Account databases marked status=DELETED that no
        longer have containers will eventually be permanently removed by the
        reclaim process within the account replicator (see
        :mod:`swift.db_replicator`).

        :param device: The device to look for accounts to be deleted.
        """
        datadir = os.path.join(self.devices, device, DATADIR)
        if not os.path.exists(datadir):
            return
        for partition in os.listdir(datadir):
            partition_path = os.path.join(datadir, partition)
            if not partition.isdigit():
                continue
            nodes = self.get_account_ring().get_part_nodes(int(partition))
            if not os.path.isdir(partition_path):
                continue
            container_shard = None
            for container_shard, node in enumerate(nodes):
                if is_local_device(self.myips, None, node['ip'], None) and \
                        (not self.bind_port or
                         self.bind_port == node['port']) and \
                        (device == node['device']):
                    break
            else:
                continue

            for suffix in os.listdir(partition_path):
                suffix_path = os.path.join(partition_path, suffix)
                if not os.path.isdir(suffix_path):
                    continue
                for hsh in os.listdir(suffix_path):
                    hsh_path = os.path.join(suffix_path, hsh)
                    if not os.path.isdir(hsh_path):
                        continue
                    for fname in sorted(os.listdir(hsh_path), reverse=True):
                        if fname.endswith('.ts'):
                            break
                        elif fname.endswith('.db'):
                            self.start_time = time()
                            broker = \
                                AccountBroker(os.path.join(hsh_path, fname),
                                              logger=self.logger)
                            if broker.is_status_deleted() and \
                                    not broker.empty():
                                self.reap_account(
                                    broker, partition, nodes,
                                    container_shard=container_shard)

 def test_empty(self):
     # Test AccountBroker.empty
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(Timestamp('1').internal)
     self.assert_(broker.empty())
     broker.put_container('o', Timestamp(time()).internal, 0, 0, 0,
                          POLICIES.default.idx)
     self.assert_(not broker.empty())
     sleep(.00001)
     broker.put_container('o', 0, Timestamp(time()).internal, 0, 0,
                          POLICIES.default.idx)
     self.assert_(broker.empty())

 def test_exception(self):
     # Test AccountBroker throwing a conn away after exception
     first_conn = None
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(normalize_timestamp('1'))
     with broker.get() as conn:
         first_conn = conn
     try:
         with broker.get() as conn:
             self.assertEqual(first_conn, conn)
             raise Exception('OMG')
     except Exception:
         pass
     self.assert_(broker.conn is None)

 def setUp(self):
     self._imported_create_account_stat_table = \
         AccountBroker.create_account_stat_table
     AccountBroker.create_account_stat_table = \
         premetadata_create_account_stat_table
     broker = AccountBroker(':memory:', account='a')
     broker.initialize(normalize_timestamp('1'))
     exc = None
     with broker.get() as conn:
         try:
             conn.execute('SELECT metadata FROM account_stat')
         except BaseException as err:
             exc = err
     self.assert_('no such column: metadata' in str(exc))

    def get_data(self, db_path):
        """
        Data for generated csv has the following columns:
        Account Hash, Container Count, Object Count, Bytes Used

        :raises sqlite3.Error: does not catch errors connecting to db
        """
        line_data = None
        broker = AccountBroker(db_path)
        if not broker.is_deleted():
            info = broker.get_info()
            line_data = '"%s",%d,%d,%d\n' % (info['account'],
                                             info['container_count'],
                                             info['object_count'],
                                             info['bytes_used'])
        return line_data

 def test_creation(self):
     # Test AccountBroker.__init__
     broker = AccountBroker(':memory:', account='a')
     self.assertEqual(broker.db_file, ':memory:')
     got_exc = False
     try:
         with broker.get() as conn:
             pass
     except Exception:
         got_exc = True
     self.assert_(got_exc)
     broker.initialize(normalize_timestamp('1'))
     with broker.get() as conn:
         curs = conn.cursor()
         curs.execute('SELECT 1')
         self.assertEqual(curs.fetchall()[0][0], 1)

    def account_audit(self, path):
        """
        Audits the given account path

        :param path: the path to an account db
        """
        start_time = time.time()
        try:
            broker = AccountBroker(path)
            if not broker.is_deleted():
                broker.get_info()
                self.logger.increment('passes')
                self.account_passes += 1
                self.logger.debug(_('Audit passed for %s') % broker)
        except (Exception, Timeout):
            self.logger.increment('failures')
            self.account_failures += 1
            self.logger.exception(_('ERROR Could not get account info %s'),
                                  path)
        self.logger.timing_since('timing', start_time)