Python Backups.upsert方法代码示例

# 需要导入模块: from bakthat.models import Backups [as 别名]
# 或者: from bakthat.models.Backups import upsert [as 别名]
def upgrade_from_shelve():
    if os.path.isfile(os.path.expanduser("~/.bakthat.db")):
        glacier_backend = GlacierBackend()
        glacier_backend.upgrade_from_shelve()

        s3_backend = S3Backend()

        regex_key = re.compile(r"(?P<backup_name>.+)\.(?P<date_component>\d{14})\.tgz(?P<is_enc>\.enc)?")

        # old regex for backward compatibility (for files without dot before the date component).
        old_regex_key = re.compile(r"(?P<backup_name>.+)(?P<date_component>\d{14})\.tgz(?P<is_enc>\.enc)?")

        for generator, backend in [(s3_backend.ls(), "s3"), ([ivt.filename for ivt in Inventory.select()], "glacier")]:
            for key in generator:
                match = regex_key.match(key)
                # Backward compatibility
                if not match:
                    match = old_regex_key.match(key)
                if match:
                    filename = match.group("backup_name")
                    is_enc = bool(match.group("is_enc"))
                    backup_date = int(datetime.strptime(match.group("date_component"), "%Y%m%d%H%M%S").strftime("%s"))
                else:
                    filename = key
                    is_enc = False
                    backup_date = 0
                if backend == "s3":
                    backend_hash = hashlib.sha512(s3_backend.conf.get("access_key") + \
                                        s3_backend.conf.get(s3_backend.container_key)).hexdigest()
                elif backend == "glacier":
                    backend_hash = hashlib.sha512(glacier_backend.conf.get("access_key") + \
                                        glacier_backend.conf.get(glacier_backend.container_key)).hexdigest()
                new_backup = dict(backend=backend,
                                  is_deleted=0,
                                  backup_date=backup_date,
                                  tags="",
                                  stored_filename=key,
                                  filename=filename,
                                  last_updated=int(datetime.utcnow().strftime("%s")),
                                  metadata=dict(is_enc=is_enc),
                                  size=0,
                                  backend_hash=backend_hash)
                try:
                    Backups.upsert(**new_backup)
                except Exception, exc:
                    print exc
        os.remove(os.path.expanduser("~/.bakthat.db"))

# 需要导入模块: from bakthat.models import Backups [as 别名]
# 或者: from bakthat.models.Backups import upsert [as 别名]
    def sync(self):
        """Draft for implementing bakthat clients (hosts) backups data synchronization.

        Synchronize Bakthat sqlite database via a HTTP POST request.

        Backups are never really deleted from sqlite database, we just update the is_deleted key.

        It sends the last server sync timestamp along with data updated since last sync.
        Then the server return backups that have been updated on the server since last sync.

        On both sides, backups are either created if they don't exists or updated if the incoming version is newer.
        """
        log.debug("Start syncing")

        self.register()

        last_sync_ts = Config.get_key("sync_ts", 0)
        to_insert_in_mongo = [b._data for b in Backups.search(last_updated_gt=last_sync_ts)]
        data = dict(sync_ts=last_sync_ts, to_insert_in_mongo=to_insert_in_mongo)
        r_kwargs = self.request_kwargs.copy()
        log.debug("Initial payload: {0}".format(data))
        r_kwargs.update({"data": json.dumps(data)})
        r = requests.post(self.get_resource("backups/sync/status"), **r_kwargs)
        if r.status_code != 200:
            log.error("An error occured during sync: {0}".format(r.text))
            return

        log.debug("Sync result: {0}".format(r.json()))
        to_insert_in_bakthat = r.json().get("to_insert_in_bakthat")
        sync_ts = r.json().get("sync_ts")
        for newbackup in to_insert_in_bakthat:
            sqlite_backup = Backups.match_filename(newbackup["stored_filename"], newbackup["backend"])
            if sqlite_backup and newbackup["last_updated"] > sqlite_backup.last_updated:
                    log.debug("Upsert {0}".format(newbackup))
                    Backups.upsert(**newbackup)
            elif not sqlite_backup:
                log.debug("Create backup {0}".format(newbackup))
                Backups.create(**newbackup)

        Config.set_key("sync_ts", sync_ts)

        log.debug("Sync succcesful")

# 需要导入模块: from bakthat.models import Backups [as 别名]
# 或者: from bakthat.models.Backups import upsert [as 别名]
    def sync(self):
        """Draft for implementing bakthat clients (hosts) backups data synchronization.

        Synchronize Bakthat sqlite database via a HTTP POST request.

        Backups are never really deleted from sqlite database, we just update the is_deleted key.

        It sends the last server sync timestamp along with data updated since last sync.
        Then the server return backups that have been updated on the server since last sync.

        Both side (bakthat and the sync server) make upserts of the latest data avaible:
        - if it doesn't exist yet, it will be created.
        - if it has been modified (e.g deleted, since it's the only action we can take) we update it.
        """
        log.debug("Start syncing")

        self.register()

        last_sync_ts = Config.get_key("sync_ts", 0)
        to_insert_in_mongo = [b._data for b in Backups.search(last_updated_gt=last_sync_ts)]
        data = dict(sync_ts=last_sync_ts, to_insert_in_mongo=to_insert_in_mongo)
        r_kwargs = self.request_kwargs.copy()
        log.debug("Initial payload: {0}".format(data))
        r_kwargs.update({"data": json.dumps(data)})
        r = requests.post(self.get_resource("backups/sync/status"), **r_kwargs)
        if r.status_code != 200:
            log.error("An error occured during sync: {0}".format(r.text))
            return

        log.debug("Sync result: {0}".format(r.json()))
        to_insert_in_bakthat = r.json().get("to_insert_in_bakthat")
        sync_ts = r.json().get("sync_ts")
        for newbackup in to_insert_in_bakthat:
            log.debug("Upsert {0}".format(newbackup))
            Backups.upsert(**newbackup)

        Config.set_key("sync_ts", sync_ts)

        log.debug("Sync succcesful")