Python FileSystem._clean方法代码示例

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
    def _force_clean(cls, recreate=False, initial=False):
        # Database resources
        from pyon.core.bootstrap import get_sys_name, CFG
        from pyon.datastore.datastore_common import DatastoreFactory
        datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
        if initial:
            datastore._init_database(datastore.database)

        dbs = datastore.list_datastores()
        clean_prefix = '%s_' % get_sys_name().lower()
        things_to_clean = [x for x in dbs if x.startswith(clean_prefix)]
        try:
            for thing in things_to_clean:
                datastore.delete_datastore(datastore_name=thing)
                if recreate:
                    datastore.create_datastore(datastore_name=thing)

        finally:
            datastore.close()

        # Broker resources
        from putil.rabbitmq.rabbit_util import RabbitManagementUtil
        rabbit_util = RabbitManagementUtil(CFG, sysname=bootstrap.get_sys_name())
        deleted_exchanges, deleted_queues = rabbit_util.clean_by_sysname()
        log.info("Deleted %s exchanges, %s queues" % (len(deleted_exchanges), len(deleted_queues)))

        # File system
        from pyon.util.file_sys import FileSystem
        FileSystem._clean(CFG)

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
    def _force_clean(cls, recreate=False):
        from pyon.core.bootstrap import get_sys_name, CFG
        from pyon.datastore.couchdb.couchdb_standalone import CouchDataStore
        datastore = CouchDataStore(config=CFG)
        dbs = datastore.list_datastores()
        things_to_clean = filter(lambda x: x.startswith('%s_' % get_sys_name().lower()), dbs)
        try:
            for thing in things_to_clean:
                datastore.delete_datastore(datastore_name=thing)
                if recreate:
                    datastore.create_datastore(datastore_name=thing)

        finally:
            datastore.close()
        FileSystem._clean(CFG)

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
    def _force_clean(cls, recreate=False):
        from pyon.core.bootstrap import get_sys_name, CFG
        from pyon.datastore.datastore_common import DatastoreFactory
        datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE, scope=get_sys_name())
        #datastore = DatastoreFactory.get_datastore(config=CFG, variant=DatastoreFactory.DS_BASE)

        dbs = datastore.list_datastores()
        things_to_clean = filter(lambda x: x.startswith('%s_' % get_sys_name().lower()), dbs)
        try:
            for thing in things_to_clean:
                datastore.delete_datastore(datastore_name=thing)
                if recreate:
                    datastore.create_datastore(datastore_name=thing)

        finally:
            datastore.close()
        FileSystem._clean(CFG)

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        import threading

        threading.current_thread().name = "CC-Main"

        # SIDE EFFECT: The import triggers static initializers: Monkey patching, setting pyon defaults
        import pyon

        from pyon.core import bootstrap, config

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()  # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if "{" in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config

                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(["res/config/pyon_min_boot.yml"])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            print "pycc: config_from_directory=True. Minimal bootstrap configuration:", bootstrap_config
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Delete sysname datastores if option "force_clean" is set
        if opts.force_clean:
            from pyon.datastore import clear_couch_util

            print "pycc: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
            clear_couch_util.clear_couch(
                bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name()
            )
            pyon_config.container.filesystem.force_clean = True

        from pyon.core.interfaces.interfaces import InterfaceAdmin

        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_bootstrap, load config and interfaces into directory
        # Note: this is idempotent and will not alter anything if this is not the first container to run
        if bootstrap_config.system.auto_bootstrap:
            print "pycc: auto_bootstrap=True."
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
#.........这里部分代码省略.........

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
 def _file_sys_clean(self):
     FileSystem._clean(CFG)

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
    def prepare_container():
        """
        Walks through pyon initialization in a deterministic way and initializes Container.
        In particular make sure configuration is loaded in correct order and
        pycc startup arguments are considered.
        """
        # SIDE EFFECT: The import triggers static initializers: Gevent monkey patching, setting pyon defaults
        import pyon

        import threading
        threading.current_thread().name = "CC-Main"

        import logging
        global log
        log = logging.getLogger('pycc')

        from pyon.core import bootstrap, config
        from pyon.util.containers import get_safe, dict_merge

        # Set global testing flag to False. We are running as capability container, because
        # we started through the pycc program.
        bootstrap.testing = False

        # Set sysname if provided in startup argument
        if opts.sysname:
            bootstrap.set_sys_name(opts.sysname)
        # Trigger any initializing default logic in get_sys_name
        bootstrap.get_sys_name()

        command_line_config = kwargs

        # This holds the minimal configuration used to bootstrap pycc and pyon and connect to datastores.
        bootstrap_config = None

        # This holds the new CFG object for pyon. Build it up in proper sequence and conditions.
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml

        # Load config override if provided. Supports variants literal and list of paths
        config_override = None
        if opts.config:
            if '{' in opts.config:
                # Variant 1: Dict of config values
                try:
                    eval_value = ast.literal_eval(opts.config)
                    config_override = eval_value
                except ValueError:
                    raise Exception("Value error in config arg '%s'" % opts.config)
            else:
                # Variant 2: List of paths
                from pyon.util.config import Config
                config_override = Config([opts.config]).data

        # Determine bootstrap_config
        if opts.config_from_directory:
            # Load minimal bootstrap config if option "config_from_directory"
            bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
            config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)
            log.info("config_from_directory=True. Minimal bootstrap configuration: %s", bootstrap_config)
        else:
            # Otherwise: Set to standard set of local config files plus command line overrides
            bootstrap_config = deepcopy(pyon_config)
            config.apply_configuration(bootstrap_config, config_override)
            config.apply_configuration(bootstrap_config, command_line_config)

        # Override sysname from config file or command line
        if not opts.sysname and bootstrap_config.get_safe("system.name", None):
            new_sysname = bootstrap_config.get_safe("system.name")
            bootstrap.set_sys_name(new_sysname)

        # Force_clean - deletes sysname datastores
        if opts.force_clean:
            from pyon.datastore import clear_db_util
            log.info("force_clean=True. DROP DATASTORES for sysname=%s", bootstrap.get_sys_name())
            clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())

        from pyon.core.interfaces.interfaces import InterfaceAdmin
        iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

        # If auto_store_interfaces: ensure that all datastores exist and directory is prepared, with config
        # WARNING: If multiple containers start concurrently, this may fail
        if get_safe(bootstrap_config, "bootstrap.auto_store_interfaces") is True:
            log.debug("auto_store_interfaces=True.")
            stored_config = deepcopy(pyon_config)
            config.apply_configuration(stored_config, config_override)
            config.apply_configuration(stored_config, command_line_config)
            iadm.create_core_datastores()
            iadm.store_config(stored_config)

        # Determine the final pyon_config:
        # - Start from standard config already set (pyon.yml + local YML files)
        # - Optionally load config from directory
        if opts.config_from_directory:
            config.apply_remote_config(bootstrap_cfg=bootstrap_config, system_cfg=pyon_config)
        # - Apply container profile specific config
        config.apply_profile_configuration(pyon_config, bootstrap_config)
        # - Reapply pyon.local.yml here again for good measure
        config.apply_local_configuration(pyon_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
        # - Last apply any separate command line config overrides
#.........这里部分代码省略.........

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
def main():
    '''
    Store configuration and interfaces into the datastore
    How to run this from command line:
        bin/store_interfaces  -s system name [ -of filename | -sf filename | -fc true|false]
        -of Load object definition file
        -sf Load service definition file
        -fc Force clean the database

     Example:
        Load all object and service definitions
        bin/python bin/store_interfaces  -s mysysname

        Load all object and service definitions with force clean the database
        bin/python bin/store_interfaces  -s mysysname -fc

        Load object definition from a file
        bin/python bin/store_interfaces  -s mysysname -of obj/data/core/org.yml

        Load service definition from a file
        bin/python bin/store_interfaces  -s mysysname -sf obj/services/core/datastore_service.yml
    '''

    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, help='Additional config files to load or dict config content.', default=[])
    parser.add_argument('-fc', '--force_clean', action='store_true', help='Force clean.')
    parser.add_argument("-of", "--object", dest="fobject", help="Load object definition from a file")
    parser.add_argument("-s", "--sysname", dest="sysname", help="System name")
    parser.add_argument("-sf", "--service", dest="fservice", help="Load service definition from a file")

    options, extra = parser.parse_known_args()
    args, command_line_config = parse_args(extra)

    print "store_interfaces: Storing SciON config and interfaces, with options:" , str(options)

    # -------------------------------------------------------------------------
    # Store config and interfaces

    # Set global testing flag to False. We are running as standalone script. This is NO TEST.
    bootstrap.testing = False

    # Set sysname if provided in startup argument
    if options.sysname:
        bootstrap.set_sys_name(options.sysname)

    # Load config override if provided. Supports variants literal and list of paths
    config_override = None
    if options.config:
        if '{' in options.config:
            # Variant 1: Dict of config values
            try:
                eval_value = ast.literal_eval(options.config)
                config_override = eval_value
            except ValueError:
                raise Exception("Value error in config arg '%s'" % options.config)
        else:
            # Variant 2: List of paths
            from pyon.util.config import Config
            config_override = Config([options.config]).data

    # bootstrap_config - Used for running this store_interfaces script
    bootstrap_config = config.read_local_configuration(['res/config/pyon_min_boot.yml'])
    config.apply_local_configuration(bootstrap_config, pyon.DEFAULT_LOCAL_CONFIG_PATHS)
    if config_override:
        config.apply_configuration(bootstrap_config, config_override)
    config.apply_configuration(bootstrap_config, command_line_config)

    # Override sysname from config file or command line
    if not options.sysname and bootstrap_config.get_safe("system.name", None):
        new_sysname = bootstrap_config.get_safe("system.name")
        bootstrap.set_sys_name(new_sysname)

    # Delete sysname datastores if option "force_clean" is set
    if options.force_clean:
        from pyon.datastore import clear_db_util
        from pyon.util.file_sys import FileSystem
        print "store_interfaces: force_clean=True. DROP DATASTORES for sysname=%s" % bootstrap.get_sys_name()
        pyon_config = config.read_standard_configuration()      # Initial pyon.yml + pyon.local.yml
        clear_db_util.clear_db(bootstrap_config, prefix=bootstrap.get_sys_name(), sysname=bootstrap.get_sys_name())
        FileSystem._clean(pyon_config)


    # ion_config - Holds the new CFG object for the system (independent of this tool's config)
    ion_config = config.read_standard_configuration()
    if config_override:
        config.apply_configuration(ion_config, config_override)
    config.apply_configuration(ion_config, command_line_config)


    # -------------------------------------------------------------------------
    # Store config and interfaces

    iadm = InterfaceAdmin(bootstrap.get_sys_name(), config=bootstrap_config)

    # Make sure core datastores exist
    iadm.create_core_datastores()

    # Store system CFG properties
    iadm.store_config(ion_config)

#.........这里部分代码省略.........

# 需要导入模块: from pyon.util.file_sys import FileSystem [as 别名]
# 或者: from pyon.util.file_sys.FileSystem import _clean [as 别名]
 def _file_sys_clean(self):
     if os.environ.get('CEI_LAUNCH_TEST', None) is None:
         FileSystem._clean(CFG)