Python arctic.Arctic方法代码示例

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def __init__(self, collection_name, mongo_db=None):


        if mongo_db is None:
            mongo_db = mongoDb()

        database_name = mongo_db.database_name
        host = mongo_db.host

        # Arctic doesn't accept a port

        store = Arctic(host)
        library_name = database_name+"."+collection_name
        store.initialize_library(library_name) # will this fail if already exists??
        library = store[library_name]

        self.database_name = database_name
        self.collection_name = collection_name
        self.host = host

        self.store = store
        self.library_name = library_name
        self.library = library 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def test_multiprocessing_safety(mongo_host, library_name):
    # Create/initialize library at the parent process, then spawn children, and start them aligned in time
    total_processes = 64
    total_writes_per_child = 100

    register_get_auth_hook(my_auth_hook)

    global MY_ARCTIC
    MY_ARCTIC = Arctic(mongo_host=mongo_host)

    MY_ARCTIC.initialize_library(library_name, VERSION_STORE)
    assert isinstance(MY_ARCTIC.get_library(library_name), VersionStore)

    processes = [Process(target=f, args=(library_name, total_writes_per_child, True)) for _ in range(total_processes)]

    for p in processes:
        p.start()

    for p in processes:
        p.join()

    for p in processes:
        assert p.exitcode == 0

    assert isinstance(MY_ARCTIC.get_library(library_name), VersionStore) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def __init__(self, TYPE):
        """
        :param TYPE: 'BS', 'IS', 'CF'
        """

        ############ SETTING #############
        self.config = GetConfig()
        self.TYPE = TYPE # 'BS', 'IS', 'CF'
        self.MONGO = self.config.MONGO
        self.CSV = self.config.CSV
        self.RAW = False
        self.outdir = self.config.fund_dir
        self.encode = self.config.encode
        self.proxypool = self.config.proxypool

        ############ CHANGE ABOVE SETTING #############

        if self.MONGO:
            from arctic import Arctic
            # mongod --dbpath D:/idwzx/project/arctic
            a = Arctic(self.config.ahost)
            a.initialize_library('ashare_{}'.format(self.TYPE))
            self.lib = a['ashare_{}'.format(self.TYPE)]

        self.result_dict = {} 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def loader_func(**kwargs):
    try:
        from arctic import Arctic
        from arctic.store.versioned_item import VersionedItem
    except ImportError:
        raise ImportError("In order to use the arctic loader you must install arctic!")
    host = Arctic(kwargs.get("host"))
    lib = host.get_library(kwargs.get("library"))
    read_kwargs = {}
    start, end = (kwargs.get(p) for p in ["start", "end"])
    if start and end:
        read_kwargs["chunk_range"] = pd.date_range(start, end)
    data = lib.read(kwargs.get("node"), **read_kwargs)
    if isinstance(data, VersionedItem):
        data = data.data
    return data


# IMPORTANT!!! This function is required for building any customized CLI loader. 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def find_loader(kwargs):
    """
    Arctic implementation of data loader which will return a function if any of the
    `click` options based on LOADER_KEY & LOADER_PROPS have been used, otherwise return None

    :param kwargs: Optional keyword arguments to be passed from `click`
    :return: data loader function for arctic implementation
    """
    arctic_opts = get_loader_options(LOADER_KEY, kwargs)
    if len([f for f in arctic_opts.values() if f]):

        def _arctic_loader():
            return loader_func(**arctic_opts)

        return _arctic_loader
    return None 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def write_to_db(target_building, iterator):
    '''write the data from a building'''

    conn = Arctic('localhost')

    #create a lib for the tgt_bldg, a lib is akin to a collection
    if target_building not in conn.list_libraries():
        conn.initialize_library(target_building, lib_type=CHUNK_STORE)
        print ('library for %s created'%target_building)

    #connect to the lib for writing
    lib = conn[target_building]

    for sensor, timestamps, data in iterator:
        df = pd.DataFrame({'date': timestamps, 'data': data})
        df.set_index('date', inplace=True)
        lib.write(sensor, df)
        #print ('writing %s is done'%sensor) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def get_store():
    """
    get Arctic store connection
    :return: arctic connection
    """

    mongo_host = conf.MONGO_HOST
    store = arctic.Arctic(mongo_host)
    return store 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def setup(self, arg):
        self.store = Arctic("127.0.0.1")
        self.store.delete_library('test.lib')
        self.store.initialize_library('test.lib')
        self.lib = self.store['test.lib'] 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def __init__(self):
        self.store = Arctic("127.0.0.1") 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def insert_random_data(config, args, n_rows):
    store = Arctic(args.mongodb, app_name="benchmark")
    lib_name = lib_name_from_args(config)
    store.delete_library(lib_name)
    store.initialize_library(lib_name, segment='month')
    lib = store[lib_name]

    for sym in range(args.symbols):
        df = gen_oneminute_dataset(n_row=n_rows, n_col=n_rows, dense=args.dense)
        lib.write('sym' + str(sym), df) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def append_random_rows(config, args, n_rows):
    store = Arctic(args.mongodb, app_name="benchmark")
    lib_name = lib_name_from_args(config)

    lib = store[lib_name]

    for _ in range(args.appends):
        for sym in range(args.symbols):
            df = gen_oneminute_dataset(n_row=APPEND_NROWS, n_col=n_rows, dense=False)
            lib.append('sym' + str(sym), df) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def test_multiprocessing_safety_parent_children_race(mongo_host, library_name):
    # Create Arctic and directly fork/start children (no wait)
    total_iterations = 12
    total_processes = 6
    total_writes_per_child = 20

    global MY_ARCTIC

    for i in range(total_iterations):
        processes = list()

        MY_ARCTIC = Arctic(mongo_host=mongo_host)
        for j in range(total_processes):
            p = Process(target=f, args=(library_name, total_writes_per_child, False))
            p.start()  # start directly, don't wait to create first all children procs
            processes.append(p)

        MY_ARCTIC.initialize_library(library_name, VERSION_STORE)  # this will unblock spinning children

        for p in processes:
            p.join()

        for p in processes:
            assert p.exitcode == 0

        MY_ARCTIC.reset()

    assert isinstance(MY_ARCTIC.get_library(library_name), VersionStore) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def test_enable_sharding():
    arctic_lib = create_autospec(ArcticLibraryBinding)
    arctic_lib.arctic = create_autospec(Arctic)
    with patch('arctic.store.bson_store.enable_sharding', autospec=True) as enable_sharding:
        arctic_lib.get_top_level_collection.return_value.database.create_collection.__name__ = 'some_name'
        arctic_lib.get_top_level_collection.return_value.database.collection_names.__name__ = 'some_name'
        bsons = BSONStore(arctic_lib)
        bsons.enable_sharding()
        # Check we always set the sharding to be hashed.
        assert enable_sharding.call_args_list == [call(arctic_lib.arctic, arctic_lib.get_name(), hashed=True, key='_id')] 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def arctic(mongo_server_module):
    disable_arctic_cache(mongo_server_module.api)
    mongo_server_module.api.drop_database("arctic")
    mongo_server_module.api.drop_database("arctic_{}".format(getpass.getuser()))
    return Arctic(mongo_server_module.api) 

# 需要导入模块: import arctic [as 别名]
# 或者: from arctic import Arctic [as 别名]
def open(self, store='chunkstore'):
		self.db = Arctic('localhost')
		try:
			self.store = self.db[store]
		except:
			self.db.initialize_library(store, lib_type=CHUNK_STORE)
			self.store = self.db[store]
			self.store._arctic_lib.set_quota(maxDBStorage * 1024 * 1024 * 1024)