Python shelve.open方法代码示例

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def __init__(self, reduce_memory=False):
        if reduce_memory:
            self.temp_dir = TemporaryDirectory()
            self.working_dir = Path(self.temp_dir.name)
            self.document_shelf_filepath = self.working_dir / 'shelf.db'
            self.document_shelf = shelve.open(str(self.document_shelf_filepath),
                                              flag='n', protocol=-1)
            self.documents = None
        else:
            self.documents = []
            self.document_shelf = None
            self.document_shelf_filepath = None
            self.temp_dir = None
        self.doc_lengths = []
        self.doc_cumsum = None
        self.cumsum_max = None
        self.reduce_memory = reduce_memory 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def create_training_file(docs, tokenizer, args, epoch_num):
    epoch_filename = args.output_dir / "epoch_{}.json".format(epoch_num)
    num_instances = 0
    with epoch_filename.open('w') as epoch_file:
        for doc_idx in trange(len(docs), desc="Document"):
            doc_instances = create_instances_from_document(
                docs, doc_idx, max_seq_length=args.max_seq_len, short_seq_prob=args.short_seq_prob,
                masked_lm_prob=args.masked_lm_prob, max_predictions_per_seq=args.max_predictions_per_seq,
                whole_word_mask=args.do_whole_word_mask, tokenizer=tokenizer,
                next_sent_prediction=args.do_next_sent_prediction)
            doc_instances = [json.dumps(instance) for instance in doc_instances]
            for instance in doc_instances:
                epoch_file.write(instance + '\n')
                num_instances += 1
    metrics_file = args.output_dir / "epoch_{}_metrics.json".format(epoch_num)
    with metrics_file.open('w') as metrics_file:
        metrics = {
            "num_training_examples": num_instances,
            "max_seq_len": args.max_seq_len
        }
        metrics_file.write(json.dumps(metrics)) 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def input_file_to_training_data(args, input_file, epoch, tokenizer, num_files):
    print(input_file)
    with DocumentDatabase(reduce_memory=args.reduce_memory) as docs:
        with open(input_file) as f:
            doc = []
            for line in tqdm(f, desc="Loading Dataset", unit=" lines"):
                line = line.strip()
                if line == "":
                    docs.add_document(doc)
                    doc = []
                else:
                    tokens = tokenizer.tokenize(line)
                    doc.append(tokens)
            if doc:
                docs.add_document(doc)  # If the last doc didn't end on a newline, make sure it still gets added
        if len(docs) <= 1:
            exit("ERROR: No document breaks were found in the input file! These are necessary to allow the script to "
                    "ensure that random NextSentences are not sampled from the same document. Please add blank lines to "
                    "indicate breaks between documents in your input file. If your dataset does not contain multiple "
                    "documents, blank lines can be inserted at any natural boundary, such as the ends of chapters, "
                    "sections or paragraphs.")

        for i in range(args.epochs_to_generate):
            create_training_file(docs, tokenizer, args, epoch + i * num_files) 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def _testKeys(self):
	"""Verify that index lookup can find each word in the index file."""
	print "Testing: ", self
	file = open(self.indexFile.file.name, _FILE_OPEN_MODE)
	counter = 0
	while 1:
	    line = file.readline()
	    if line == '': break
	    if line[0] != ' ':
		key = string.replace(line[:string.find(line, ' ')], '_', ' ')
		if (counter % 1000) == 0:
		    print "%s..." % (key,),
		    import sys
		    sys.stdout.flush()
		counter = counter + 1
		self[key]
	file.close()
	print "done." 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def val_dump(rels, db):
    """
    Make a ``Valuation`` from a list of relation metadata bundles and dump to
    persistent database.

    :param rels: bundle of metadata needed for constructing a concept
    :type rels: list of dict
    :param db: name of file to which data is written.
               The suffix '.db' will be automatically appended.
    :type db: string
    """
    concepts = process_bundle(rels).values()
    valuation = make_valuation(concepts, read=True)
    db_out = shelve.open(db, 'n')

    db_out.update(valuation)

    db_out.close() 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def label_indivs(valuation, lexicon=False):
    """
    Assign individual constants to the individuals in the domain of a ``Valuation``.

    Given a valuation with an entry of the form ``{'rel': {'a': True}}``,
    add a new entry ``{'a': 'a'}``.

    :type valuation: Valuation
    :rtype: Valuation
    """
    # collect all the individuals into a domain
    domain = valuation.domain
    # convert the domain into a sorted list of alphabetic terms
    # use the same string as a label
    pairs = [(e, e) for e in domain]
    if lexicon:
        lex = make_lex(domain)
        with open("chat_pnames.cfg", 'w') as outfile:
            outfile.writelines(lex)
    # read the pairs into the valuation
    valuation.update(pairs)
    return valuation 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def textFile( readFromFile ):
   
   # open text file
   try:
      outputFile = open( "print.txt", "w" )
   except IOError:
      print >> sys.stderr, "File could not be opened."
      sys.exit( 1 )

   print >> outputFile, "Account".ljust( 10 ),
   print >> outputFile, "Last Name".ljust( 10 ),
   print >> outputFile, "First Name".ljust( 10 ),
   print >> outputFile, "Balance".rjust( 10 )

   # print shelve values to text file
   for key in readFromFile.keys():
      print >> outputFile, key.ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 0 ].ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 1 ].ljust( 10 ),
      print >> outputFile, readFromFile[ key ][ 2 ].rjust( 10 )

   outputFile.close()

# update account balance 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def loadState():
    """
    Attempt to load the program from a pickled state in the saves directory.
    """
    savePath = pathlib.Path("RedditDataExtractor", "saves")
    if not savePath.exists():
        savePath.mkdir()
    shelf = shelve.open(str(savePath / "settings.db"))
    rddtDataExtractor = None
    try:
        rddtDataExtractor = shelf['rddtDataExtractor']
        userListSettings = shelf['userLists']
        subredditListSettings = shelf['subredditLists']
        rddtDataExtractor.userLists = {}
        rddtDataExtractor.subredditLists = {}
        # Reconstruct the lists because GUI stuff isn't pickleable
        for key, val in userListSettings.items():
            rddtDataExtractor.userLists[key] = ListModel(val, User)
        for key, val in subredditListSettings.items():
            rddtDataExtractor.subredditLists[key] = ListModel(val, Subreddit)
    except KeyError:
        pass
    finally:
        shelf.close()
        return rddtDataExtractor 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def _allow_host_pkeys(self):
        device_path = "/sys/class/infiniband/{0}".format(self._ibdev_name)
        num_ports = len(os.listdir(os.path.join(device_path, "ports")))

        for port in xrange(1, num_ports + 1):
            pkeys_path = os.path.join(device_path, "ports", str(port),
                                      "pkeys")
            pkey_idx_path = os.path.join(device_path, "iov", self._dev_addr,
                                         "ports", str(port), "pkey_idx")

            idx = 0
            for pkey_idx in os.listdir(pkeys_path):
                p = os.path.join(pkeys_path, pkey_idx)
                with open(p) as f:
                    try:
                        this_pkey_value = int(f.read().strip(), 0)
                    except ValueError:
                        continue

                    if this_pkey_value:
                        with open(os.path.join(pkey_idx_path, str(idx)), 'w') as f:
                            f.write(pkey_idx)
                        idx+=1 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def ibdev_find_pkey_idx(device_name, pkey_value):
    pkey_idx_path = "/sys/class/infiniband/%s/ports/1/pkeys" % (
        device_name)

    for pkey_idx in os.listdir(pkey_idx_path):
        this_pkey_idx_path = os.path.join(pkey_idx_path, pkey_idx)
        with open(this_pkey_idx_path) as f:
            try:
                this_pkey_value = int(f.read().strip(), 0)
            except ValueError:
                continue

            if this_pkey_value & 0x7fff == pkey_value & 0x7fff:
                return pkey_idx

    raise NetworkSetupError('pkey %s not found on device %s' % (
        hex(pkey_value), device_name)) 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def openflow_manager(br_name, command, flow_options=None, ovs=None):
    """
    Manager openvswitch flow rules

    :param br_name: name of the bridge
    :param command: manager cmd(add-flow, del-flows, dump-flows..)
    :param flow_options: open flow options
    :param ovs: OpenVSwitch object.
    """
    if ovs is None:
        ovs = __ovs

    if ovs is None or br_name not in ovs.list_br():
        raise OpenflowSwitchError(br_name)

    manager_cmd = "ovs-ofctl %s %s" % (command, br_name)
    if flow_options:
        manager_cmd += " %s" % flow_options
    return process.run(manager_cmd) 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def test_03_db(self):
        """
        Load from database created in test_02_db, verify data against params
        """
        # Verify on-disk data matches dummy data just written
        self.zero_counter()
        db = shelve.open(self.db_filename)
        db_keys = list(db.keys())
        self.assertEqual(len(db_keys), self.db_item_count)
        for key in db_keys:
            db_value = eval(db[key], {}, {})
            self.assert_(isinstance(db_value, list))
            self.assert_(len(db_value) > 0)
            self.assert_(isinstance(db_value[0], dict))
            for nic in db_value:
                mac = nic.get('mac')
                if mac:
                    # Another test already checked mac_is_valid behavior
                    self.assert_(utils_net.VirtIface.mac_is_valid(mac))
            self.print_and_inc()
        db.close() 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def close(self):
        """Cleanup batch job."""
        self.groups_shelf.close()
        self.indicators_shelf.close()
        if self.debug and self.enable_saved_file:
            fqfn = os.path.join(self.tcex.args.tc_temp_path, 'xids-saved')
            if os.path.isfile(fqfn):
                os.remove(fqfn)  # remove previous file to prevent duplicates
            with open(fqfn, 'w') as fh:
                for xid in self.saved_xids:
                    fh.write(f'{xid}\n')
        else:
            # delete saved files
            if os.path.isfile(self.group_shelf_fqfn):
                os.remove(self.group_shelf_fqfn)
            if os.path.isfile(self.group_shelf_fqfn):
                os.remove(self.indicator_shelf_fqfn) 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def loadSetting(self):
        """加载配置"""
        try:
            with open(self.settingFilePath) as f:
                l = json.load(f)
                
                for setting in l:
                    result, msg = self.createSpread(setting)
                    self.writeLog(msg)
                    
                self.writeLog(u'价差配置加载完成')
        except:
            content = u'价差配置加载出错，原因：' + traceback.format_exc()
            self.writeLog(content)
    
    #---------------------------------------------------------------------- 

# 需要导入模块: import shelve [as 别名]
# 或者: from shelve import open [as 别名]
def update_self():
    logger.debug('def update_self started')
    logger.info('Update - will restart')
    cfg.update_date = datetime.now().strftime("%d-%m %H:%M:%S")
    prem_config.set('update', 'update_date', cfg.update_date)
    with open(os.path.join(ConfDir, 'settings.cfg'), 'w') as configfile:  # save
        prem_config.write(configfile)
    db.close()
    socketio.stop()
    if os_arg == '--windows':
        subprocess.call(['python', os.path.join(runningdir, 'utils.py'), '--update', '--windows'])
        os._exit(1)
    else:
        subprocess.Popen(['python', os.path.join(runningdir, 'utils.py'), '--update', '--none'],
                         shell=False,
                         close_fds=True)
        os._exit(1)


# noinspection PyProtectedMember