Python config.get方法代码示例

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def got_init_config(self, ch, method, properties, body):
		logging.info("Received intitial config %r" % (body))
		if self.corr_id == properties.correlation_id: #we got the right config
			try: #TODO: add check if response is empty...
				new_conf = json.loads(body)
				new_conf["rabbitmq"] = config.get("rabbitmq")
			except Exception as e:
				logging.exception("Wasn't able to read JSON config from manager:\n%s" % e)
				time.sleep(60) #sleep for X seconds and then ask again
				self.fetch_init_config()
				return
		
			logging.info("Trying to apply config and reconnect")
			self.apply_config(new_conf)
			self.connection_cleanup()
			self.connect() #hope this is the right spot
			logging.info("Initial config activated")
			self.start()
		else:
			logging.info("This config isn't meant for us")
	
	# Create a zip of all the files which were collected while actions were executed 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def __init__(self, args, config):
        self.args = args
        self.config = config
        self.cache_dir = utils.get_cache_dir(config)
        self.model_dir = utils.get_model_dir(config)
        self.category = utils.get_category(config, self.cache_dir if os.path.exists(self.cache_dir) else None)
        self.draw_bbox = utils.visualize.DrawBBox(self.category, colors=args.colors, thickness=args.thickness)
        self.anchors = torch.from_numpy(utils.get_anchors(config)).contiguous()
        self.height, self.width = tuple(map(int, config.get('image', 'size').split()))
        self.path, self.step, self.epoch = utils.train.load_model(self.model_dir)
        state_dict = torch.load(self.path, map_location=lambda storage, loc: storage)
        self.dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config, state_dict), self.anchors, len(self.category))
        self.dnn.load_state_dict(state_dict)
        self.inference = model.Inference(config, self.dnn, self.anchors)
        self.inference.eval()
        if torch.cuda.is_available():
            self.inference.cuda()
        logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
        self.cap = self.create_cap()
        self.keys = set(args.keys)
        self.resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
        self.transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
        self.transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split()) 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def main():
    args = make_args()
    config = configparser.ConfigParser()
    utils.load_config(config, args.config)
    for cmd in args.modify:
        utils.modify_config(config, cmd)
    with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
        logging.config.dictConfig(yaml.load(f))
    height, width = tuple(map(int, config.get('image', 'size').split()))
    cache_dir = utils.get_cache_dir(config)
    model_dir = utils.get_model_dir(config)
    category = utils.get_category(config, cache_dir if os.path.exists(cache_dir) else None)
    anchors = utils.get_anchors(config)
    anchors = torch.from_numpy(anchors).contiguous()
    path, step, epoch = utils.train.load_model(model_dir)
    state_dict = torch.load(path, map_location=lambda storage, loc: storage)
    dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config, state_dict), anchors, len(category))
    inference = model.Inference(config, dnn, anchors)
    inference.eval()
    logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in inference.state_dict().values())))
    dnn.load_state_dict(state_dict)
    image = torch.autograd.Variable(torch.randn(args.batch_size, 3, height, width), volatile=True)
    path = model_dir + '.onnx'
    logging.info('save ' + path)
    torch.onnx.export(dnn, image, path, export_params=True, verbose=args.verbose) # PyTorch's bug 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def get_loader(self):
        paths = [os.path.join(self.cache_dir, phase + '.pkl') for phase in self.config.get('eval', 'phase').split()]
        dataset = utils.data.Dataset(utils.data.load_pickles(paths))
        logging.info('num_examples=%d' % len(dataset))
        size = tuple(map(int, self.config.get('image', 'size').split()))
        try:
            workers = self.config.getint('data', 'workers')
        except configparser.NoOptionError:
            workers = multiprocessing.cpu_count()
        collate_fn = utils.data.Collate(
            transform.parse_transform(self.config, self.config.get('transform', 'resize_eval')),
            [size],
            transform_image=transform.get_transform(self.config, self.config.get('transform', 'image_test').split()),
            transform_tensor=transform.get_transform(self.config, self.config.get('transform', 'tensor').split()),
        )
        return torch.utils.data.DataLoader(dataset, batch_size=self.args.batch_size, num_workers=workers, collate_fn=collate_fn) 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def main():
    args = make_args()
    config = configparser.ConfigParser()
    utils.load_config(config, args.config)
    for cmd in args.modify:
        utils.modify_config(config, cmd)
    with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
        logging.config.dictConfig(yaml.load(f))
    category = utils.get_category(config)
    anchors = torch.from_numpy(utils.get_anchors(config)).contiguous()
    dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config), anchors, len(category))
    inference = model.Inference(config, dnn, anchors)
    inference.train()
    optimizer = eval(config.get('train', 'optimizer'))(filter(lambda p: p.requires_grad, inference.parameters()), args.learning_rate)
    scheduler = eval(config.get('train', 'scheduler'))(optimizer)
    for epoch in range(args.epoch):
        scheduler.step(epoch)
        lr = scheduler.get_lr()
        print('\t'.join(map(str, [epoch] + lr))) 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def __init__(self, env):
        super(SummaryWorker, self).__init__()
        self.env = env
        self.config = env.config
        self.queue = multiprocessing.Queue()
        try:
            self.timer_scalar = utils.train.Timer(env.config.getfloat('summary', 'scalar'))
        except configparser.NoOptionError:
            self.timer_scalar = lambda: False
        try:
            self.timer_image = utils.train.Timer(env.config.getfloat('summary', 'image'))
        except configparser.NoOptionError:
            self.timer_image = lambda: False
        try:
            self.timer_histogram = utils.train.Timer(env.config.getfloat('summary', 'histogram'))
        except configparser.NoOptionError:
            self.timer_histogram = lambda: False
        with open(os.path.expanduser(os.path.expandvars(env.config.get('summary_histogram', 'parameters'))), 'r') as f:
            self.histogram_parameters = utils.RegexList([line.rstrip() for line in f])
        self.draw_bbox = utils.visualize.DrawBBox(env.category)
        self.draw_feature = utils.visualize.DrawFeature() 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def run(self):
        self.writer = SummaryWriter(os.path.join(self.env.model_dir, self.env.args.run))
        try:
            height, width = tuple(map(int, self.config.get('image', 'size').split()))
            tensor = torch.randn(1, 3, height, width)
            step, epoch, dnn = self.env.load()
            self.writer.add_graph(dnn, (torch.autograd.Variable(tensor),))
        except:
            traceback.print_exc()
        while True:
            name, kwargs = self.queue.get()
            if name is None:
                break
            func = getattr(self, 'summary_' + name)
            try:
                func(**kwargs)
            except:
                traceback.print_exc() 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def get_loader(self):
        paths = [os.path.join(self.cache_dir, phase + '.pkl') for phase in self.config.get('train', 'phase').split()]
        dataset = utils.data.Dataset(
            utils.data.load_pickles(paths),
            transform=transform.augmentation.get_transform(self.config, self.config.get('transform', 'augmentation').split()),
            one_hot=None if self.config.getboolean('train', 'cross_entropy') else len(self.category),
            shuffle=self.config.getboolean('data', 'shuffle'),
            dir=os.path.join(self.model_dir, 'exception'),
        )
        logging.info('num_examples=%d' % len(dataset))
        try:
            workers = self.config.getint('data', 'workers')
            if torch.cuda.is_available():
                workers = workers * torch.cuda.device_count()
        except configparser.NoOptionError:
            workers = multiprocessing.cpu_count()
        collate_fn = utils.data.Collate(
            transform.parse_transform(self.config, self.config.get('transform', 'resize_train')),
            utils.train.load_sizes(self.config),
            maintain=self.config.getint('data', 'maintain'),
            transform_image=transform.get_transform(self.config, self.config.get('transform', 'image_train').split()),
            transform_tensor=transform.get_transform(self.config, self.config.get('transform', 'tensor').split()),
            dir=os.path.join(self.model_dir, 'exception'),
        )
        return torch.utils.data.DataLoader(dataset, batch_size=self.args.batch_size * torch.cuda.device_count() if torch.cuda.is_available() else self.args.batch_size, shuffle=True, num_workers=workers, collate_fn=collate_fn, pin_memory=torch.cuda.is_available()) 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def enable_codejail(self, codejail_config):
        """
        Enable codejail for the process.
        codejail_config is a dict like this:
        {
            "name": "python",
            "bin_path": "/path/to/python",
            "user": "sandbox_username",
            "limits": {
                "CPU": 1,
                ...
            }
        }
        limits are optional
        user defaults to the current user
        """
        name = codejail_config["name"]
        bin_path = codejail_config['bin_path']
        user = codejail_config.get('user', getpass.getuser())
        jail_code.configure(name, bin_path, user=user)
        limits = codejail_config.get("limits", {})
        for name, value in limits.items():
            jail_code.set_limit(name, value)
        self.log.info("configured codejail -> %s %s %s", name, bin_path, user)
        return name 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def _get_action_endpoint(action):
    """
    Return the endpoint base on the view's action
    :param action:
    :return:
    """
    _endpoint = None
    if inspect.ismethod(action) and hasattr(action, "_rule_cache"):
        rc = action._rule_cache
        if rc:
            k = list(rc.keys())[0]
            rules = rc[k]
            len_rules = len(rules)
            if len_rules == 1:
                rc_kw = rules[0][1]
                _endpoint = rc_kw.get("endpoint", None)
                if not _endpoint:
                    _endpoint = _make_routename_from_endpoint(action)
            elif len_rules > 1:
                _prefix = _make_routename_from_endpoint(action)
                for r in Assembly._app.url_map.iter_rules():
                    if ('GET' in r.methods or 'POST' in r.methods) and _prefix in r.endpoint:
                        _endpoint = r.endpoint
                        break
    return _endpoint 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def trace(self, kwargs):
        exc_type, exc_value, exc_traceback = sys.exc_info()
        stack = traceback.extract_tb(exc_traceback)
        lines = []
        for i, s in enumerate(stack):
            filename = s.filename
            l = len(filename)
            shortfile = kwargs.get('shortfile', 40)
            if l > shortfile:
                filename = filename[filename.find('/', l - shortfile):]
            line = '%-40s:%-4s %s' % (
                blue() + filename, yellow() + str(s.lineno),
                '|' + '-' * (i * 4) + cyan() + s.name + ':' + red() + s.line)
            lines.append(line)
        lines = '\n\t'.join(lines)
        kwargs['extra'] = {
            'trace': magenta() + str(exc_type) + ' ' + bold() + magenta() + str(exc_value) + '\n\t' + lines} 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def unflatten(dictionary, sep: str = "."):
    """Turn a flattened dict into a nested dict.

    :param dictionary: The dict to unflatten.
    :param sep: This character represents a nesting level in a flattened key
    :returns: The nested dict
    """
    nested = {}
    for k, v in dictionary.items():
        keys = k.split(sep)
        it = nested
        for key in keys[:-1]:
            # If key is in `it` we get the value otherwise we get a new dict.
            # .setdefault will also set the new dict to the value of `it[key]`.
            # assigning to `it` will move us a step deeper into the nested dict.
            it = it.setdefault(key, {})
        it[keys[-1]] = v
    return nested 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def order_json(data):
    """Sort json to a consistent order.
    When you hash json that has the some content but is different orders you get
    different fingerprints.
    In:  hashlib.sha1(json.dumps({'a': 12, 'b':14}).encode('utf-8')).hexdigest()
    Out: '647aa7508f72ece3f8b9df986a206d95fd9a2caf'
    In:  hashlib.sha1(json.dumps({'b': 14, 'a':12}).encode('utf-8')).hexdigest()
    Out: 'a22215982dc0e53617be08de7ba9f1a80d232b23'
    This function sorts json by key so that hashes are consistent.
    Note:
        In our configs we only have lists where the order doesn't matter so we
        can sort them for consistency. This would have to change if we add a
        config field that needs order we will need to refactor this.
    :param data: dict, The json data.
    :returns:
        collections.OrderedDict: The data in a consistent order (keys sorted alphabetically).
    """
    new = OrderedDict()
    for (key, value) in sorted(data.items(), key=lambda x: x[0]):
        if isinstance(value, dict):
            value = order_json(value)
        new[key] = value
    return new 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def remove_extra_keys(config, keys=KEYS):
    """Remove config items that don't effect the model.
    We base most things off of the sha1 hash of the model configs but there
    is a problem. Some things in the config file don't effect the model such
    as the name of the `conll_output` file or if you are using `visdom`
    reporting. This strips out these kind of things so that as long as the model
    parameters match the sha1 will too.
    :param config: dict, The json data.
    :param keys: Set[Tuple[str]], The keys to remove.
    :returns:
        dict, The config with certain keys removed.
    """
    c = deepcopy(config)
    for key in keys:
        x = c
        for k in key[:-1]:
            x = x.get(k)
            if x is None:
                break
        else:
            _ = x.pop(key[-1], None)
    return c 

# 需要导入模块: from logging import config [as 别名]
# 或者: from logging.config import get [as 别名]
def _get_home():
    """Find user's home directory if possible.
    Otherwise, returns None.

    :see:  http://mail.python.org/pipermail/python-list/2005-February/325395.html

    This function is copied from matplotlib version 1.4.3, Jan 2016
    """
    try:
        if six.PY2 and sys.platform == 'win32':
            path = os.path.expanduser(b"~").decode(sys.getfilesystemencoding())
        else:
            path = os.path.expanduser("~")
    except ImportError:
        # This happens on Google App Engine (pwd module is not present).
        pass
    else:
        if os.path.isdir(path):
            return path
    for evar in ('HOME', 'USERPROFILE', 'TMP'):
        path = os.environ.get(evar)
        if path is not None and os.path.isdir(path):
            return path
    return None