Python utils.utils方法代码示例

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def run(self):
        self.set_logger()

        # Initialize progress bar
        bar = utils.set_progress_bar(self.total_iters)

        for cycle_num in range(int(self.total_iters / self.iters)):
            self.model.train()

            self.cycle(bar, cycle_num)

            with torch.no_grad():
                self.run_evaluation_cycle()

            self.log_losses(self.opt, self.losses)
            self.update_top_score(self.opt)
            self.save_model(self.get_tracked_score())

        self.stop_logger() 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def load_data(self, path):
        if ".pickle" in path:
            print("Loading data from: {}".format(path))
            data_utils.load_existing_data_loader(self, path)

            return True

        for split in self.data:
            file_name = "v4_atomic_{}.csv".format(map_name(split))

            df = pandas.read_csv("{}/{}".format(path, file_name), index_col=0)
            df.iloc[:, :9] = df.iloc[:, :9].apply(
                lambda col: col.apply(json.loads))

            for cat in self.categories:
                attr = df[cat]
                self.data[split]["total"] += utils.zipped_flatten(zip(
                    attr.index, ["<{}>".format(cat)] * len(attr), attr.values))

        if do_take_partial_dataset(self.opt.data):
            self.data["train"]["total"] = select_partial_dataset(
                self.opt.data, self.data["train"]["total"])

        return False 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def handle_underscores(suffix, text_encoder, prefix=False):
    encoder = text_encoder.encoder
    if prefix:
        tok = "___"
    else:
        tok = find_underscore_length(suffix)

    suffix_parts = [i.strip() for i in suffix.split("{}".format(tok))]
    to_flatten = []
    for i, part in enumerate(suffix_parts):
        if part:
            to_flatten.append(text_encoder.encode([part], verbose=False)[0])

            if i != len(suffix_parts) - 1 and suffix_parts[i+1]:
                to_flatten.append([encoder["<blank>"]])
        else:
            to_flatten.append([encoder["<blank>"]])

    final_suffix = utils.flatten(to_flatten)

    return final_suffix 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def dataloader_create(self, args):
        from torch.utils.data import DataLoader
        from myDatasets_stereo import dataset_stereo_by_name as dataset_stereo
        args.mode = args.mode.lower()
        if args.mode == 'test' or args.mode == 'submit':
            # dataloader
            transform=myTransforms.Stereo_eval()
            dataset = dataset_stereo(names_dataset=args.dataset, root=args.root, Train=False, transform=transform)
            self.dataloader_val = DataLoader(dataset, batch_size=args.batchsize, shuffle=False, num_workers=4, drop_last=False)
            msg = "Val dataset: %s " % (args.dataset)
            logging.info(msg)
        else:
            # dataloader
            transform=myTransforms.Stereo_Spatial(size_crop=[768, 384], scale_delt=0, shift_max=32)
            dataset = dataset_stereo(names_dataset=args.dataset, root=args.root, Train=True, transform=transform)
            self.dataloader_train = DataLoader(dataset, batch_size=args.batchsize, shuffle=True, num_workers=4, drop_last=False)
            transform=myTransforms.Stereo_ToTensor()
            dataset_val = dataset_stereo(names_dataset=args.dataset_val, root=args.root, Train=False, transform=transform)
            self.dataloader_val = DataLoader(dataset_val, batch_size=args.batchsize, shuffle=False, num_workers=4, drop_last=False)
            msg = "Train dataset: %s , Val dataset: %s " % (args.dataset, args.dataset_val)
            logging.info(msg) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def dataloader_create(self, args):
        from torch.utils.data import DataLoader
        from myDatasets_stereo import dataset_stereo_by_name as dataset_stereo
        import myTransforms
        args.mode = args.mode.lower()
        if args.mode == 'test' or args.mode == 'submit':
            # dataloader
            transform=myTransforms.Stereo_eval()
            dataset = dataset_stereo(names_dataset=args.dataset, root=args.root, Train=False, transform=transform)
            self.dataloader_val = DataLoader(dataset, batch_size=args.batchsize, shuffle=False, num_workers=4, drop_last=False)
            msg = "%s dataset: %s , model name: %s " % (args.mode, args.dataset, args.net)
            logging.info(msg)
        else:
            # dataloader
            transform = myTransforms.Stereo_train(size_crop=[768, 384], scale_delt=0, shift_max=32)
            dataset = dataset_stereo(names_dataset=args.dataset, root=args.root, Train=True, transform=transform)
            self.dataloader_train = DataLoader(dataset, batch_size=args.batchsize, shuffle=True, num_workers=4, drop_last=False)
            transform=myTransforms.Stereo_eval()
            dataset_val = dataset_stereo(names_dataset=args.dataset_val, root=args.root, Train=False, transform=transform)
            self.dataloader_val = DataLoader(dataset_val, batch_size=args.batchsize, shuffle=False, num_workers=4, drop_last=False)
            msg = "Train dataset: %s , val dataset: %s " % (args.dataset, args.dataset_val)
            logging.info(msg) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

# FIXME: Move to utils, just like we're doing with the mnist module 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def set_logger(self):
        if cfg.toy:
            self.logger = SummaryWriter(utils.make_name(
                self.opt, prefix="garbage/logs/", eval_=True, do_epoch=False))
        else:
            self.logger = SummaryWriter(utils.make_name(
                self.opt, prefix="logs/", eval_=True, do_epoch=False))
        print("Logging Tensorboard Files at: {}".format(self.logger.logdir)) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def epoch(self):
        nums = self.reset_losses()

        # Initialize progress bar
        bar = utils.initialize_progress_bar(
            self.data_loader.sequences["train"])

        reset = False

        while not reset:
            loss, nums, reset = self.do_forward_pass(nums)
            self.do_backward_pass(loss)
            self.update_parameters()

            bar.update(self.opt.train.dynamic.bs)
            self.count += 1

            for loss_name in self.losses["train"]:
                self.logger.add_scalar(
                    "train/{}".format(loss_name),
                    loss.item() / self.opt.train.dynamic.bs,
                    self.count)

            if cfg.toy and self.counter(nums) > 300:
                break

        with torch.no_grad():
            self.run_evaluation_cycle()

        self.log_losses(self.opt, self.losses)
        self.update_top_score(self.opt)
        self.save_model(self.get_tracked_score())

        self.data_loader.reset_offsets("train") 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def clip_gradients(self):
        if self.opt.train.static.clip:
            torch.nn.utils.clip_grad_norm_(
                self.model.parameters(), self.opt.train.static.clip) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def save_step(model, vocab, optimizer, opt, length, lrs):
    if cfg.test_save:
        name = "{}.pickle".format(utils.make_name(
            opt, prefix="garbage/models/", is_dir=False, eval_=True))
    else:
        name = "{}.pickle".format(utils.make_name(
            opt, prefix="models/", is_dir=False, eval_=True))
    save_checkpoint({
        "epoch": length, "state_dict": model.state_dict(),
        "optimizer": optimizer.state_dict(), "opt": opt,
        "vocab": vocab, "epoch_learning_rates": lrs},
        name) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def load_atomic_data(opt):
    # Hacky workaround, you may have to change this
    # if your models use different pad lengths for e1, e2, r
    if opt.data.get("maxe1", None) is None:
        opt.data.maxe1 = 17
        opt.data.maxe2 = 35
        opt.data.maxr = 1
    path = "data/atomic/processed/generation/{}.pickle".format(
        utils.make_name_string(opt.data))
    data_loader = data.make_data_loader(opt, opt.data.categories)
    loaded = data_loader.load_data(path)

    return data_loader 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def save_checkpoint(self, epoch, best_prec, is_best):
        state = {
                'epoch': epoch,
                'best_prec': best_prec,
                'state_dict': self.model.state_dict(),
                'optim' : self.optim.state_dict(),
                }
        utils.save_checkpoint(state, is_best, dirpath=self.dirpath, filename='model_checkpoint.pkl')
        if(is_best):
            path_save = os.path.join(self.dirpath, 'weight_best.pkl')
            torch.save({'state_dict': self.model.state_dict()}, path_save) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def load_checkpoint(self, best=False):
        state = utils.load_checkpoint(self.dirpath, best)
        if state is not None:
            msg = 'load checkpoint successly: %s \n' % self.dirpath
            logging.info(msg)
            self.epoch = state['epoch'] + 1
            self.best_prec = state['best_prec']
            self.model.load_state_dict(state['state_dict'])
            self.optim.load_state_dict(state['optim']) 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def detect_and_draw(model, bev_maps, Tensor, is_front=True):

    # If back side bev, flip around vertical axis
    if not is_front:
        bev_maps = torch.flip(bev_maps, [2, 3])
    imgs = Variable(bev_maps.type(Tensor))

    # Get Detections
    img_detections = []
    with torch.no_grad():
        detections = model(imgs)
        detections = utils.non_max_suppression_rotated_bbox(detections, opt.conf_thres, opt.nms_thres)

    img_detections.extend(detections)

    # Only supports single batch
    display_bev = np.zeros((cnf.BEV_WIDTH, cnf.BEV_WIDTH, 3))
    
    bev_map = bev_maps[0].numpy()
    display_bev[:, :, 2] = bev_map[0, :, :]  # r_map
    display_bev[:, :, 1] = bev_map[1, :, :]  # g_map
    display_bev[:, :, 0] = bev_map[2, :, :]  # b_map

    display_bev *= 255
    display_bev = display_bev.astype(np.uint8)

    for detections in img_detections:
        if detections is None:
            continue
        # Rescale boxes to original image
        detections = utils.rescale_boxes(detections, opt.img_size, display_bev.shape[:2])
        for x, y, w, l, im, re, conf, cls_conf, cls_pred in detections:
            yaw = np.arctan2(im, re)
            # Draw rotated box
            bev_utils.drawRotatedBox(display_bev, x, y, w, l, yaw, cnf.colors[int(cls_pred)])

    return display_bev, img_detections 

# 需要导入模块: from utils import utils [as 别名]
# 或者: from utils.utils import utils [as 别名]
def save_img(self, fig, name):
        utils.save_img(fig, self.model_name, name, self.result_dir)
        return