Python nn.init方法代碼示例

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def init_layers(net, init_fn_name):
    '''Primary method to initialize the weights of the layers of a network'''
    if init_fn_name is None:
        return

    # get nonlinearity
    nonlinearity = get_nn_name(net.hid_layers_activation).lower()
    if nonlinearity == 'leakyrelu':
        nonlinearity = 'leaky_relu'  # guard name

    # get init_fn and add arguments depending on nonlinearity
    init_fn = getattr(nn.init, init_fn_name)
    if 'kaiming' in init_fn_name:  # has 'nonlinearity' as arg
        assert nonlinearity in ['relu', 'leaky_relu'], f'Kaiming initialization not supported for {nonlinearity}'
        init_fn = partial(init_fn, nonlinearity=nonlinearity)
    elif 'orthogonal' in init_fn_name or 'xavier' in init_fn_name:  # has 'gain' as arg
        gain = nn.init.calculate_gain(nonlinearity)
        init_fn = partial(init_fn, gain=gain)
    else:
        pass

    # finally, apply init_params to each layer in its modules
    net.apply(partial(init_params, init_fn=init_fn)) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def init_params(module, init_fn):
    '''Initialize module's weights using init_fn, and biases to 0.0'''
    bias_init = 0.0
    classname = util.get_class_name(module)
    if 'Net' in classname:  # skip if it's a net, not pytorch layer
        pass
    elif any(k in classname for k in ('BatchNorm', 'Conv', 'Linear')):
        init_fn(module.weight)
        nn.init.constant_(module.bias, bias_init)
    elif 'GRU' in classname:
        for name, param in module.named_parameters():
            if 'weight' in name:
                init_fn(param)
            elif 'bias' in name:
                nn.init.constant_(param, bias_init)
    else:
        pass


# params methods 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def _initialize_v(self, G, v, H=None):
        G = [g for g in G if g.vcount() > v]
        if len(G) == 0:
            return
        if H is not None:
            idx = [i for i, g in enumerate(G) if g.vcount() > v]
            H = H[idx]
        v_types = [g.vs[v]['type'] for g in G]
        X = self._one_hot(v_types, self.nvt)
        if H is None:
            Hg = self._get_graph_state(G, 0, 1, init=True)  # exclude v itself
        else:  
            Hg = H
        Hv = self.finit(torch.cat([X, Hg], -1))
        for i, g in enumerate(G):
            g.vs[v]['H_forward'] = Hv[i:i+1]
        return 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def _get_graph_state(self, G, start=0, end_offset=0, init=False):
        # get the graph states, the R function
        Hg = []
        max_n_nodes = max(g.vcount() for g in G)
        for g in G:
            hg = [g.vs[i]['H_forward'] for i in range(start, g.vcount() - end_offset)]
            hg = torch.cat(hg, 0)
            hg = hg.unsqueeze(0)  # 1 * n * hs
            if g.vcount() < max_n_nodes:
                hg = torch.cat([hg, 
                    torch.zeros(1, max_n_nodes - g.vcount(), hg.shape[2]).to(self.get_device())],
                    1)  # 1 * max_n * hs
            Hg.append(hg)
        # gated sum node states as the graph state
        Hg = torch.cat(Hg, 0)  # batch * max_n * hs
        if not init:
            Hg = self._gated(Hg, self.gate, self.mapper).sum(1)  # batch * gs
        else:
            Hg = self._gated(Hg, self.gate_init, self.mapper_init).sum(1)  # batch * gs

        return Hg  # batch * gs 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def _initialize_weights(self):
        for m in self.modules():
            # 卷積的初始化方法
            if isinstance(m, nn.Conv2d):
                # TODO: 使用正態分布進行初始化（0, 0.01) 網絡權重看看
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # He kaiming 初始化, 方差為2/n. math.sqrt(2. / n) 或者直接使用現成的nn.init中的函數。在這裏會梯度爆炸
                m.weight.data.normal_(0, 0.001)    # # math.sqrt(2. / n)
                # torch.nn.init.kaiming_normal_(m.weight)
                # bias都初始化為0
                if m.bias is not None:  # 當有BN層時，卷積層Con不加bias！
                    m.bias.data.zero_()
            # batchnorm使用全1初始化 bias全0
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

            elif isinstance(m, nn.Linear):
                torch.nn.init.normal_(m.weight.data, 0, 0.01)  # todo: 0.001?
                # m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_() 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def __init__(self, input_size, hidden_size, output_size, args):
        super(SimpleSeparateSelectionModule, self).__init__()
        self.output_size = output_size
        self.hidden_size = hidden_size

        self.encoder = nn.Sequential(
            nn.Linear(input_size, hidden_size),
            nn.Tanh(),
            nn.Dropout(args.dropout))

        self.decoders = nn.ModuleList()
        for i in range(6):
            self.decoders.append(nn.Linear(hidden_size, output_size))

        # init
        init_cont(self.encoder, args.init_range)
        init_cont(self.decoders, args.init_range) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def set_global_nets(algorithm, global_nets):
    '''For Hogwild, set attr built in init_global_nets above. Use in algorithm init.'''
    # set attr first so algorithm always has self.global_{net} to pass into train_step
    for net_name in algorithm.net_names:
        setattr(algorithm, f'global_{net_name}', None)
    # set attr created in init_global_nets
    if global_nets is not None:
        util.set_attr(algorithm, global_nets)
        logger.info(f'Set global_nets attr {list(global_nets.keys())} for Hogwild') 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def init(self):
        stdv = 1. / math.sqrt(self.W.weight.size(1))
        self.W.weight.data.uniform_(-stdv, stdv) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def init_weights(self, pretrained=True):
        if pretrained:
            # print('=> init resnet deconv weights from normal distribution')
            print('=> init deconv weights from normal distribution')
            for name, m in self.deconv_layers.named_modules():
                if isinstance(m, nn.BatchNorm2d):
                    nn.init.constant_(m.weight, 1)
                    nn.init.constant_(m.bias, 0)
            #pretrained_state_dict = torch.load(pretrained)
            #address = "/data/pretrained_model/shufflenetv2_x1_69.390_88.412.pth.tar"
            #pretrained_state_dict = torch.load(address)
            #self.load_state_dict(pretrained_state_dict, strict=False) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def k_means_cpu(weight, n_clusters, init='k-means++', max_iter=50):
    # flatten the weight for computing k-means
    org_shape = weight.shape
    weight = weight.reshape(-1, 1)  # single feature
    if n_clusters > weight.size:
        n_clusters = weight.size

    k_means = KMeans(n_clusters=n_clusters, init=init, n_init=1, max_iter=max_iter)
    k_means.fit(weight)

    centroids = k_means.cluster_centers_
    labels = k_means.labels_
    labels = labels.reshape(org_shape)
    return torch.from_numpy(centroids).cuda().view(1, -1), torch.from_numpy(labels).int().cuda() 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def reset_parameters(self):
        nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
        if self.bias is not None:
            fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight)
            bound = 1 / math.sqrt(fan_in)
            nn.init.uniform_(self.bias, -bound, bound) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def on_train_begin(self, **kargs) -> None:
        r'''
        Sets the callback to initialise the model the first time that `on_epoch_begin` is called.
        '''

        self.init = False
        self.gg = {'hook_position':0, 'total_fc_conv_layers':0,'done_counter':-1,'hook':None,'act_dict':{},'counter_to_apply_correction':0,
                   'correction_needed':False,'current_coef':1.0} 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def on_epoch_begin(self, by:BatchYielder, **kargs) -> None:
        r'''
        If the LSUV process has yet to run, then it will run using all of the input data provided by the `BatchYielder`

        Arguments:
            by: BatchYielder providing data for the upcoming epoch
        '''

        if not self.init:
            print('Running LSUV initialisation')
            self._run_lsuv(by.get_inputs(on_device=True))
            self.init = True 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def _orthogonal_weights_init(self, m:nn.Module) -> None:
        if self._check_layer(m):
            if hasattr(m, 'weight_v'):
                w_ortho = self._svd_orthonormal(m.weight_v.data.cpu().numpy())
                m.weight_v.data = torch.from_numpy(w_ortho)
            else:
                w_ortho = self._svd_orthonormal(m.weight.data.cpu().numpy())
                m.weight.data = torch.from_numpy(w_ortho)
            if hasattr(m, 'bias') and m.bias is not None: nn.init.zeros_(m.bias) 

# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import init [as 別名]
def _run_lsuv(self, data:Union[Tensor,Tuple[Tensor,Tensor]]) -> None:
        cuda = next(self.model.model.parameters()).is_cuda
        self.model.model.eval()
        self.model.model.apply(self._count_conv_fc_layers)
        if self.verbose: print(f'Total layers to process: {self.gg["total_fc_conv_layers"]}')
        if self.do_orthonorm:
            self.model.model.apply(self._orthogonal_weights_init)
            if self.verbose: print('Orthonorm done')
            if cuda: self.model.model = self.model.model.cuda()
        for layer_idx in range(self.gg['total_fc_conv_layers']):
            if self.verbose: print(f'Checking layer {layer_idx}')
            self.model.model.apply(self._add_current_hook)
            self.model.model(data)
            current_std = self.gg['act_dict'].std()
            if self.verbose: print(f'std at layer {layer_idx} = {current_std}')
            attempts = 0
            while np.abs(current_std-self.needed_std) > self.std_tol:
                self.gg['current_coef'] = self.needed_std/(current_std+1e-8)
                self.gg['correction_needed'] = True
                self.model.model.apply(self._apply_weights_correction)
                self.model.model(data)
                current_std = self.gg['act_dict'].std()
                if self.verbose: print(f'std at layer {layer_idx} = {current_std} mean = {self.gg["act_dict"].mean()}')
                attempts += 1
                if attempts > self.max_attempts:
                    print(f'Cannot converge in {self.max_attempts} iterations')
                    break
            if self.gg['hook'] is not None: self.gg['hook'].remove()
            self.gg['done_counter'] += 1
            self.gg['counter_to_apply_correction'] = 0
            self.gg['hook_position'] = 0
            self.gg['hook'] = None
            if self.verbose: print(f'Initialised layer {layer_idx}')
        if self.verbose: print('LSUV init done!')
        if not cuda: self.model.model = self.model.model.cpu()