Python cma.CMAEvolutionStrategy方法代码示例

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def __init__(self, num_params,      # number of model parameters
               sigma_init=0.10,       # initial standard deviation
               popsize=255,           # population size
               weight_decay=0.01):    # weight decay coefficient

    self.num_params = num_params
    self.sigma_init = sigma_init
    self.popsize = popsize
    self.weight_decay = weight_decay
    self.solutions = None

    import cma
    self.es = cma.CMAEvolutionStrategy( self.num_params * [0],
                                        self.sigma_init,
                                        {'popsize': self.popsize,
                                        }) 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def ask(self, trials, last_told_trial_number):
        # type: (List[FrozenTrial], int) -> Dict[str, Any]

        individual_index = len(self._collect_target_trials(trials, last_told_trial_number))
        popsize = self._es.popsize

        # individual_index may exceed the population size due to the parallel execution of multiple
        # trials. In such cases, `cma.cma.CMAEvolutionStrategy.ask` is called multiple times in an
        # iteration, and that may affect the optimization performance of CMA-ES.
        # In addition, please note that some trials may suggest the same parameters when multiple
        # samplers invoke this method simultaneously.
        while individual_index >= popsize:
            individual_index -= popsize
            self._es.ask()
        cma_params = self._es.ask()[individual_index]

        ret_val = {}
        for param_name, value in zip(self._param_names, cma_params):
            ret_val[param_name] = self._to_optuna_params(self._search_space, param_name, value)
        return ret_val 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def es(self) -> tp.Any:  # typing not possible since cmaes not imported :(
        if self._es is None:
            if not self._fcmaes:
                inopts = {"popsize": self._popsize, "randn": self._rng.randn, "CMA_diagonal": self._diagonal, "verbose": 0}
                self._es = cma.CMAEvolutionStrategy(x0=self._rng.normal(size=self.dimension) if self._random_init else np.zeros(self.dimension, dtype=np.float), sigma0=self._scale, inopts=inopts)
            else:
                try:
                    from fcmaes import cmaes  # pylint: disable=import-outside-toplevel
                except ImportError as e:
                    raise ImportError("Please install fcmaes (pip install fcmaes) to use FCMA optimizers") from e
                self._es = cmaes.Cmaes(x0=np.zeros(self.dimension, dtype=np.float),
                                       input_sigma=self._scale,
                                       popsize=self._popsize, randn=self._rng.randn)
        return self._es 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def train(self, runner):
        """Initialize variables and start training.

        Args:
            runner (LocalRunner): LocalRunner is passed to give algorithm
                the access to runner.step_epochs(), which provides services
                such as snapshotting and sampler control.

        Returns:
            float: The average return in last epoch cycle.

        """
        init_mean = self.policy.get_param_values()
        self._es = cma.CMAEvolutionStrategy(init_mean, self._sigma0,
                                            {'popsize': self._n_samples})
        self._all_params = self._sample_params()
        self._cur_params = self._all_params[0]
        self.policy.set_param_values(self._cur_params)
        self._all_returns = []

        # start actual training
        last_return = None

        for _ in runner.step_epochs():
            for _ in range(self._n_samples):
                runner.step_path = runner.obtain_samples(runner.step_itr)
                last_return = self.train_once(runner.step_itr,
                                              runner.step_path)
                runner.step_itr += 1

        return last_return 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def __init__(self, expt_dir):

        raise NotImplementedError('The CMA chooser is not yet implemented!')
        
        self.state_pkl = os.path.join(expt_dir, self.__module__ + ".pkl")

        #TODO: params needs to be an array of starting values
        # - need to figure out how to map Spearmint params into
        # all floats usable by the evolution strategy.
        self.optimizer = CMAEvolutionStrategy(params) 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def __init__(
            self, seed, policy, init_sigma, population_size, l2_coefficient):
        """Initialization.

        Instantiate the wrapper.

        Args:
            seed: int. Initial random seed.
            policy: An instance of ars.policy. The policy to optimize.
            init_sigma: Float. Initial standard deviation value.
            population_size: Integer. Size of the population.
            l2_coefficient: Float. Coefficient for the L2 regularization,
                a negative value disables L2 regularization.
        """
        self._policy = policy
        self._init_sigma = init_sigma
        self._population_size = population_size
        self._l2_coefficient = l2_coefficient
        self._solutions = None
        self._cma = cma.CMAEvolutionStrategy(
            self._policy.get_params(),
            self._init_sigma,
            inopts={'popsize': self._population_size,
                    'seed': seed if seed > 0 else 42,  # cma ignores if seed==0
                    'randn': np.random.randn})
        self._set_seed(seed) 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def train(self, args, callback, env_kwargs=None, train_kwargs=None):
        args.num_cpu = args.num_population
        env = self.makeEnv(args, env_kwargs=env_kwargs)

        args.__dict__.update(train_kwargs)

        if args.continuous_actions:
            action_space = np.prod(env.action_space.shape)
        else:
            action_space = env.action_space.n

        if args.srl_model != "raw_pixels":
            net = MLPPolicyPytorch(np.prod(env.observation_space.shape), [100], action_space)
        else:
            net = CNNPolicyPytorch(env.observation_space.shape[-1], action_space)

        self.policy = PytorchPolicy(net, args.continuous_actions, srl_model=(args.srl_model != "raw_pixels"),
                                    cuda=args.cuda, deterministic=args.deterministic)
        self.n_population = args.num_population
        self.mu = args.mu
        self.sigma = args.sigma
        self.continuous_actions = args.continuous_actions
        self.es = cma.CMAEvolutionStrategy(self.policy.getParamSpace() * [self.mu], self.sigma,
                                           {'popsize': self.n_population})
        self.best_model = np.array(self.policy.getParamSpace() * [self.mu])
        num_updates = int(args.num_timesteps)

        start_time = time.time()
        step = 0
        while step < num_updates:
            obs = env.reset()
            r = np.zeros((self.n_population,))
            # here, CMAEvolutionStrategy will return a list of param for each of the population
            population = self.es.ask()
            done = np.full((self.n_population,), False)
            while not done.all():
                actions = []
                for k in range(self.n_population):
                    if not done[k]:
                        self.policy.setParam(population[k])
                        action = self.policy.getAction(np.array([obs[k]]))[0]
                        actions.append(action)
                    else:
                        actions.append(None)  # do nothing, as we are done

                obs, reward, new_done, info = env.step(actions)
                step += np.sum(~done)

                done = np.bitwise_or(done, new_done)

                # cumulate the reward for every enviroment that is not finished
                r[~done] += reward[~done]

                if callback is not None:
                    callback(locals(), globals())

            print("{} steps - {:.2f} FPS".format(step, step / (time.time() - start_time)))
            self.es.tell(population, -r)
            self.best_model = self.es.result.xbest 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def run_cma_es(params, iterations=100):
    """Run CMA-ES optimization.

    :param params:
    :param iterations:
    :return:
    """
    start_prob = 0
    best_iter = 0
    best_prob = 0
    best_params = None
    first_hit = -1
    param_array = dict2array(params)
    es = cma.CMAEvolutionStrategy(param_array, 1.0, {"popsize": 18})
    for current_iter in range(iterations):
        logging.info(current_iter)
        population = es.ask()
        pop_params = [array2dict(individual) for individual in population]
        pop_fitnesses = []
        pop_probs = []
        pop_labels = []
        for params in pop_params:
            set_all_params(params)
            image = RENDERER.render()
            with torch.no_grad():
                out = MODEL(image)

            loss = CRITERION(out, LABELS).item()
            pop_fitnesses.append(loss)

            probs = torch.nn.functional.softmax(out, dim=1)
            probs_np = probs[0].detach().cpu().numpy()
            target_prob = probs_np[TARGET_CLASS]
            pop_probs.append(target_prob)
            max_index = probs_np.argmax()
            pop_labels.append(max_index)

        es.tell(population, pop_fitnesses)

        max_individual = np.argmax(pop_probs)
        target_prob = pop_probs[max_individual]
        if current_iter == 0:
            start_prob = target_prob

        if target_prob > best_prob:
            best_iter = current_iter
            best_prob = target_prob
            best_params = pop_params[max_individual]

        logging.info(target_prob)
        logging.info(start_prob)
        logging.info(best_prob)

        max_index = pop_labels[max_individual]
        if first_hit == -1 and max_index == TARGET_CLASS:
            first_hit = current_iter

    return (best_prob, best_iter, best_params, first_hit) 

# 需要导入模块: import cma [as 别名]
# 或者: from cma import CMAEvolutionStrategy [as 别名]
def __init__(
        self,
        search_space,  # type: Dict[str, BaseDistribution]
        x0,  # type: Dict[str, Any]
        sigma0,  # type: float
        cma_stds,  # type: Optional[Dict[str, float]]
        cma_opts,  # type: Dict[str, Any]
    ):
        # type: (...) -> None

        self._search_space = search_space
        self._param_names = list(sorted(self._search_space.keys()))

        lows = []
        highs = []
        for param_name in self._param_names:
            dist = self._search_space[param_name]
            if isinstance(dist, CategoricalDistribution):
                # Handle categorical values by ordinal representation.
                # TODO(Yanase): Support one-hot representation.
                lows.append(-0.5)
                highs.append(len(dist.choices) - 0.5)
            elif isinstance(dist, UniformDistribution) or isinstance(dist, LogUniformDistribution):
                lows.append(self._to_cma_params(search_space, param_name, dist.low))
                highs.append(self._to_cma_params(search_space, param_name, dist.high))
            elif isinstance(dist, DiscreteUniformDistribution):
                r = dist.high - dist.low
                lows.append(0 - 0.5 * dist.q)
                highs.append(r + 0.5 * dist.q)
            elif isinstance(dist, IntUniformDistribution):
                lows.append(dist.low - 0.5 * dist.step)
                highs.append(dist.high + 0.5 * dist.step)
            elif isinstance(dist, IntLogUniformDistribution):
                lows.append(self._to_cma_params(search_space, param_name, dist.low - 0.5))
                highs.append(self._to_cma_params(search_space, param_name, dist.high + 0.5))
            else:
                raise NotImplementedError("The distribution {} is not implemented.".format(dist))

        # Set initial params.
        initial_cma_params = []
        for param_name in self._param_names:
            initial_cma_params.append(
                self._to_cma_params(self._search_space, param_name, x0[param_name])
            )
        cma_option = {
            "BoundaryHandler": cma.BoundTransform,
            "bounds": [lows, highs],
        }

        if cma_stds:
            cma_option["CMA_stds"] = [cma_stds.get(name, 1.0) for name in self._param_names]

        cma_opts.update(cma_option)

        self._es = cma.CMAEvolutionStrategy(initial_cma_params, sigma0, cma_opts)