本文整理汇总了Python中pyspark.mllib.common._java2py函数的典型用法代码示例。如果您正苦于以下问题:Python _java2py函数的具体用法?Python _java2py怎么用?Python _java2py使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了_java2py函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: load
def load(cls, sc, path):
"""Load a IsotonicRegressionModel."""
java_model = sc._jvm.org.apache.spark.mllib.regression.IsotonicRegressionModel.load(
sc._jsc.sc(), path)
py_boundaries = _java2py(sc, java_model.boundaryVector()).toArray()
py_predictions = _java2py(sc, java_model.predictionVector()).toArray()
return IsotonicRegressionModel(py_boundaries, py_predictions, java_model.isotonic)
示例2: load
def load(cls, sc, path):
java_model = sc._jvm.org.apache.spark.mllib.classification.NaiveBayesModel.load(
sc._jsc.sc(), path)
py_labels = _java2py(sc, java_model.labels())
py_pi = _java2py(sc, java_model.pi())
py_theta = _java2py(sc, java_model.theta())
return NaiveBayesModel(py_labels, py_pi, numpy.array(py_theta))
示例3: __init__
def __init__(self, regressionCoeff=None, d=0, q=0, coefficients=None, hasIntercept=False, jmodel=None, sc=None):
"""
Parameters
----------
regressionCoeff:
coefficients for regression , including intercept , for example. if model has 3 regressors
then length of [[regressionCoeff]] is 4
arimaOrders:
p,d,q for the arima error structure, length of [[arimaOrders]] must be 3
arimaCoeff:
AR, d, and MA terms, length of arimaCoeff = p+d+q
"""
assert sc != None, "Missing SparkContext"
self._ctx = sc
if jmodel == None:
self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.RegressionARIMAModel(
_py2java_double_array(self._ctx, regressionCoeff),
_py2java_int_array(self._ctx, arimaOrders),
_py2scala_arraybuffer(self._ctx, arimaCoeff),
)
else:
self._jmodel = jmodel
self.regressionCoeff = _java2py(sc, self._jmodel.regressionCoeff())
self.arimaOrders = _java2py(sc, self._jmodel.arimaOrders())
self.arimaCoeff = _java2py(sc, self._jmodel.arimaCoeff())
示例4: load
def load(cls, sc, path):
java_model = sc._jvm.org.apache.spark.mllib.classification.NaiveBayesModel.load(
sc._jsc.sc(), path)
# Can not unpickle array.array from Pyrolite in Python3 with "bytes"
py_labels = _java2py(sc, java_model.labels(), "latin1")
py_pi = _java2py(sc, java_model.pi(), "latin1")
py_theta = _java2py(sc, java_model.theta(), "latin1")
return NaiveBayesModel(py_labels, py_pi, numpy.array(py_theta))
示例5: __init__
def __init__(self, p=0, d=0, q=0, coefficients=None, hasIntercept=False, jmodel=None, sc=None):
self._ctx = sc
if jmodel == None:
self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.ARIMAModel(p, d, q, _py2java(self._ctx, coefficients), hasIntercept)
else:
self._jmodel = jmodel
self.p = _java2py(sc, self._jmodel.p())
self.d = _java2py(sc, self._jmodel.d())
self.q = _java2py(sc, self._jmodel.q())
self.coefficients = _java2py(sc, self._jmodel.coefficients())
self.has_intercept = _java2py(sc, self._jmodel.hasIntercept())
示例6: __init__
def __init__(self, p=0, d=0, q=0, coefficients=None, hasIntercept=True, jmodel=None, sc=None):
assert sc != None, "Missing SparkContext"
self._ctx = sc
if jmodel == None:
self._jmodel = self._ctx._jvm.com.cloudera.sparkts.models.ARIMAModel(p, d, q, _py2java_double_array(self._ctx, coefficients), hasIntercept)
else:
self._jmodel = jmodel
self.p = _java2py(sc, self._jmodel.p())
self.d = _java2py(sc, self._jmodel.d())
self.q = _java2py(sc, self._jmodel.q())
self.coefficients = _java2py(sc, self._jmodel.coefficients())
self.has_intercept = _java2py(sc, self._jmodel.hasIntercept())
示例7: gradient_log_likelihood_css_arma
def gradient_log_likelihood_css_arma(self, diffedy):
"""
Calculates the gradient for the log likelihood function using CSS
Derivation:
L(y | \theta) = -\frac{n}{2}log(2\pi\sigma^2) - \frac{1}{2\pi}\sum_{i=1}^n \epsilon_t^2 \\
\sigma^2 = \frac{\sum_{i = 1}^n \epsilon_t^2}{n} \\
\frac{\partial L}{\partial \theta} = -\frac{1}{\sigma^2}
\sum_{i = 1}^n \epsilon_t \frac{\partial \epsilon_t}{\partial \theta} \\
\frac{\partial \epsilon_t}{\partial \theta} = -\frac{\partial \hat{y}}{\partial \theta} \\
\frac{\partial\hat{y}}{\partial c} = 1 +
\phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial c} \\
\frac{\partial\hat{y}}{\partial \theta_{ar_i}} = y_{t - i} +
\phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial \theta_{ar_i}} \\
\frac{\partial\hat{y}}{\partial \theta_{ma_i}} = \epsilon_{t - i} +
\phi_{t-q}^{t-1}*\frac{\partial \epsilon_{t-q}^{t-1}}{\partial \theta_{ma_i}} \\
Parameters
----------
diffedY:
array of differenced values
returns the gradient log likelihood as an array of double
"""
# need to copy diffedy to a double[] for Java
result = self._jmodel.gradientlogLikelihoodCSSARMA(_py2java_double_array(self._ctx, diffedy))
return _java2py(self._ctx, result)
示例8: load
def load(cls, sc, path):
java_model = sc._jvm.org.apache.spark.mllib.regression.RidgeRegressionModel.load(
sc._jsc.sc(), path)
weights = _java2py(sc, java_model.weights())
intercept = java_model.intercept()
model = RidgeRegressionModel(weights, intercept)
return model
示例9: _call_java
def _call_java(sc, java_obj, name, *args):
"""
Method copied from pyspark.ml.wrapper. Uses private Spark APIs.
"""
m = getattr(java_obj, name)
java_args = [_py2java(sc, arg) for arg in args]
return _java2py(sc, m(*java_args))
示例10: forecast
def forecast(self, ts, nfuture):
"""
Provided fitted values for timeseries ts as 1-step ahead forecasts, based on current
model parameters, and then provide `nFuture` periods of forecast. We assume AR terms
prior to the start of the series are equal to the model's intercept term (or 0.0, if fit
without and intercept term).Meanwhile, MA terms prior to the start are assumed to be 0.0. If
there is differencing, the first d terms come from the original series.
Parameters
----------
ts:
Timeseries to use as gold-standard. Each value (i) in the returning series
is a 1-step ahead forecast of ts(i). We use the difference between ts(i) -
estimate(i) to calculate the error at time i, which is used for the moving
average terms. Numpy array.
nFuture:
Periods in the future to forecast (beyond length of ts)
Returns a series consisting of fitted 1-step ahead forecasts for historicals and then
`nFuture` periods of forecasts. Note that in the future values error terms become
zero and prior predictions are used for any AR terms.
"""
jts = _py2java(self._ctx, Vectors.dense(ts))
jfore = self._jmodel.forecast(jts, nfuture)
return _java2py(self._ctx, jfore)
示例11: perform_pca
def perform_pca(matrix, row_count, nr_principal_components=2):
"""Return principal components of the input matrix.
This function uses MLlib's ``RowMatrix`` to compute principal components.
Args:
matrix: An RDD[int, (int, float)] representing a sparse matrix. This
is returned by ``center_matrix`` but it is not required to center
the matrix first.
row_count: The size (N) of the N x N ``matrix``.
nr_principal_components: Number of components we want to obtain. This
value must be less than or equal to the number of rows in the input
square matrix.
Returns:
An array of ``nr_principal_components`` columns, and same number of rows
as the input ``matrix``. This array is a ``numpy`` array.
"""
py_rdd = matrix.map(lambda row: linalg.Vectors.sparse(row_count, row))
sc = pyspark.SparkContext._active_spark_context
java_rdd = mllib_common._py2java(sc, py_rdd)
scala_rdd = java_rdd.rdd()
sc = pyspark.SparkContext._active_spark_context
row_matrix = (sc._jvm.org.apache.spark.mllib.linalg.distributed.
RowMatrix(scala_rdd)
)
pca = row_matrix.computePrincipalComponents(nr_principal_components)
pca = mllib_common._java2py(sc, pca)
return pca.toArray()
示例12: log_likelihood
def log_likelihood(self, ts):
"""
Returns the log likelihood of the parameters on the given time series.
Based on http://www.unc.edu/~jbhill/Bollerslev_GARCH_1986.pdf
"""
likelihood = self._jmodel.logLikelihood(_py2java(self._ctx, Vectors.dense(ts)))
return _java2py(self._ctx, likelihood)
示例13: to_double_rdd
def to_double_rdd(self, column_index):
"""
Returns a RDD by converting values to double of given column index
:param column_index:One column index in TransformableRDD
:return:RDD
"""
rdd = self._transformable_rdd.toDoubleRDD(column_index).rdd()
return _java2py(self.spark_context, rdd)
示例14: multiply_columns
def multiply_columns(self, first_column, second_column):
"""
Returns a RDD which is a product of the values in @first_column and @second_column
:param first_column: One column index
:param second_column: Another column index
:return: RDD
"""
_rdd = self._transformable_rdd.multiplyColumns(first_column, second_column).rdd()
return _java2py(self.spark_context, _rdd)
示例15: smooth
def smooth(self, column_index, smoothing_method):
"""
Returns a new RDD containing smoothed values of @column_index using @smoothing_method
:param column_index: Index of the column
:param smoothing_method: smoothing method by which you want to smooth data
:return: RDD
"""
method = smoothing_method._get_smoothing_method(self.spark_context)
rdd = self._transformable_rdd.smooth(column_index, method)
return _java2py(self.spark_context, rdd.rdd())