Python DenseVector.dot方法代码示例

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
 def test_dot(self):
     sv = SparseVector(4, {1: 1, 3: 2})
     dv = DenseVector(array([1.0, 2.0, 3.0, 4.0]))
     lst = DenseVector([1, 2, 3, 4])
     mat = array([[1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 3.0, 4.0]])
     self.assertEquals(10.0, sv.dot(dv))
     self.assertTrue(array_equal(array([3.0, 6.0, 9.0, 12.0]), sv.dot(mat)))
     self.assertEquals(30.0, dv.dot(dv))
     self.assertTrue(array_equal(array([10.0, 20.0, 30.0, 40.0]), dv.dot(mat)))
     self.assertEquals(30.0, lst.dot(dv))
     self.assertTrue(array_equal(array([10.0, 20.0, 30.0, 40.0]), lst.dot(mat)))

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
 def test_dot(self):
     sv = SparseVector(4, {1: 1, 3: 2})
     dv = DenseVector(array([1., 2., 3., 4.]))
     lst = DenseVector([1, 2, 3, 4])
     mat = array([[1., 2., 3., 4.],
                  [1., 2., 3., 4.],
                  [1., 2., 3., 4.],
                  [1., 2., 3., 4.]])
     arr = pyarray.array('d', [0, 1, 2, 3])
     self.assertEqual(10.0, sv.dot(dv))
     self.assertTrue(array_equal(array([3., 6., 9., 12.]), sv.dot(mat)))
     self.assertEqual(30.0, dv.dot(dv))
     self.assertTrue(array_equal(array([10., 20., 30., 40.]), dv.dot(mat)))
     self.assertEqual(30.0, lst.dot(dv))
     self.assertTrue(array_equal(array([10., 20., 30., 40.]), lst.dot(mat)))
     self.assertEqual(7.0, sv.dot(arr))

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
    def get_ratings(self, res_id, ratings, top):
        if res_id not in self.models.keys():
            logger.info("Keys: " + str(self.models.keys()))
            logger.info("Key Type: " + str(type(self.models.keys()[0])))
            logger.info("res_id: " + str(res_id))
            logger.info("res_id type:" + str(type(res_id)))
            logger.info("res_id not known")
            return []
        
        pf = self.models[res_id].productFeatures()
         
        user_pf = pf.filter(lambda x: x[0] in ratings)
        if len(user_pf.collect()) == 0:
            logger.info("No product matches")
            return []
        user_f = user_pf.collect()
        tmp = DenseVector(user_f[0][1])
        for i in xrange(1, len(user_f)):
            tmp = tmp + user_f[i][1]
        #user_f = user_pf.reduce(lambda x, y : DenseVector(x[1]) + DenseVector(y[1]))
        estimate_score = pf.map(lambda x: (x[0], tmp.dot(DenseVector(x[1])))).filter(lambda x: x[0] not in ratings).takeOrdered(top, lambda (k,v): -v)
 
        #estimate_score = pf.map(lambda x: (x[0], DenseVector(user_f).dot(DenseVector(x[1])))).filter(lambda x: x[0] not in ratings).takeOrdered(top, lambda (k,v): -v)
        estimate_pid = map(lambda x: x[0], estimate_score)
        
        return estimate_pid

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
def gradientSummand(weights, lp):
    """Calculates the gradient summand for a given weight and `LabeledPoint`.

    Note:
        `DenseVector` behaves similarly to a `numpy.ndarray` and they can be used interchangably
        within this function.  For example, they both implement the `dot` method.

    Args:
        weights (DenseVector): An array of model weights (betas).
        lp (LabeledPoint): The `LabeledPoint` for a single observation.

    Returns:
        DenseVector: An array of values the same length as `weights`.  The gradient summand.
    """
    return (DenseVector.dot(lp.features, weights) - lp.label) * lp.features

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
def getLabeledPrediction(weights, observation):
    """Calculates predictions and returns a (label, prediction) tuple.

    Note:
        The labels should remain unchanged as we'll use this information to calculate prediction
        error later.

    Args:
        weights (np.ndarray): An array with one weight for each features in `trainData`.
        observation (LabeledPoint): A `LabeledPoint` that contain the correct label and the
            features for the data point.

    Returns:
        tuple: A (label, prediction) tuple.
    """
    return (observation.label, DenseVector.dot(observation.features, weights))

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
# In[22]:

from pyspark.mllib.linalg import DenseVector


# In[25]:

# TODO: Replace <FILL IN> with appropriate code
numpyVector = np.array([-3, -4, 5])
print '\nnumpyVector:\n{0}'.format(numpyVector)

# Create a DenseVector consisting of the values [3.0, 4.0, 5.0]
myDenseVector = DenseVector([3.0, 4.0, 5.0])
# Calculate the dot product between the two vectors.
denseDotProduct = myDenseVector.dot(numpyVector)

print 'myDenseVector:\n{0}'.format(myDenseVector)
print '\ndenseDotProduct:\n{0}'.format(denseDotProduct)


# In[26]:

# TEST PySpark's DenseVector (3c)
Test.assertTrue(isinstance(myDenseVector, DenseVector), 'myDenseVector is not a DenseVector')
Test.assertTrue(np.allclose(myDenseVector, np.array([3., 4., 5.])),
                'incorrect value for myDenseVector')
Test.assertTrue(np.allclose(denseDotProduct, 0.0), 'incorrect value for denseDotProduct')


# ### ** Part 4: Python lambda expressions **

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
# In[29]:

from pyspark.mllib.linalg import DenseVector


# In[31]:

# TODO: Replace <FILL IN> with appropriate code
numpyVector = np.array([-3, -4, 5])
print '\nnumpyVector:\n{0}'.format(numpyVector)

# Create a DenseVector consisting of the values [3.0, 4.0, 5.0]
myDenseVector = DenseVector(np.array([3.0,4.0,5.0]))
# Calculate the dot product between the two vectors.
denseDotProduct = DenseVector.dot(myDenseVector, numpyVector)

print 'myDenseVector:\n{0}'.format(myDenseVector)
print '\ndenseDotProduct:\n{0}'.format(denseDotProduct)


# In[32]:

# TEST PySpark's DenseVector (3c)
Test.assertTrue(isinstance(myDenseVector, DenseVector), 'myDenseVector is not a DenseVector')
Test.assertTrue(np.allclose(myDenseVector, np.array([3., 4., 5.])),
                'incorrect value for myDenseVector')
Test.assertTrue(np.allclose(denseDotProduct, 0.0), 'incorrect value for denseDotProduct')


# ### ** Part 4: Python lambda expressions **

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
# In[97]:

from pyspark.mllib.linalg import DenseVector


# In[98]:

# TODO: Replace <FILL IN> with appropriate code
numpyVector = np.array([-3, -4, 5])
print '\nnumpyVector:\n{0}'.format(numpyVector)

# Create a DenseVector consisting of the values [3.0, 4.0, 5.0]
myDenseVector = DenseVector(np.array([3.0,4.0,5.0]))
# Calculate the dot product between the two vectors.
denseDotProduct = DenseVector.dot(DenseVector(numpyVector),myDenseVector)

print 'myDenseVector:\n{0}'.format(myDenseVector)
print '\ndenseDotProduct:\n{0}'.format(denseDotProduct)


# In[99]:

# TEST PySpark's DenseVector (3c)
Test.assertTrue(isinstance(myDenseVector, DenseVector), 'myDenseVector is not a DenseVector')
Test.assertTrue(np.allclose(myDenseVector, np.array([3., 4., 5.])),
                'incorrect value for myDenseVector')
Test.assertTrue(np.allclose(denseDotProduct, 0.0), 'incorrect value for denseDotProduct')


# ### ** Part 4: Python lambda expressions **

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
# In[28]:

from pyspark.mllib.linalg import DenseVector


# In[31]:

# TODO: Replace <FILL IN> with appropriate code
numpyVector = np.array([-3, -4, 5])
print "\nnumpyVector:\n{0}".format(numpyVector)

# Create a DenseVector consisting of the values [3.0, 4.0, 5.0]
myDenseVector = DenseVector([3.0, 4.0, 5.0])  # <FILL IN>
# Calculate the dot product between the two vectors.
denseDotProduct = myDenseVector.dot(DenseVector(numpyVector))  # <FILL IN>

print "myDenseVector:\n{0}".format(myDenseVector)
print "\ndenseDotProduct:\n{0}".format(denseDotProduct)


# In[32]:

# TEST PySpark's DenseVector (3c)
Test.assertTrue(isinstance(myDenseVector, DenseVector), "myDenseVector is not a DenseVector")
Test.assertTrue(np.allclose(myDenseVector, np.array([3.0, 4.0, 5.0])), "incorrect value for myDenseVector")
Test.assertTrue(np.allclose(denseDotProduct, 0.0), "incorrect value for denseDotProduct")


# ### ** Part 4: Python lambda expressions **

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
expectedError = [79.72013547, 30.27835699,  9.27842641,  9.20967856,  9.19446483]
Test.assertTrue(np.allclose(exampleErrorTrain, expectedError),
                'value of exampleErrorTrain is incorrect')


# #### ** (3d) Train the model **
# #### Now let's train a linear regression model on all of our training data and evaluate its accuracy on the validation set.  Note that the test set will not be used here.  If we evaluated the model on the test set, we would bias our final results.
# #### We've already done much of the required work: we computed the number of features in Part (1b); we created the training and validation datasets and computed their sizes in Part (1e); and, we wrote a function to compute RMSE in Part (2b).

# In[44]:

# TODO: Replace <FILL IN> with appropriate code
numIters = 50
weightsLR0, errorTrainLR0 = linregGradientDescent(parsedTrainData, numIters);

labelsAndPreds = parsedValData.map(lambda lp: (lp.label,DenseVector.dot(weightsLR0,lp.features)))
rmseValLR0 = calcRMSE(labelsAndPreds)

print 'Validation RMSE:\n\tBaseline = {0:.3f}\n\tLR0 = {1:.3f}'.format(rmseValBase,
                                                                       rmseValLR0)


# In[45]:

# TEST Train the model (3d)
expectedOutput = [22.64535883, 20.064699, -0.05341901, 8.2931319, 5.79155768, -4.51008084,
                  15.23075467, 3.8465554, 9.91992022, 5.97465933, 11.36849033, 3.86452361]
Test.assertTrue(np.allclose(weightsLR0, expectedOutput), 'incorrect value for weightsLR0')


# #### ** Visualization 4: Training error **

# 需要导入模块: from pyspark.mllib.linalg import DenseVector [as 别名]
# 或者: from pyspark.mllib.linalg.DenseVector import dot [as 别名]
zeros = np.zeros(8) # returns an array of 8 0s [ 0.  0.  0.  0.  0.  0.  0.  0.]
ones = np.ones(8) # returns an array of 8 1s [ 1.  1.  1.  1.  1.  1.  1.  1.]
print 'zeros:\n{0}'.format(zeros)
print '\nones:\n{0}'.format(ones)

zerosThenOnes = np.hstack((zeros,ones))   #notice the "(("
# hstack will return [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  1.  1.  1.  1.  1.  1.  1.]
zerosAboveOnes = np.vstack((zeros,ones))   # A 2 by 8 array 
# vstack in the above example will return 	[[ 0.  0.  0.  0.  0.  0.  0.  0.]
# 											[ 1.  1.  1.  1.  1.  1.  1.  1.]]

print '\nzerosThenOnes:\n{0}'.format(zerosThenOnes)
print '\nzerosAboveOnes:\n{0}'.format(zerosAboveOnes)

# When using PySpark, we use DenseVector instead of numpy vector. Example below:

from pyspark.mllib.linalg import DenseVector

numpyVector = np.array([-3, -4, 5])
print '\nnumpyVector:\n{0}'.format(numpyVector)

# Create a DenseVector consisting of the values [3.0, 4.0, 5.0]
myDenseVector = DenseVector([3.0, 4.0, 5.0])
# Calculate the dot product between the two vectors.
denseDotProduct = DenseVector.dot(myDenseVector, numpyVector) # DenseVector.dot() does the dot product

print 'myDenseVector:\n{0}'.format(myDenseVector)
print '\ndenseDotProduct:\n{0}'.format(denseDotProduct)