Python transformable_rdd.TransformableRDD类代码示例

 def test_should_smooth_data_by_Simple_Moving_Average(self):
     initial_dataset = self.sc.parallelize(
             ["52,3,53", "23,4,64", "23,5,64", "23,6,64", "23,7,64", "23,8,64", "23,9,64"], 3)
     transformable_rdd = TransformableRDD(initial_dataset, "csv")
     transformed = transformable_rdd.smooth(1, SimpleMovingAverage(3))
     excepted = 4.0
     self.assertEquals(excepted, transformed.first())

 def test_replace_values_should_replace_cluster_values_with_given_text(self):
     initial_dataset = self.sc.parallelize(["XA,Y", "A,B", "AX,Z", "A,Q", "A,E"])
     transformable_rdd = TransformableRDD(initial_dataset)
     clusters = transformable_rdd.clusters(0, NGramFingerprintAlgorithm(1))
     one_cluster = clusters.get_all_clusters()[0]
     values = transformable_rdd.replace_values(one_cluster, "Hello", 0).collect()
     self.assertTrue(values.__contains__("Hello,B"))

 def test_pivot_table_by_count_should_give_pivoted_table(self):
     initial_dataSet = self.sc.parallelize([
         "known,new,long,home,skips",
         "unknown,new,short,work,reads",
         "unknown,follow Up,long,work,skips",
         "known,follow Up,long,home,skips",
         "known,new,short,home,reads",
         "known,follow Up,long,work,skips",
         "unknown,follow Up,short,work,skips",
         "unknown,new,short,work,reads",
         "known,follow Up,long,home,skips",
         "known,new,long,work,skips",
         "unknown,follow Up,short,home,skips",
         "known,new,long,work,skips",
         "known,follow Up,short,home,reads",
         "known,new,short,work,reads",
         "known,new,short,home,reads",
         "known,follow Up,short,work,reads",
         "known,new,short,home,reads",
         "unknown,new,short,work,reads"
     ])
     initial_rdd = TransformableRDD(initial_dataSet, "csv")
     table = initial_rdd.pivot_by_count(4, [0, 1, 2, 3])
     entry = table.value_at("skips", "known")
     self.assertEqual(6, entry)
     self.assertEqual(3, table.value_at("skips", "unknown"))

    def test_should_split_given_column_indexes_split_by_delimiter_with_retain_column(self):
        initial_data_set = self.sc.parallelize(["FirstName LastName MiddleName,850"])
        initial_rdd = TransformableRDD(initial_data_set, "csv")

        split_with_retained_columns = initial_rdd.split_by_delimiter(0, " ", True)
        self.assertEquals("FirstName LastName MiddleName,850,FirstName,LastName,MiddleName",
                          split_with_retained_columns.first())

 def test_clusters_should_give_clusters_of_given_column_index(self):
     rdd = self.sc.parallelize(["CLUSTER Of Finger print", "finger print of cluster", "finger print for cluster"])
     transformable_rdd = TransformableRDD(rdd, 'csv')
     clusters = transformable_rdd.clusters(0, SimpleFingerprint())
     list_of_clusters = clusters.get_all_clusters()
     one_cluster = list_of_clusters[0]
     self.assertTrue(one_cluster.__contains__("CLUSTER Of Finger print"))
     self.assertFalse(one_cluster.__contains__("finger print for cluster"))

 def test_clusters_should_give_clusters_By_n_gram_fingerprint(self):
     rdd = self.sc.parallelize(["CLUSTER Of Finger print", "finger print of cluster", "finger print for cluster"])
     transformable_rdd = TransformableRDD(rdd, 'csv')
     clusters = transformable_rdd.clusters(0, NGramFingerprintAlgorithm(1))
     list_of_clusters = clusters.get_all_clusters()
     one_cluster = list_of_clusters[0]
     self.assertTrue(one_cluster.__contains__("CLUSTER Of Finger print"))
     self.assertTrue(one_cluster.__contains__("finger print for cluster"))

 def test_multiply_column_should_multiply_two_given_column(self):
     initial_dataset = self.sc.parallelize(["1,1", "1,2", "1,3"])
     transformable_rdd = TransformableRDD(initial_dataset)
     multiplied_rdd = transformable_rdd.multiply_columns(0, 1)
     collected = multiplied_rdd.collect()
     self.assertTrue(collected.__contains__(1.0))
     self.assertTrue(collected.__contains__(2.0))
     self.assertTrue(collected.__contains__(3.0))

 def test_to_double_rdd_should_change_string_to_double_rdd(self):
     initial_dataset = self.sc.parallelize(["1,1", "5,2", "8,3"])
     transformable_rdd = TransformableRDD(initial_dataset)
     rdd = transformable_rdd.to_double_rdd(0)
     collected = rdd.collect()
     self.assertTrue(collected.__contains__(1.0))
     self.assertTrue(collected.__contains__(5.0))
     self.assertTrue(collected.__contains__(8.0))

 def test_map_should_give_Transformable_rdd(self):
     initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
     transformable_rdd = TransformableRDD(initial_dataset, "csv")
     rdd_map = transformable_rdd.map(lambda line: line + "yes")
     deduplicate = rdd_map.deduplicate()
     collected = deduplicate.collect()
     self.assertEqual(2, collected.__len__())
     expected = "1,2yes"
     self.assertTrue(collected.__contains__(expected))

    def test_should_split_given_column_by_field_length_with_retained_columns(self):
        data = ["John,Male,21,+914382313832,Canada", "Smith, Male, 30,+015314343462, UK",
                "Larry, Male, 23,+009815432975, USA", "Fiona, Female,18,+891015709854,USA"]
        initial_data_set = self.sc.parallelize(data)
        initial_rdd = TransformableRDD(initial_data_set, "csv")

        result = initial_rdd.split_by_field_length(3, [3, 10], True).collect()

        self.assertTrue(len(result) == 4)
        self.assertTrue(result.__contains__("John,Male,21,+914382313832,Canada,+91,4382313832"))
        self.assertTrue(result.__contains__("Smith,Male,30,+015314343462,UK,+01,5314343462"))

 def test_transformableRDD_can_impute_the_missing_values_by_NaiveBayesSubstitution(self):
     rdd = self.sc.parallelize(["Drew,No,Blue,Short,Male",
                                "Claudia,Yes,Brown,Long,Female",
                                "Drew,No,Blue,Long,Female",
                                "Drew,No,Blue,Long,Female",
                                "Alberto,Yes,Brown,Short,Male",
                                "Karin,No,Blue,Long,Female",
                                "Nina,Yes,Brown,Short,Female",
                                "Sergio,Yes,Blue,Long,Male",
                                "Drew,Yes,Blue,Long,"])
     transformable_rdd = TransformableRDD(rdd, 'csv')
     imputed_rdd = transformable_rdd.impute(4, NaiveBayesSubstitution(0, 1, 2, 3))
     self.assertTrue(imputed_rdd.collect().__contains__("Drew,Yes,Blue,Long,Female"))

    def test_should_split_the_given_column_by_delimiter_into_given_number_of_split(self):
        data = [
            "John\tMale\t21\t+91-4382-313832\tCanada",
            "Smith\tMale\t30\t+01-5314-343462\tUK",
            "Larry\tMale\t23\t+00-9815-432975\tUSA",
            "Fiona\tFemale\t18\t+89-1015-709854\tUSA"
        ]
        initial_data_set = self.sc.parallelize(data)
        initial_rdd = TransformableRDD(initial_data_set, "tsv")
        new_dataset = initial_rdd.split_by_delimiter(3, "-", False, 2)

        list_of_records = new_dataset.collect()

        self.assertEqual(4, list_of_records.__len__())
        self.assertTrue(list_of_records.__contains__("John\tMale\t21\tCanada\t+91\t4382-313832"))
        self.assertTrue(list_of_records.__contains__("Smith\tMale\t30\tUK\t+01\t5314-343462"))

    def test_should_smooth_data_by_Weighted_Moving_Average(self):
        initial_dataset = self.sc.parallelize(["10", "12", "16", "13", "17", "19", "15", "20", "22", "19", "21", "19"],
                                              3)
        transformable_rdd = TransformableRDD(initial_dataset, "csv")

        weights = Weights(3)
        weights.add(0.166)
        weights.add(0.333)
        weights.add(0.5)

        moving_average = WeightedMovingAverage(3, weights)
        rdd = transformable_rdd.smooth(0, moving_average)

        expected = 13.656
        actual = rdd.first()
        self.assertEquals(expected, actual)

    def test_should_normalize_by_Z_Score_normalization(self):
        initial_dataset = self.sc.parallelize([
            "07434677419,07371326239,Incoming,211,Wed Sep 15 19:17:44 +0100 2010",
            "07641036117,01666472054,Outgoing,0,Mon Feb 11 07:18:23 +0000 1980",
            "07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
            "07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
            "07641036117,07681546436,Missed,12,Mon Feb 11 08:04:42 +0000 1980"])
        transformable_rdd = TransformableRDD(initial_dataset, 'csv')
        final_rdd = transformable_rdd.normalize(3, ZScoreNormalizer())
        normalized_durations = final_rdd.select(3).collect()
        expected1 = "1.944528306701421"
        expected2 = "-0.8202659838241843"
        expected3 = "-0.2306179123850742"
        expected4 = "-0.2306179123850742"
        expected5 = "-0.6630264981070882"

        self.assertTrue(normalized_durations.__contains__(expected1))
        self.assertTrue(normalized_durations.__contains__(expected2))
        self.assertTrue(normalized_durations.__contains__(expected3))
        self.assertTrue(normalized_durations.__contains__(expected4))
        self.assertTrue(normalized_durations.__contains__(expected5))

    def test_should_normalize_by_Decimal_Scale(self):
        initial_dataset = self.sc.parallelize([
            "07434677419,07371326239,Incoming,211,Wed Sep 15 19:17:44 +0100 2010",
            "07641036117,01666472054,Outgoing,0,Mon Feb 11 07:18:23 +0000 1980",
            "07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
            "07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
            "07641036117,07681546436,Missed,12,Mon Feb 11 08:04:42 +0000 1980"])
        transformable_rdd = TransformableRDD(initial_dataset, 'csv')
        final_rdd = transformable_rdd.normalize(3, DecimalScalingNormalizer())
        normalized_durations = final_rdd.select(3).collect()
        expected1 = "2.11"
        expected2 = "0.0"
        expected3 = "0.45"
        expected4 = "0.45"
        expected5 = "0.12"

        self.assertTrue(normalized_durations.__contains__(expected1))
        self.assertTrue(normalized_durations.__contains__(expected2))
        self.assertTrue(normalized_durations.__contains__(expected3))
        self.assertTrue(normalized_durations.__contains__(expected4))
        self.assertTrue(normalized_durations.__contains__(expected5))