Python SparkContext.binaryFiles方法代码示例

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
def main(argv):
    logging.config.fileConfig(os.path.join(os.path.dirname(os.path.realpath(__file__)), "logging.ini"))
    parsed_args = parse_args(argv)
    spark_conf = SparkConf()
    sc = SparkContext(conf=spark_conf)
    with open(parsed_args.config) as in_config:
        preprocess_conf = json.load(in_config)
    if preprocess_conf.get("binary_input", True):
        files = sc.binaryFiles(preprocess_conf["input"], preprocess_conf.get('partitions', 4000))
    else:
        files = sc.wholeTextFiles(preprocess_conf["input"], preprocess_conf.get('partitions', 4000))
    files = files.repartition(preprocess_conf.get('partitions', 4000))
    metadata = parse_metadata(preprocess_conf["labeled"]["metadata"])
    labeled = sc.textFile(preprocess_conf["labeled"]["file"], preprocess_conf.get('partitions', 4000)).\
                          map(lambda x: parse_labeled_line(x, metadata, True)).filter(lambda x: x.iloc[0]["label"] != 4).map(transform_labels)
    header, resampled = prep.preprocess(sc, files, labeled, label=preprocess_conf.get('label', True),
                                        cut=preprocess_conf.get("cut", {"low": 6300, "high": 6700}),
                                pca=preprocess_conf.get("pca", None), partitions=preprocess_conf.get('partitions', 100))
    resampled.map(lambda x: x.to_csv(None, header=None).rstrip("\n")).saveAsTextFile(preprocess_conf["output"])

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
if __name__ == '__main__':
    from pyspark import SparkContext, SparkConf

    parser = argparse.ArgumentParser()

    parser.add_argument('src_tif_dir', help='Directory with files to reproject')
    parser.add_argument('dst_dir', help='Directory to write reproject files')
    parser.add_argument('--data-name', help='Optional identifer to prefix files with', default='')
    parser.add_argument('--dst-crs', help='CRS to reproject files to', default='EPSG:3857')
    parser.add_argument('--extension', help='Only consider files ending in this extension', default='')
    parser.add_argument('--region', help='Region for the S3 client to use', default='')

    args = parser.parse_args()

    spark_conf = SparkConf().setAppName('Rainfall-Reprojection')
    sc = SparkContext(conf=spark_conf)

    raw_tifs = sc.binaryFiles(args.src_tif_dir)

    if args.extension:
        raw_tifs = raw_tifs.filter(lambda (path, _): path.endswith(args.extension))

    reprojected_tifs = raw_tifs.map(
        lambda (src_tif_path_remote, tif_bytes): process_tif(
            src_tif_path_remote, tif_bytes, args.data_name, args.dst_crs, args.dst_dir, args.region
        )
    )

    num_reprojected = reprojected_tifs.count()

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]

# (field0, series.y)
def eval_flow_cde(x):
    return eval_flow_spark(x, bc_output_dir.value)

def plotImage(x):
    t0 = time.time()
    distribution_plot_subsets_spark(x[1], bc_output_dir.value)
    print "plotImage used:  ", t0 - time.time()
    # print("-----"+x[0],x[1])

# hdfs
startTime = time.time()
t0 = time.time()
hdfsFile = sc.binaryFiles(input_dir).persist(StorageLevel.MEMORY_AND_DISK)
#########################################################Total Time Used: 8.73000001907
adaf_objs = hdfsFile.map(read_dat_hdfs)\
    .map(ExtractVIN) \
    .map(process_dat_adaf).map(sort_adaf) \
    .map(vehical_config) \
    .filter(do_filter)\
    .map(subsetMetaData)
#########################################################Total Time Used: 25.1680002213

# rdd_vehical_config = hdfsFile.map(read_dat_hdfs) \
#     .map(ExtractVIN) \
#     .map(process_dat_adaf).map(sort_adaf) \
#     .map(vehical_config)
# rdd_vehical_config.count()
# print("rdd_vehical_config Used Time: {}".format(time.time() - t0))

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
import re
import os
import sys
import numpy as np

srtm_dtype = np.dtype('>i2')
filename_regex = re.compile('([NSEW]\d+[NSEW]\d+).*')

# The data directory, needs to be available to all node in the cluster
data_files = '/media/bitbucket/srtm/version2_1/SRTM3/North_America'

# Build up the context, using the master URL
sc = SparkContext('spark://ulex:7077', 'srtm')

# Now load all the zip files into a RDD
data = sc.binaryFiles(data_files)

# The two accumulators are used to collect values across the cluster
num_samples_acc = sc.accumulator(0)
sum_acc = sc.accumulator(0)

# Function to array
def read_array(data):
    hgt_2darray = np.flipud(np.fromstring(data, dtype=srtm_dtype).reshape(1201, 1201))

    return hgt_2darray

# Function to process a HGT file
def process_file(file):
    (name, content) = file

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
import boto
import datetime

sc = SparkContext()

# AWS S3 credentials:

AWS_KEY = ""
AWS_SECRET = ""
sc._jsc.hadoopConfiguration().set("fs.s3n.awsAccessKeyId", AWS_KEY)
sc._jsc.hadoopConfiguration().set("fs.s3n.awsSecretAccessKey", AWS_SECRET)


directory = 's3n://amlyelp/subset/trainnew/'

images = sc.binaryFiles(directory)

image_to_array = lambda rawdata: np.asarray(Image.open(StringIO(rawdata)))

image_array = images.map(lambda x: (x[0],image_to_array(x[1])))

image_array_flatten = image_array.map(lambda x: (x[0],x[1].flatten())).cache()
del image_array
del images

train = image_array_flatten.values().repartition(200).cache()

clusters = KMeans.train(train, 50, maxIterations=50)

clusters.save(sc, 's3n://amlyelp/subset/model/kmeans/50_iters_'+\
              str(datetime.datetime.now()).replace(' ', '_')+'/')

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
        '--partitions', default=250, type=int,
        help=('Number of partitions to coalesce geotiffs to else '
              'each geotiff will end up in its own partition'))
    parser.add_argument(
        '--sampling-method', default="nearest",
        choices=SAMPLING_METHODS.keys(),
        help=('Sampling method to use during reprojection')
    )
    parser.add_argument(
        '--no-data-value', default=None,
        help='Value to represent no data if not set in original geotiff'
    )

    args = parser.parse_args()

    spark_conf = SparkConf().setAppName('Azavea-Data-Hub-Reprojection')
    sc = SparkContext(conf=spark_conf)

    sampling_method = SAMPLING_METHODS.get(args.sampling_method, RESAMPLING.nearest)

    raw_tifs = sc.binaryFiles(args.src_tif_dir).coalesce(args.partitions)

    reprojected_tifs = raw_tifs.map(
        lambda (src_tif_path_remote, tif_bytes): reproject_tif(
            src_tif_path_remote, tif_bytes, args.dst_crs,
            sampling_method, args.no_data_value
        )
    )

    reprojected_tifs.saveAsSequenceFile(args.rdd_dst)

# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
    t0=tbegin=time.time()

    if gen_num_blocks>0 and gen_block_size>0:
        rdd=sc.parallelize(range(gen_num_blocks),args.nodes*12*args.nparts)
        gen_block_count=gen_block_size*1E6/24  # 24 bytes per vector
        print("generating %d blocks of %d vectors each..."%(gen_num_blocks,gen_block_count))
        outfile.write("generating data...\n")
        outfile.write("partition_multiplier: "+str(args.nparts)+"\n")
        outfile.write("gen_num_blocks: "+str(gen_num_blocks)+"\n")
        outfile.write("gen_block_size: "+str(gen_block_size)+"\n")
        outfile.write("total_data_size: "+str(gen_num_blocks*gen_block_size)+"\n")
        A=rdd.map(lambda x:generate(x,gen_block_count))
    elif args.src:
        outfile.write("reading data...\n")
        outfile.write(args.src+"\n")
        rdd = sc.binaryFiles(args.src)
        A = rdd.map(parseVectors)
    else:
        print("either --src or --generate must be specified")
        sc.stop();
        from sys import exit
        exit(-1)

    #rdd.foreach(noop)  #useful to force pipeline to execute for debugging
    tmark=time.time()
    outfile.write("read/parse or generate partitions: %0.6f\n"%(tmark-t0))
    outfile.write("numPartitions(%d,%s): %d\n"%(A.id(),A.name(),A.getNumPartitions()))
    t0=tmark

    # apply simple operation (V'=V+V0)
    shift=np.array([25.25,-12.125,6.333],dtype=np.float64)