本文整理汇总了Python中pyspark.SparkContext.binaryFiles方法的典型用法代码示例。如果您正苦于以下问题:Python SparkContext.binaryFiles方法的具体用法?Python SparkContext.binaryFiles怎么用?Python SparkContext.binaryFiles使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspark.SparkContext
的用法示例。
在下文中一共展示了SparkContext.binaryFiles方法的7个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: main
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
def main(argv):
logging.config.fileConfig(os.path.join(os.path.dirname(os.path.realpath(__file__)), "logging.ini"))
parsed_args = parse_args(argv)
spark_conf = SparkConf()
sc = SparkContext(conf=spark_conf)
with open(parsed_args.config) as in_config:
preprocess_conf = json.load(in_config)
if preprocess_conf.get("binary_input", True):
files = sc.binaryFiles(preprocess_conf["input"], preprocess_conf.get('partitions', 4000))
else:
files = sc.wholeTextFiles(preprocess_conf["input"], preprocess_conf.get('partitions', 4000))
files = files.repartition(preprocess_conf.get('partitions', 4000))
metadata = parse_metadata(preprocess_conf["labeled"]["metadata"])
labeled = sc.textFile(preprocess_conf["labeled"]["file"], preprocess_conf.get('partitions', 4000)).\
map(lambda x: parse_labeled_line(x, metadata, True)).filter(lambda x: x.iloc[0]["label"] != 4).map(transform_labels)
header, resampled = prep.preprocess(sc, files, labeled, label=preprocess_conf.get('label', True),
cut=preprocess_conf.get("cut", {"low": 6300, "high": 6700}),
pca=preprocess_conf.get("pca", None), partitions=preprocess_conf.get('partitions', 100))
resampled.map(lambda x: x.to_csv(None, header=None).rstrip("\n")).saveAsTextFile(preprocess_conf["output"])
示例2: SparkConf
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
if __name__ == '__main__':
from pyspark import SparkContext, SparkConf
parser = argparse.ArgumentParser()
parser.add_argument('src_tif_dir', help='Directory with files to reproject')
parser.add_argument('dst_dir', help='Directory to write reproject files')
parser.add_argument('--data-name', help='Optional identifer to prefix files with', default='')
parser.add_argument('--dst-crs', help='CRS to reproject files to', default='EPSG:3857')
parser.add_argument('--extension', help='Only consider files ending in this extension', default='')
parser.add_argument('--region', help='Region for the S3 client to use', default='')
args = parser.parse_args()
spark_conf = SparkConf().setAppName('Rainfall-Reprojection')
sc = SparkContext(conf=spark_conf)
raw_tifs = sc.binaryFiles(args.src_tif_dir)
if args.extension:
raw_tifs = raw_tifs.filter(lambda (path, _): path.endswith(args.extension))
reprojected_tifs = raw_tifs.map(
lambda (src_tif_path_remote, tif_bytes): process_tif(
src_tif_path_remote, tif_bytes, args.data_name, args.dst_crs, args.dst_dir, args.region
)
)
num_reprojected = reprojected_tifs.count()
示例3: eval_flow_cde
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
# (field0, series.y)
def eval_flow_cde(x):
return eval_flow_spark(x, bc_output_dir.value)
def plotImage(x):
t0 = time.time()
distribution_plot_subsets_spark(x[1], bc_output_dir.value)
print "plotImage used: ", t0 - time.time()
# print("-----"+x[0],x[1])
# hdfs
startTime = time.time()
t0 = time.time()
hdfsFile = sc.binaryFiles(input_dir).persist(StorageLevel.MEMORY_AND_DISK)
#########################################################Total Time Used: 8.73000001907
adaf_objs = hdfsFile.map(read_dat_hdfs)\
.map(ExtractVIN) \
.map(process_dat_adaf).map(sort_adaf) \
.map(vehical_config) \
.filter(do_filter)\
.map(subsetMetaData)
#########################################################Total Time Used: 25.1680002213
# rdd_vehical_config = hdfsFile.map(read_dat_hdfs) \
# .map(ExtractVIN) \
# .map(process_dat_adaf).map(sort_adaf) \
# .map(vehical_config)
# rdd_vehical_config.count()
# print("rdd_vehical_config Used Time: {}".format(time.time() - t0))
示例4: SparkContext
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
import re
import os
import sys
import numpy as np
srtm_dtype = np.dtype('>i2')
filename_regex = re.compile('([NSEW]\d+[NSEW]\d+).*')
# The data directory, needs to be available to all node in the cluster
data_files = '/media/bitbucket/srtm/version2_1/SRTM3/North_America'
# Build up the context, using the master URL
sc = SparkContext('spark://ulex:7077', 'srtm')
# Now load all the zip files into a RDD
data = sc.binaryFiles(data_files)
# The two accumulators are used to collect values across the cluster
num_samples_acc = sc.accumulator(0)
sum_acc = sc.accumulator(0)
# Function to array
def read_array(data):
hgt_2darray = np.flipud(np.fromstring(data, dtype=srtm_dtype).reshape(1201, 1201))
return hgt_2darray
# Function to process a HGT file
def process_file(file):
(name, content) = file
示例5: SparkContext
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
import boto
import datetime
sc = SparkContext()
# AWS S3 credentials:
AWS_KEY = ""
AWS_SECRET = ""
sc._jsc.hadoopConfiguration().set("fs.s3n.awsAccessKeyId", AWS_KEY)
sc._jsc.hadoopConfiguration().set("fs.s3n.awsSecretAccessKey", AWS_SECRET)
directory = 's3n://amlyelp/subset/trainnew/'
images = sc.binaryFiles(directory)
image_to_array = lambda rawdata: np.asarray(Image.open(StringIO(rawdata)))
image_array = images.map(lambda x: (x[0],image_to_array(x[1])))
image_array_flatten = image_array.map(lambda x: (x[0],x[1].flatten())).cache()
del image_array
del images
train = image_array_flatten.values().repartition(200).cache()
clusters = KMeans.train(train, 50, maxIterations=50)
clusters.save(sc, 's3n://amlyelp/subset/model/kmeans/50_iters_'+\
str(datetime.datetime.now()).replace(' ', '_')+'/')
示例6: SparkConf
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
'--partitions', default=250, type=int,
help=('Number of partitions to coalesce geotiffs to else '
'each geotiff will end up in its own partition'))
parser.add_argument(
'--sampling-method', default="nearest",
choices=SAMPLING_METHODS.keys(),
help=('Sampling method to use during reprojection')
)
parser.add_argument(
'--no-data-value', default=None,
help='Value to represent no data if not set in original geotiff'
)
args = parser.parse_args()
spark_conf = SparkConf().setAppName('Azavea-Data-Hub-Reprojection')
sc = SparkContext(conf=spark_conf)
sampling_method = SAMPLING_METHODS.get(args.sampling_method, RESAMPLING.nearest)
raw_tifs = sc.binaryFiles(args.src_tif_dir).coalesce(args.partitions)
reprojected_tifs = raw_tifs.map(
lambda (src_tif_path_remote, tif_bytes): reproject_tif(
src_tif_path_remote, tif_bytes, args.dst_crs,
sampling_method, args.no_data_value
)
)
reprojected_tifs.saveAsSequenceFile(args.rdd_dst)
示例7: print
# 需要导入模块: from pyspark import SparkContext [as 别名]
# 或者: from pyspark.SparkContext import binaryFiles [as 别名]
t0=tbegin=time.time()
if gen_num_blocks>0 and gen_block_size>0:
rdd=sc.parallelize(range(gen_num_blocks),args.nodes*12*args.nparts)
gen_block_count=gen_block_size*1E6/24 # 24 bytes per vector
print("generating %d blocks of %d vectors each..."%(gen_num_blocks,gen_block_count))
outfile.write("generating data...\n")
outfile.write("partition_multiplier: "+str(args.nparts)+"\n")
outfile.write("gen_num_blocks: "+str(gen_num_blocks)+"\n")
outfile.write("gen_block_size: "+str(gen_block_size)+"\n")
outfile.write("total_data_size: "+str(gen_num_blocks*gen_block_size)+"\n")
A=rdd.map(lambda x:generate(x,gen_block_count))
elif args.src:
outfile.write("reading data...\n")
outfile.write(args.src+"\n")
rdd = sc.binaryFiles(args.src)
A = rdd.map(parseVectors)
else:
print("either --src or --generate must be specified")
sc.stop();
from sys import exit
exit(-1)
#rdd.foreach(noop) #useful to force pipeline to execute for debugging
tmark=time.time()
outfile.write("read/parse or generate partitions: %0.6f\n"%(tmark-t0))
outfile.write("numPartitions(%d,%s): %d\n"%(A.id(),A.name(),A.getNumPartitions()))
t0=tmark
# apply simple operation (V'=V+V0)
shift=np.array([25.25,-12.125,6.333],dtype=np.float64)