Python MPIPool.map方法代码示例

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
        while jj< len(y):
            print 'icount, jj',icount,jj
            iipix_mask,iipix = WLanalysis.coords2grid(x[jj:jj+istep], y[jj:jj+istep], idata.flatten().reshape(1,-1)[:,jj:jj+istep], size=sizes[Wx-1])
            ipix_mask += iipix_mask
            ipix += iipix
            jj+=istep
    print icount,'W%i done coords2grid %s'%(Wx,icount)#,time.strftime("%Y-%m-%d %H:%M")    
    
    save(mask_dir+'smaller/weight0_W%i_%i_numpix'%(Wx,icount), ipix)
    save(mask_dir+'smaller/weight0_W%i_%i_nummask'%(Wx,icount), ipix_mask)
    #ipix is the num. of pixels fall in that big pix, ipix_mask is the mask
    return ipix, ipix_mask

p = MPIPool()    
if not p.is_master():
    p.wait()
    sys.exit(0)

#p.map(partialdata2grid, range(63))
ismall_map=p.map(partialdata2grid, range(63))
small_map = sum(array(ismall_map),axis=0)
save(mask_dir+'weight0_W%i_smaller_mask.npy'%(Wx),small_map)
weight=1-small_map[1]/small_map[0]
weight[isnan(weight)]=0
save(mask_dir+'ludoweight_weight0_W%i.npy'%Wx, weight)
mask=weight/weight
mask[isnan(mask)]=0
save(mask_dir+'ludomask_weight0_W%i.npy'%Wx, mask)

p.close()

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
	create 500 noise maps by randomly rotate galaxies, also include weights and (1+m) correction. all maps are smoothed over 1 arcmin.
	'''
	#Wx=int(sys.argv[1])
	p = MPIPool()
	
	bmap_fn = lambda Wx, iseed: kSZ_dir+'CFHT/Noise/W%i_Noise_sigmaG10_%04d.npy'%(Wx, iseed)
	Mexw = lambda Wx, txt: WLanalysis.readFits(kSZ_dir+'CFHT/Me_Mw_galn/W%i_M%s_1.3_lo.fit'%(Wx,txt))
	Me1, Me2, Mwm = Mexw(Wx, 'e1w'), Mexw(Wx, 'e2w'), Mexw(Wx, 'wm')
	def randmap (iseed, Wx=Wx):	
		Me1rnd, Me2rnd = WLanalysis.rndrot(Me1, Me2, iseed=iseed)
		Me1smooth = WLanalysis.weighted_smooth(Me1rnd, Mwm)
		Me2smooth = WLanalysis.weighted_smooth(Me2rnd, Mwm)
		kmap_rand = WLanalysis.KSvw(Me1smooth, Me2smooth)
		print Wx, iseed, kmap_rand.shape
		np.save(bmap_fn(Wx, iseed), kmap_rand)
	p.map(randmap, arange(500))
	print 'done creating 500 noise KS maps'

if cross_correlate_kSZ_noise:
	kSZmap_arr = map(kSZmapGen, range(1,5))
	mask_arr = map(maskGen, range(1,5))
	masked_kSZ_arr = [kSZmap_arr[i]*mask_arr[i] for i in range(4)]
	def kSZxNoise(iinput):
		'''iinput = (Wx, iseed)
		return the cross power between kSZ and convergence maps, both with smoothed mask.
		'''
		Wx, iseed = iinput
		print 'kSZxNoise', Wx, iseed
		bmap = bmapGen(Wx, iseed)*mask_arr[Wx-1]
		kSZmap = masked_kSZ_arr[Wx-1]
		edges = edgesGen(Wx)

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
import WLanalysis
import numpy as np
from scipy import *
from emcee.utils import MPIPool
import sys

CMBlensing_dir = '/work/02977/jialiu/CMBnonGaussian/'

PPA = 2048.0/(sqrt(12.25)*60.0)
sigmaG = PPA*1.0#9.7523809523809533
kmapGen = lambda r: WLanalysis.readFits('/scratch/02977/jialiu/CMB_hopper/CMB_batch_storage/Om0.296_Ol0.704_w-1.000_si0.786/1024b600/Maps/WLconv_z1100.00_%04dr.fits'%(r))

def k3Gen(r):
    print r
    kmap=kmapGen(r)
    kmap_smoothed=WLanalysis.smooth(kmap,sigmaG)
    k3=mean(kmap_smoothed**3)
    return k3

pool=MPIPool()
if not pool.is_master():
    pool.wait()
    sys.exit(0)

k3_arr = array(pool.map(k3Gen, range(1,10241)))
save(CMBlensing_dir+'k3_arr.npy',k3_arr)
pool.close()
print 'done-done-done'

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
import os
from emcee.utils import MPIPool


inDIR = '/scratch/02977/jialiu/CMB_hopper/CMB_batch_storage/'
outDIR = '/scratch/02977/jialiu/ranch_archive/CMB_batch_storage/'

os.system("lfs setstripe -c 4 %s"%(outDIR))

def create_tarball (FNs):
    inFN, outFN = FNs
    os.system("tar cfv %s.tar %s"%(outFN, inFN))

pool = MPIPool()
cosmo_arr = os.listdir(inDIR)
FNs_arr = [['%s%s'%(inDIR, cosmo_arr[j]),'%s%s'%(outDIR, cosmo_arr[j])] for j in range(len(cosmo_arr))]
pool.map(create_tarball, FNs_arr)
    
    

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
############ end: functions ##############

############ begein: calculate ############

#1. process maps
if processmaps:
## Make sure the thread we're running on is the master
#if not pool.is_master():
	#pool.wait()
	#sys.exit(0)
	## logger.debug("Running with MPI...")
	iRcosmo_pk = [[i, sigmaG, zg, bins, cosmo] for i in i_arr for sigmaG in sigmaG_arr for zg in zg_arr for bins in bins_arr for cosmo in cosmo_arr]
	iRcosmo_ps = [[i, sigmaG, zg, 0, cosmo] for i in i_arr for sigmaG in sigmaG_arr for zg in zg_arr for cosmo in cosmo_arr]
	pool = MPIPool()
	pool.map(Pmat, iRcosmo_ps+iRcosmo_pk)
	pool.close()

#2. cosmology model
#2.1 covariance matrix
# (1.8 arcmin, 25bins)[3:17] # 14 bins
# (3.5 arcmin, 25bins)[5:12] # 7 bins
## build array of sigmaG, bins, start, end, to prepare for cov, fisher mat
# config_2_21 = array([[1.8, 25, 3, 17],
		     #[3.5, 25, 5, 12]])

if fitcosmo:
	## no need to do fit with ASTRO, can do it on my laptop
	
	bintol = int(sum(config[:,-1]-config[:,-2])) # total bins used in cosmo model
	cosmo_mat = zeros((4, Rtol, bintol))

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
				xiplus_arr [ibin] += sum(w[i]*w[j] * real(ei * conj(ej)))
				ximinus_arr[ibin] += sum(w[i]*w[j] * real(ei * ej))
				norm_arr [ibin] += sum(w[i]*w[j]*(1+m[i])*(1+m[j]))
				ipair += step
			
		K = array([xiplus_arr, ximinus_arr, norm_arr])
		savetxt(fn,K)
		return K
	
	#xiplus_arr[1:] -= xiplus_arr[:-1]
	#ximinus_arr[1:] -= ximinus_arr[:-1]
	#norm_arr[1:] -= norm_arr[:-1]

		
pool = MPIPool()
pool.map(twoPCF, range(1,14))

print 'done-done-done: calculating 2pcf, now sum up all subfields'
# -------------------- grab everything ---------------------
bigmatrix = zeros(shape=(13, 3, len(bins)))

for isf in range(1,14):
	#fn = twoPCF_dir+'twoPCF_subfield%i'%(isf)
	#bigmatrix [isf-1] = genfromtxt(fn)
	bigmatrix [isf-1] = twoPCF(isf)
	
# cumulative sum to sum in each bin - no need anymore, since got rid of this step in query_pairs
#bigmatrix[:, :, 1:] -= bigmatrix[:, :, :-1]
sum13fields = sum(bigmatrix, axis=0)
xip = sum13fields[0]/sum13fields[-1]
xim = sum13fields[1]/sum13fields[-1]

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
hi_m='mQ3-512b240_Om0.290_Ol0.710_w-1.000_ns0.960_si0.800'
cosmo_arr=(fidu,hi_m,hi_w,hi_s)

def galn_gen(i):
	print i
	y, x, z = WLanalysis.readFits(emucat_dir+'emulator_subfield%i_zcut0213.fit'%(i)).T
	Mz, galn = WLanalysis.coords2grid(x, y, np.array([z,]))
	WLanalysis.writeFits(galn, galn_fn)
	
# map(galn_gen,range(1,14))

# power spectrum for all of the rz1, 4 cosmo
def ps (R):
	Mk = WLanalysis.readFits(Mk_fn(i, cosmo, R))
	galn = WLanalysis.readFits(galn_fn(i))
	Mk_smooth = WLanalysis.weighted_smooth(Mk, galn, sigmaG)
	ps = WLanalysis.PowerSpectrum(Mk_smooth)
	return ps[1]
	
for i in range(1, 14):
	for cosmo in cosmo_arr:
		print i, cosmo
		p = MPIPool()
		pmat = np.array(p.map(ps, arange(1,1001)))
		pmat_fn = KS_dir + 'powspec_Mk/SIM_powspec_sigma05_subfield%i_rz1_%s_1000R.fit'%(i,cosmo)
		p.close()
		try:
			WLanalysis.writeFits(pmat,pmat_fn)
		except Exception:
			pass
print 'done'

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
	Mexw = lambda Wx, txt: np.load(cmb_dir+'cfht/W%i_M%s_nocut.npy'%(Wx,txt))
	Me1_arr = [Mexw(Wx, 'e1w') for Wx in range(1,5)]
	Me2_arr = [Mexw(Wx, 'e2w') for Wx in range(1,5)]
	Mwm_arr = [Mexw(Wx, 'wm') for Wx in range(1,5)]
	def randmap (iseedWx):
		iseed, Wx = iseedWx
		Me1, Me2 = Me1_arr[Wx-1], Me2_arr[Wx-1]
		Mwm = Mwm_arr[Wx-1]
		Me1rnd, Me2rnd = WLanalysis.rndrot(Me1, Me2, iseed=iseed)
		Me1smooth = WLanalysis.weighted_smooth(Me1rnd, Mwm)
		Me2smooth = WLanalysis.weighted_smooth(Me2rnd, Mwm)
		kmap_rand = WLanalysis.KSvw(Me1smooth, Me2smooth)
		print Wx, iseed, kmap_rand.shape
		np.save(bmap_fn(Wx, iseed), kmap_rand)
	p.map(randmap, [[iseed, Wx] for iseed in arange(500) for Wx in range(1,5)])
	print 'done creating 500 noise KS maps'
	
if cross_correlate_cmbl_noise:
	cmblmap_arr = map(cmblGen, range(1,5))
	mask_arr = map(maskGen, range(1,5))
	masked_cmbl_arr = [cmblmap_arr[i]*mask_arr[i] for i in range(4)]
	def cmblxNoise(iinput):
		'''iinput = (Wx, iseed)
		return the cross power between cmbl and convergence maps, both with smoothed mask.
		'''
		Wx, iseed = iinput
		print 'cmblxNoise', Wx, iseed
		bmap = bmapGen(Wx, iseed)*mask_arr[Wx-1]
		cmblmap = masked_cmbl_arr[Wx-1]
		edges = edgesGen(Wx)

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
		KS_fn = KS_dir+'CFHT_KS_sigma%02d_subfield%02d.fits'%(sigmaG*10,i)
		mask_fn = '/scratch/02977/jialiu/KSsim/mask/CFHT_mask_ngal%i_sigma%02d_subfield%02d.fits'%(ngal_arcmin,sigmaG*10,i)
		
		if WLanalysis.TestComplete((KS_fn,mask_fn),rm=True):
			kmap = WLanalysis.readFits(KS_fn)
			Mmask = WLanalysis.readFits(mask_fn)
		else:
			Me1_smooth = WLanalysis.weighted_smooth(Me1, Mw, PPA=PPA512, sigmaG=sigmaG)
			Me2_smooth = WLanalysis.weighted_smooth(Me2, Mw, PPA=PPA512, sigmaG=sigmaG)
			galn_smooth = snd.filters.gaussian_filter(galn.astype(float),sigmaG*PPA512, mode='constant')
			## KS
			kmap = WLanalysis.KSvw(Me1_smooth, Me2_smooth)
			## mask
			maskidx = where(galn_smooth < ngal_cut) #cut at ngal=5
			Mmask = ones(shape=galn.shape)
			Mmask[maskidx]=0
			
			WLanalysis.writeFits(kmap, KS_fn)
			WLanalysis.writeFits(Mmask, mask_fn)

# Initialize the MPI pool
pool = MPIPool()

# Make sure the thread we're running on is the master
if not pool.is_master():
    pool.wait()
    sys.exit(0)
# logger.debug("Running with MPI...")

pool.map(KSmap, arange(1,14))
savetxt(KS_dir+'done.ls',zeros(5))

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
import os, sys
import time

# Third-party
import numpy as np
from emcee.utils import MPIPool

def f(x):
    time.sleep(0.1)
    return np.sqrt(x)

# Initialize the MPI pool
pool = MPIPool()

# Make sure the thread we're running on is the master
if not pool.is_master():
    pool.wait()
    sys.exit(0)

v = np.random.random(size=1000)

a = time.time()
results = pool.map(f, v)
pool.close()
print(time.time() - a, "MPI map")

# now try in serial
a = time.time()
map(f, v)
print(time.time() - a, "map")

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
#!~/anaconda/bin/python
# Jia Liu 2014/5/21
# What the code does: create mass maps for 100 cosmologies, for the CFHT emulator project
# Cluster: XSEDE Stampede

from emcee.utils import MPIPool
import numpy as np
from scipy import *
from multiprocessing import Pool
#from scoop import futures

print 'start'

i_arr=arange(1,14)

def testMPIPool(i):
	savetxt('/home1/02977/jialiu/test%i'%(i),zeros(5))

pool = MPIPool()
#pool = Pool(len(i_arr))
pool.map(testMPIPool, i_arr)
#pool.close()
#futures.map(testMPIPool, i_arr)

print 'DONE-DONE-DONE'

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
		
		###### create new gadget snapshot ############
		halo_snap = Gadget2Snapshot()
		hg = Gadget2Snapshot.open(snap_fn_arr[0]).header #header_gadget
		halo_snap.setPositions(array(halo_position)*halo_ID_position[0][1].unit)	
		halo_snap.setHeaderInfo(Om0=hg['Om0'], Ode0=hg['Ode0'], w0=hg['w0'], wa=hg['wa'], h=hg['h'], redshift=hg['redshift'], box_size=hg['box_size'])
		
		###### write the new snapshot to file ##########
		
		halo_snap.write(new_snap_fn, files = len(snap_fn_arr))
		print "WROTE", new_snap_fn
		### test on laptop
		#halo_snap.write('snapshots_amiga/snapshot_%03d'%(snap_id), files = len(snap_fn_arr))

print 'start job'
pool.map(halo_particles, [[ID, snap_id] for ID in ID_arr for snap_id in snap_id_arr])


################ there're problems if only 8 amiga files, so re-write into 16 files ############
def snapposition (fn):
	snap = Gadget2Snapshot.open(fn)
	return snap.getPositions()

def reorganize_snaps (IDsnap_id):
	ID, snap_id = IDsnap_id
	cosmo_id,geometry_id, ic_id = ID.split("|")
	new_snap_fn = os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga/snapshot_%03d'%(snap_id))
	os.system('mkdir -p %s'%(os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga')))
	
	amiga8_arr = glob.glob(os.path.join(storage, cosmo_id, geometry_id, ic_id, 'snapshots_amiga8/snapshot_%03d.*'%(snap_id)))
	halo_position = concatenate(array(map(snapposition, amiga8_arr)),axis=0)

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
# This is a short script to test the MPI implementation with the pattern used
# by Prospector.  However, try/except blocks are minimized to enable more
# useful error messages, and the code is simple enough to *only* test the MPI
# implementation.

# Invoke with:
# mpirun -np 4 python mpi_hello_world.py

import numpy as np
import sys
from mpi4py import MPI

def speak(i):
    print("I am core {} with task {}".format(pool.rank, i))
    return i, pool.rank

from emcee.utils import MPIPool
pool = MPIPool(debug=False, loadbalance=True)
if not pool.is_master():
    # Wait for instructions from the master process.
    pool.wait()
    sys.exit(0)


if __name__ == "__main__":
    
    result = pool.map(speak, np.arange(10))
    print(result)
    pool.close()

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
				theta = sqrt((x_fore-x_back)**2+(y_fore-y_back)**2)
				print '%i\t%s\t%.2f\t%.3f\t%.3f\t%.4f\t%.6f'%(i, jj,log10(jMvir), z_fore, z_back, rad2arcmin(theta), kappa_temp)	
		return ikappa

	#a=map(kappa_individual_gal, randint(0,len(idx_back)-1,5))
	step=2e3
	
	def temp (ix):
		print ix
		temp_fn = obsPK_dir+'temp/kappa_proj%i_%07d.npy'%(Wx, ix)
		if not os.path.isfile(temp_fn):
			kappa_all = map(kappa_individual_gal, arange(ix, amin([len(idx_back), ix+step])))
			np.save(temp_fn,kappa_all)
	pool = MPIPool()
	ix_arr = arange(0, len(idx_back), step)
	pool.map(temp, ix_arr)
	
	all_kappa_proj = concatenate([np.load(obsPK_dir+'temp/kappa_proj%i_%07d.npy'%(Wx, ix)) for ix in ix_arr])
	np.save(obsPK_dir+'kappa_predict_W%i.npy'%(Wx), all_kappa_proj)
	
#########################################################
####################### plotting correlation ############
#########################################################
make_predict_maps = 0
plot_predict_maps = 0
peak_proj_vs_lensing = 1
cross_correlate = 0

#kmap_predict_Gen = lambda Wx, sigmaG: np.load(obsPK_dir+'maps/r20arcmin_varyingcNFW_VO06/kmap_W%i_predict_sigmaG%02d.npy'%(Wx, sigmaG*10))
kmap_predict_Gen = lambda Wx, sigmaG: np.load(obsPK_dir+'maps/kmap_W%i_predict_sigmaG%02d.npy'%(Wx, sigmaG*10))

# 需要导入模块: from emcee.utils import MPIPool [as 别名]
# 或者: from emcee.utils.MPIPool import map [as 别名]
	k, s1, s2 = kappaGen(r)[:3]
	A, galn = WLanalysis.coords2grid(x, y, array([s1,s2 ]))
	Ms1,Ms2 = A
	s1_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG)
	s2_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG)
	kmap = WLanalysis.KSvw(s1_smooth, s2_smooth)
	pk = WLanalysis.peaks_mask_hist(kmap, mask, bins=25, kmin = -0.04, kmax = 0.12)
	return pk

def kmapPk_1sim (r):
	print r,'1sim'
	k, s1, s2 = kappaGen_1sim(r)[:3]
	A, galn = WLanalysis.coords2grid(x, y, array([s1,s2]))
	Ms1,Ms2 = A
	s1_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG)
	s2_smooth = WLanalysis.weighted_smooth(Ms1, galn, PPA=PPA512, sigmaG=sigmaG)
	kmap = WLanalysis.KSvw(s1_smooth, s2_smooth)
	pk = WLanalysis.peaks_mask_hist(kmap, mask, bins=25, kmin = -0.04, kmax = 0.12)
	return pk


pool = MPIPool()
#ps_mat = pool.map(kmapPs, range(1,1001))
#WLanalysis.writeFits(ps_mat,KS_dir+'ps_mat_sf1_fidu_shear_noiseless_mask.fit')

#pk_mat = pool.map(kmapPk, range(1,1001))
#np.save(KS_dir+'cfhtcov-512b240_Om0.260_Ol0.740_w-1.000_ns0.960_si0.800/pk_sigmaG10_sf1_cov.npy',pk_mat)

pk_mat1sim = pool.map(kmapPk_1sim, range(1,1001))
np.save(KS_dir+'cfhtcov-512b240_Om0.260_Ol0.740_w-1.000_ns0.960_si0.800/pk_sigmaG10_sf1_1sim.npy',pk_mat1sim)
print 'Done'