Python gearman.GearmanClient类代码示例

def pushSocket(queue):
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    serverAddr = ('0.0.0.0', 28282);
    sock.bind(serverAddr)

    send_thread = SendThread(sock, queue)
    send_thread.start()

    gm_client = GearmanClient(['localhost:4730'])

    while True:
        msg, addr = sock.recvfrom(2048)
        if not msg:
            continue
        #print "recevied:", msg, "from", addr

        try:
            data = json.loads(msg)
            topic = str(data['topic'])
            data['address'] = addr[0] + ':' + str(addr[1])
        except:
            continue

        gm_request = gm_client.submit_job(topic, json.dumps(data), background=False, wait_until_complete=False)

    sock.close()

def test_mailchimp():
    'test mailchimp integration'
    """
    For more info about setting config settings see demisauce/manage.py
    """
    from gearman import GearmanClient
    from gearman.task import Task
    gearman_client = GearmanClient(options.gearman_servers)
    #send emails
    list_id, mc_apikey = '0',''
    site = Site.GET(1)
    assert site.has_attribute('mailchimp_api_key')
    assert site.has_attribute('mailchimp_listid')
    list_id = site.get_attribute('mailchimp_listid').value
    mc_apikey = site.get_attribute('mailchimp_api_key').value
    jsondict = {
        'template_name':'thank_you_for_registering_with_demisauce',
        'user':{"email":"[email protected]"},
        'mailchimp_listid':list_id,
        'mailchimp_api_key':mc_apikey,
        'attributes':[{"name":"BetaUsers","category":"event"},{"name":"NewSegment3","category":"event"}]
    }
    #'BetaUsers',"NewSegment","NewSegment2"
    num_sent = gearman_client.do_task(Task("mailchimp_addtolist",json.dumps(jsondict), background=False))
    logging.debug("test emailsend num_sent = %s" % (num_sent))
    assert num_sent == '1'

def videoCreationBatch():
    from auth import get_user, is_superuser
    if not is_superuser():
        return action_401()
    if request.method=="GET":
        chdir(config.INGEST_DIRECTORY)
        files=[f for f in listdir(getcwd()) if f[0] != '.']
        return json.dumps(files)
    else:
        from PIL import Image
        from shutil import move
        from helpers import getVideoInfo
        packet=request.json
        for up in packet:
            filepath=unicode(config.INGEST_DIRECTORY + up['filepath'])
	    new_file=unicode(config.MEDIA_DIRECTORY + up['id'] + ".mp4")
	    if path.isfile(new_file):
		return bundle_400("That file already exists; try another unique ID.")
            if path.isfile(filepath.encode('utf-8')):
                md=getVideoInfo(filepath.encode('utf-8'))
                poster = config.POSTERS_DIRECTORY + "%s.jpg" % (up["id"])
                thumb = config.POSTERS_DIRECTORY + "%s_thumb.jpg" % (up["id"])
                move(filepath.encode('utf-8'), new_file.encode('utf-8'))
                assets.update({"_id":up["pid"]},{"$set":{
                    "@graph.ma:frameRate":float(md["framerate"]),
                    "@graph.ma:averageBitRate":int(float(md["bitrate"])),
                    "@graph.ma:frameWidth":int(md["width"]),
                    "@graph.ma:frameHeight":int(md["height"]),
                    "@graph.ma:duration":int( round(float(md["duration"])) )/60,
                    "@graph.ma:locator": [
                        {
                            "@id": up["id"],
                            "ma:hasFormat": "video/mp4",
                            "ma:hasCompression": {"@id":"http://www.freebase.com/view/en/h_264_mpeg_4_avc","name": "avc.42E01E"}
                        },
                        {
                            "@id": up["id"],
                            "ma:hasFormat": "video/webm",
                            "ma:hasCompression": {"@id":"http://www.freebase.com/m/0c02yk5","name":"vp8.0"}
                        }
                    ]
                }})
                imgcmd = "avconv -i '%s' -q:v 1 -r 1 -t 00:00:01 -ss 00:00:30 -f image2 '%s'" % (new_file,poster)
                system(imgcmd.encode('utf-8'))
                chmod(poster,0775)
                im=Image.open(poster)
                im.thumbnail((160,90))
                im.save(thumb)
                chmod(thumb,0775)
                
                if not app.config.get('TESTING'):
                    from gearman import GearmanClient
                    client = GearmanClient(config.GEARMAN_SERVERS)
                    client.submit_job("generate_webm", str(up["id"]))
                else:
                    from ingest import generate_webm
                    result = generate_webm(file_id=up['id'])
                    if result == "ERROR":
                      raise Exception("Could not convert media file.")
	return "Success"

    def obj_create(self, bundle, **kwargs):
        bundle.obj = Task(procedure_url=bundle.data["ordered_tasks"][0], 
                        input_dataset=bundle.data["input_dataset"], output_dataset=bundle.data["output_dataset"])
        
        bundle.obj.save()
        
        parent_task = bundle.obj
        for t_url in bundle.data["ordered_tasks"][1:]:
            if "aggregator" in t_url:
                continue
            temp_task = Task(procedure_url=t_url, parent=parent_task,
                                input_dataset=bundle.data["output_dataset"], output_dataset=bundle.data["output_dataset"])
            temp_task.save()
            parent_task = temp_task
            #statsd.gauge('outstanding_tasks', 1, delta=True)

        #statsd.gauge('outstanding_tasks', 1, delta=True)

        #now that we've created the dependency chain, we will schedule the first task
        dat = {}
        dat["task"] = bundle.data["ordered_tasks"][0]
        dat["task_id"] = bundle.obj.id
        dat["output_dataset"] = bundle.data["output_dataset"]
        dat["input_dataset"] = bundle.data["input_dataset"]
        dat["cassandra_nodes"] = CassandraNode.get_nodeip_list()
        client = GearmanClient(GearmanNode.get_nodeip_list())
        client.submit_job("pre_schedule", pickle.dumps(dat),background=True)
        
        return bundle.obj

 def deal(self):
     self.write('work received<br>')
     new_client = GearmanClient(['192.168.5.41:4730'])
     current_request = new_client.submit_job('task_kanjia','heal the world')
     new_result = current_request.result
     print new_result
     self.write('work finished')

 def send(self):
     log.error("in send of email api")
     emailjson = json.loads(self.request.body)
     if emailjson and 'template_name' in emailjson:
         log.error("weehah, body json = %s" % emailjson)
         #TODO:  revamp and use self.db.gearman_client
         gearman_client = GearmanClient(options.gearman_servers)
         gearman_client.do_task(Task("email_send",self.request.body, background=True))

def send_email(name,user,data):
    jsondict = {
        'template_name':name,
        'emails':[user.email],
        'template_data':data,
        'apikey':options.demisauce_api_key
    }
    data = json.dumps(jsondict)
    url = "%s/api/email/%s/send.json?apikey=%s" % (options.demisauce_url,name,options.demisauce_api_key)
    gearman_client = GearmanClient(options.gearman_servers)
    gearman_client.do_task(Task("email_send",data, background=True))

def run(simulation, run):
	rivers = simulation.rivers.all()
	start_point = simulation.start_point
	end_point = simulation.end_point
	start_elevation = usgs_elevation(start_point.x, start_point.y)
	end_elevation = usgs_elevation(end_point.x, end_point.y)
	river_length = 0 
	for river in rivers:
		river_length += river.geom.length

	# This is dumb, we need to do a proper calculation but for now this is good enough
	river_length_ft = river_length * 69.1 * 5280
	number_of_cross_sections = 1
	upstream_width = 30
	downstream_width = 30

	distance_ft = start_point.distance(end_point) * 69.1 * 5280

	channels = [
		{
			'length' : distance_ft, 
			'cross_sections' : number_of_cross_sections,
			'start_elevation' : start_elevation, 
			'end_elevation' : end_elevation,
			'upstream_width' : upstream_width, 
			'downstream_width': downstream_width
		},
	]

	model_parameters = {
		'maxtimesteps' : 80, 
		'time_step': 300, 
		'time_weight': 0.6, 
		'amplitude': 1999.5, 
		'period': 3.33, 
		'phase_angle': 1.67, 
		'start_time': 0.0, 
		'end_time': 1.667
	}

	run_parameters = {
		'channels': channels, 
		'model_parameters': model_parameters,
		'simulation_id': simulation.id, 
		'run_id': run.id
	}
	
	client = GearmanClient(settings.GEARMAN_SERVERS)
	jobdata = json.dumps(run_parameters)
	client.submit_job('fourpt', jobdata, background=True)

	return True

def doWork_fetchDependencyInfo(params):
    """
        params = {
            'projectId':projectId,
            'projectPath':projectInfo['projectPath'],
            'appName':data['appName'],
            'dependencyType':1,
        }
    """
    client = GearmanClient([GearmanConfig.gearmanConnection])
    data = json.dumps(params)
    request = client.submit_job(JobList.Job_fetchDependencyInfo, data, wait_until_complete=True)
    return request.result

def generate(image, options, force_creation = False):
  simulation = image.simulation

  points = options['points']
  for point in points:
    logging.debug("Point: %d,%d" % (point['x'], point['y']))
  natural_width = int(options['naturalWidth'])
  natural_height = int(options['naturalHeight'])
  image.channel_width_points = json.dumps(points)
  image.image_natural_width = natural_width
  image.image_natural_height = natural_height
  image.save()

  geotiff_image = simulation.aerialmap.filname
  channel_image = simulation.channelmap.filename
  width_image = simulation.channelwidthmap.filename

  if (not os.path.isfile(width_image)) or force_creation == True :
    logging.debug("Channel width map image %s doesn't exist, generating..." % (width_image))

    if image.job_handle and force_creation == False:
      logging.debug("Job handle: %s already exists, not re-queueing" % (image.job_handle))
      return None
    else:
        run_parameters = {
           'channel_image' : simulation.channel_image,
           'channel_width_image' : simulation.channel_width_image,
           'elevation_map_image' : elevation_image
        }

        client = GearmanClient(settings.GEARMAN_SERVERS)
        jobdata = json.dumps(run_parameters)
        jobrequest = client.submit_job('elevation_map', jobdata, background=True)
            
        simulation.elevation_map_job_handle = jobrequest.gearman_job.handle
        simulation.elevation_map_job_complete = False
        simulation.save()

        return None
  else:
    img = Image.open(width_image)
    return img

def send_request(req):
    new_client = GearmanClient([req.IP])

    s = time.time()
    request_dict={}
    request_dict["header"] = req.header
    request_dict["request"]= req.request
    if "pack_in" in req.params and req.params["pack_in"] == "0":
        current_request = new_client.submit_job(req.worker,request_dict)
    else:
        current_request = new_client.submit_job(req.worker,msgpack.packb(request_dict))

    if "pack_out" in req.params and req.params["pack_out"] == "0":
        current_result = current_request.result
    else:
        current_result = msgpack.unpackb(current_request.result)
    e = time.time()
    print "using time:%f" % (e-s)

    return current_result

def get_gearman_status(job_handle):
    try:
        # Query gearmand
        client = GearmanClient(settings.GEARMAN_SERVERS)
        client.establish_connection(client.connection_list[0])

        # configure the job to request status for - the last four is not needed for Status requests.
        j = gearman.job.GearmanJob(client.connection_list[0], str(job_handle), None, None, None)

        # create a job request 
        jr = gearman.job.GearmanJobRequest(j)
        jr.state = 'CREATED'

        # request the state from gearmand
        res = client.get_job_status(jr)

        # the res structure should now be filled with the status information about the task
        return res
    except: 
        print "Unexpected error:", sys.exc_info()[0]
        return -1

class TestGearman(unittest.TestCase):
    def setUp(self):
        self.last_exception = (None, None)
        self.worker = GearmanWorker(job_servers)
        self.worker.register_function("echo", echo)
        self.worker.register_function("fail", fail)
        self.worker.register_function("sleep", sleep, timeout=1)
        class Hooks(object):
            @staticmethod
            def start(job):
                pass
            @staticmethod
            def complete(job, res):
                pass
            @staticmethod
            def fail(job, exc):
                self.last_exception = (job.func, exc)
        import thread
        thread.start_new_thread(self.worker.work, tuple(), dict(hooks=Hooks)) # TODO: Shouldn't use threads.. but we do for now (also, the thread is never terminated)
        self.client = GearmanClient(job_servers)

    def tearDown(self):
        del self.worker
        del self.client

    def testComplete(self):
        self.failUnlessEqual(self.client.do_task(Task("echo", "bar")), 'bar')

    def testFail(self):
        self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("fail", "bar")))
        # self.failUnlessEqual(self.last_exception[0], "fail")

    def testTimeout(self):
        self.failUnlessEqual(self.client.do_task(Task("sleep", "0.1")), '0.1')
        self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("sleep", "1.5")))

    def testCall(self):
        self.failUnlessEqual(self.client("echo", "bar"), 'bar')

def generate(simulation_id, force_creation = False):
  simulation = Simulation.objects.get(pk = simulation_id)

  geotiff_image = simulation.aerial_geotiff
  channel_image = simulation.channel_image

      
  if (not os.path.isfile(channel_image)) or force_creation == True:

    if simulation.channel_tile_job_handle and force_creation == False:
        logging.debug("Job handle: %s already exists, not re-queueing" % (simulation.channel_tile_job_handle)) 
        return None
    else:
        logging.debug("Channel image %s doesn't exist, generating..." % (channel_image))

        run_parameters = {
          'tile_path': settings.RIVER_TILES_PATH,
          'geotiff_image': geotiff_image, 
          'channel_image': channel_image,
          'ortho_tiles': [tile.tile for tile in simulation.get_ortho_tiles()],
          'tile_width': 5000, 
          'tile_height': 5000 
        }

        client = GearmanClient(settings.GEARMAN_SERVERS)
        jobdata = json.dumps(run_parameters)
        jobrequest = client.submit_job('channel_image', jobdata, background=True)

        simulation.channel_tile_job_handle = jobrequest.gearman_job.handle
        simulation.channel_tile_job_complete = False
        simulation.save()

        return None

  else:  
    img = Image.open(channel_image)
    return img

def download(request):

    if "folder" not in request.GET or "title" not in request.GET or "url" not in request.GET:
        return HttpResponse("failed to enqueue, one or more params missing")
    
    folder = request.GET["folder"]
    name = request.GET["title"]
    video_url = request.GET["url"]

    if folder =='' or name =='' or video_url=='':
        return HttpResponse("failed to enqueue")
    
    info = Info()
    video_id = info.add_to_queue(folder,video_url,name)

    #outtmpl="%s/%s" % (folder,name)

    folder="%s/%s" %(settings.DOWNLOAD_PATH,folder)

    d = {'folder':folder,'name':name,'url':video_url,'id':video_id}

    client = GearmanClient(["127.0.0.1"])
    res = client.dispatch_background_task("download", d)
    return HttpResponse("enqueued video for successfully")