Python Bucket.get_object_contents方法代码示例

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def initialize(*args):
    global trie, FREQ, total, min_freq, initialized
    if len(args)==0:
        dictionary = DICTIONARY
    else:
        dictionary = args[0]
    with DICT_LOCK:
        if initialized:
            return
        if trie:
            del trie
            trie = None

        t1 = time.time()

        cache_file = 'jieba.cache'
        default_dict = dictionary
        default_bucket = getattr(settings, 'STORAGE_BUCKET_NAME')
        bucket = Bucket(default_bucket)

        cache_file_content = bucket.get_object_contents(dictionary)
        dict_stamp = bucket.stat_object(default_dict)['timestamp']

        load_from_cache_fail = True
        try:
            cache_stamp = bucket.stat_object(cache_file)['timestamp']
        except:
            cache_exists = False
        else:
            if cache_stamp > dict_stamp:
                logger.debug("loading model from cache %s" % cache_file)
                try:
                    cache_content = bucket.get_object_contents(cache_file)
                    trie,FREQ,total,min_freq = marshal.loads(cache_content)
                    load_from_cache_fail = False
                except:
                    load_from_cache_fail = True

        if load_from_cache_fail:
            trie,FREQ,total = gen_trie(cache_file_content)
            FREQ = dict([(k,log(float(v)/total)) for k,v in FREQ.iteritems()]) #normalize
            min_freq = min(FREQ.itervalues())
            logger.debug("dumping model to file cache %s" % cache_file)
            try:
                tmp_suffix = "."+str(random.random())
                cache_file = 'dict' + tmp_suffix + '.cache'
                cache_file = os.path.join(tempfile.gettempdir(), cache_file)
                with open(cache_file,'wb') as temp_cache_file:
                    marshal.dump((trie,FREQ,total,min_freq),temp_cache_file)
                if cache_exists:
                    bucket.delete_object('jieba.cache')
                bucket.put_object('jieba.cache', open(cache_file, 'rb'))
            except:
                logger.error("dump cache file failed.")
                logger.exception("")

        initialized = True

        logger.debug("loading model cost %s seconds." % (time.time() - t1))
        logger.debug("Trie has been built succesfully.")

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def initialize(dictionary=None):
    global pfdict, FREQ, total, min_freq, initialized, DICTIONARY, DICT_LOCK
    if not dictionary:
        dictionary = DICTIONARY
    with DICT_LOCK:
        if initialized:
            return

        logger.debug("Building prefix dict from %s ..." % X_CACHE_FILE)
        t1 = time.time()
        cache_file = X_CACHE_FILE
        if settings.DEBUG:
            bucket = Bucket('xkong1946')
        else:
            bucket = Bucket()

        dict_stamp = bucket.stat_object(dictionary)['timestamp']
        cache_stamp = bucket.stat_object(cache_file)['timestamp']

        load_from_cache_fail = True
        if cache_stamp > dict_stamp:
            logger.debug("Loading model from cache %s" % cache_file)
            try:
                cf = bucket.get_object_contents(cache_file)
                pfdict, FREQ, total = marshal.loads(cf)
                # prevent conflict with old version
                load_from_cache_fail = not isinstance(pfdict, set)
            except Exception, e:
                print e
                load_from_cache_fail = True

        if load_from_cache_fail:
            dict_file_content = bucket.get_object_contents(dictionary)
            pfdict, FREQ, total = gen_pfdict(dict_file_content)
            logger.debug("Dumping model to file cache %s" % cache_file)
            try:
                import StringIO
                fd = StringIO.StringIO()
                fd.write(marshal.dumps((pfdict, FREQ, total)))
                if bucket.stat_object(X_CACHE_FILE):
                    bucket.delete_object(X_CACHE_FILE)
                bucket.put_object(X_CACHE_FILE, fd.getvalue())

            except Exception:
                logger.exception("Dump cache file failed.")

        initialized = True

        logger.debug("Loading model cost %s seconds." % (time.time() - t1))
        logger.debug("Prefix dict has been built succesfully.")

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
 def inPage(self, site, filename):
     try:
         conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
         bucket = Bucket(site, conn)
         page = bucket.get_object_contents(filename)
     except Exception, e:
         print e

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
 def inPageAmazon(self, params):
     try:
         conn = Connection(accesskey='ym51nzx10z', secretkey='h0kxmzj2ly13jjj1m0jjly41li1wimizzz2w2m32', retries=3)
         bucket = Bucket('amazon', conn)
         page = bucket.get_object_contents(params['category'] + '/' + params['in_page'])
     except Exception, e:
         print e

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
 def get(self, sub_path):
     from sae.storage import Bucket
     bucket = Bucket('oerp')                                  # 从云平台拿到一个Bucket容器
     #imagebinary = meta['body']
     response = bucket.get_object_contents(sub_path, chunk_size=10)    # 取文件    bucket.get_object_contents(u'oerp', r'/uploadimg/' + sub_path)
     self.set_header('Content-Type', 'text/xml; charset=utf-8')
     self.write(response.next())

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def exportImport(filename,tp):
	result={}
	# for client debug
	if settings.DEBUG:
		data = xlrd.open_workbook(settings.MEDIA_ROOT+filename)
	# for sae
	else:
		bucket = Bucket('resources')
		obj = bucket.get_object_contents(filename)
		data=xlrd.open_workbook(file_contents=obj)
	table = data.sheets()[0]
	# check the column
	ncols=table.ncols
	nrows=table.nrows
	# for student
	if (tp==0 and (not ncols==11)) or (tp==1 and (not ncols==9)):
		result['status']='failured'
		result['tip']='excel列数不对'
	elif nrows<2:
		result['status']='failured'
		result['tip']='至少需要一条记录'
	else:
		statistic=executeImport(table,tp)
		result['status']='success'
		result['tip']='导入成功，共 %d 人，成功导入 %d 人，跳过 %d 人' \
		% (statistic['sum'],statistic['count'],statistic['existed'])
		result['usernames']=statistic['usernames']
	# delete the uploaded temp file
	#  for client debug
	if settings.DEBUG:
		os.remove(settings.MEDIA_ROOT+filename)
	#  for sae
	else:
		bucket.delete_object(filename)
	return result

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def ReadZipFile(filename):

    """
    从storage中读取数据，还原到kvdb中
    参数 filename 要还原数据的文件名
    """
    bucket = Bucket('backup')
    # print(filename)
    CryptData = bucket.get_object_contents(filename)
    # print(CryptData)
    # -FileBuffer.close()
    key = config.keyDataBackUp
    cipher = AES.new(key, AES.MODE_ECB)
    iv = cipher.decrypt(CryptData[:16])
    # print(str(iv))
    cipher = AES.new(key, AES.MODE_CBC, iv)
    bytebuffer = cipher.decrypt(CryptData[16:])
    lendata = ord(bytebuffer[-1])
    FileBuffer = io.BytesIO(bytebuffer[:-lendata])
    zfile = zipfile.ZipFile(FileBuffer,mode='r')
    namelist = zfile.namelist()
    for name in namelist:
        bytedata = zfile.read(name)
        kv.set(name,bytedata.decode("utf-8"))
        
    return u"数据已还原"

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
 def get(self, request, *args, **kwargs):
     from utils.kvdb.bucket import Bucket as SBucket
     filekey = kwargs.get('filekey')
     bucket = SBucket()
     contents = bucket.get_object_contents(filekey)
     r = HttpResponse(contents)
     r['Content-Disposition'] = 'attachment; filename={}'.format(filekey)
     return r

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def edit0(request):
    pname = pcomment = pmood = newname = 0
    from sae.storage import Bucket
    bucket = Bucket('abc')
    if request.POST:
        if request.POST.has_key('correct'):
            if request.GET.has_key('atitle'):
                pname = request.GET['atitle']
                pn = t.objects.all()
                if (len(pn)!= 0):
                    pname = pn[0].title
                    for i in pn:
                        i.delete()
                we = imagess.objects.filter(title = pname)
                if (len(we)!= 0):
                    img = bucket.get_object_contents('stati/'+pname)
                    im = Image.open(StringIO.StringIO(img))
                    imgout = StringIO.StringIO()
                    im.save(imgout,"jpeg")
                    img_data = imgout.getvalue()
                    we[0].title = request.POST['cname']+'.jpg'
                    newname = we[0].title
                    if (newname != pname):
                        ne = t(title = newname)
                        ne.save()
                        bucket.put_object('stati/'+newname, img_data)
                        im = Image.open(StringIO.StringIO(img))
                        out = im.resize((128, 128))
                        imgout = StringIO.StringIO()
                        out.save(imgout,"jpeg")
                        img_data = imgout.getvalue()
                        bucket.put_object('manage/'+newname, img_data)
                        bucket.delete_object('manage/'+pname)
                        bucket.delete_object('stati/'+pname)
                    pname = newname
                    we[0].comment = request.POST['ccomment']
                    we[0].mood = request.POST['cmood']
                    we[0].save()
            pname = request.POST['cname']+'.jpg'
            pcomment = request.POST['ccomment']
            pmood = request.POST['cmood']
    elif request.GET.has_key('atitle'):
        if (pname == 0):
            pname = request.GET['atitle']
            p = t(title = pname)
            p.save()
            we = imagess.objects.filter(title = pname)
            if (len(we)!= 0):
                pcomment = we[0].comment
                pmood = we[0].mood
    if (pname!=0):
        pname = pname[:-4]
        
    return render_to_response('editt.html',{'pname':pname,'newname':newname, \
    'pmood': pmood, 'pcomment':pcomment},context_instance=RequestContext(request))   

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def thumbnail(request,filename):
    bucket = Bucket('upload')
    bucket.put()
    bucket.put_object("image/"+filename, request.FILES['file'])

    obj = bucket.get_object_contents("image/"+filename)
    image = Image.open(StringIO(obj))
    image.thumbnail((160,120),Image.ANTIALIAS)
    imgOut = StringIO()
    image.save(imgOut, 'jpeg')
    img_data = imgOut.getvalue()
    bucket.put_object('thumbnail/'+filename, img_data)
    imgOut.close()

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
    def get(self, request, *args, **kwargs):

        bucket_name = getattr(settings, "SAE_STORAGE_BUCKET_NAME",
                              'xkong1946')
        sae_bucket = SaeBucket(bucket_name)
        kv_bucket = Bucket()
        files = ['dict.txt', 'jieba.cache']
        ret = []
        for file_ in files:
            contents = sae_bucket.get_object_contents(file_)
            kv_bucket.save(file_, contents)
            ret.append(file_)
        # or return json
        return HttpResponse("done")

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def generate_city_list():
    values = []
    #read citylist
    BUCKET = 'citylist'
    # read citylist
    bucket = Bucket(BUCKET)
    citylist_content = bucket.get_object_contents('cities.json')
    citylist = json.loads(citylist_content)

    values.append('<ul>')
    for city in citylist:
        line = '''<li><a href="%s/%s">%s</li>''' % ('http://aqidatapuller.applinzi.com', city['city'].lower(), city['city'])
        values.append(line)
    values.append('</ul>')
    return ''.join(values)

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def cron_task():
    BUCKET = 'citylist'
    # read citylist
    bucket = Bucket(BUCKET)
    citylist_content = bucket.get_object_contents('cities.json')
    cities = json.loads(citylist_content)
    # datepuller
    data_puller = DataPuller()
    # mysql
    mysql = Mysql()

    for city in cities:
        value = data_puller.pull_data(int(city['cid']))
        if value:
            mysql.insert_data(city['city'], int(value))

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
class SaeStorageSaver:
    def __init__(self, key):
        self.bucket = Bucket(key)

    def StoreTxtFile(self, path, content):
        self.bucket.put_object(path, content)

    def StoreBinFile(self, path, content):
        self.bucket.put_object(path, content)

    def GetObjectByPath(self, path):
        return self.bucket.get_object_contents(path)

    def GetItemUnder(self, path):
        return [x for x in self.bucket.list(path)]

    def GetBackupList(self):
        return self.GetItemUnder(g_backup_path)

    def DeleteObject(self, obj):
        self.bucket.delete_object(obj)

# 需要导入模块: from sae.storage import Bucket [as 别名]
# 或者: from sae.storage.Bucket import get_object_contents [as 别名]
def start(request):
    url = 0
    from sae.storage import Bucket
    bucket = Bucket('abc')
    if request.POST:
        if request.POST.has_key('save'):
            post = request.POST
            if request.FILES:
                f = request.FILES['file']
                new_img = imagess(picture = f, comment = post['writecomment'], mood = post['writemood'],\
                    title = str(f),lat = 0,lon = 0)
                new_img.save()
                img = bucket.get_object_contents('stati/'+f.name)
                im = Image.open(StringIO.StringIO(img))
                out = im.resize((128, 128))
                imgout = StringIO.StringIO()
                out.save(imgout,"jpeg")
                img_data = imgout.getvalue()
                bucket.put_object('manage/'+f.name, img_data)

                exif = get_exif_data(img)
                if exif.has_key('GPSInfo'):
                    w1 = exif['GPSInfo'][2][0][0]
                    w2 = exif['GPSInfo'][2][1][0]
                    w3 = exif['GPSInfo'][2][2][0]*1.0/100
                    lat = w1+w2*1.0/60 + w3*1.0/60*1.0/60
                    j1 = exif['GPSInfo'][4][0][0]
                    j2 = exif['GPSInfo'][4][1][0]
                    j3 = exif['GPSInfo'][4][2][0]*1.0/100
                    lon = j1+j2*1.0/60 + j3*1.0/60*1.0/60
                    new = imagess.objects.filter(title = f.name)
                    if (len(new) != 0):
                        new[0].lat = lat
                        new[0].lon = lon
                        new[0].save()


                
    return render_to_response('start.html',\
            context_instance=RequestContext(request))