Python SDBConnection.get_domain方法代码示例

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
 def __init__(self, api=MongoStampedAPI(), logsQuery=logsQuery()):
     self.stamp_collection = api._stampDB._collection
     self.acct_collection = api._userDB._collection
     self.query = logsQuery
     self.writer = statWriter("dashboard")
     conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
     self.domain = conn.get_domain("dashboard")

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class SimpleDBBackend:
    
    def __init__(self):
        self.connection = SDBConnection(aws_credentials.accessKey,
                                        aws_credentials.secretKey)
        self.domain = self.connection.get_domain(TABLE_NAME)      


    def put(self, key, value):
        sampler.begin()
        try:
            self.domain.put_attributes(key, {VALUE:value})        
        finally:
            sampler.end()


    def get(self, key):
        sampler.begin()
        try:
            # First try an eventually consistent read.
            result = self.domain.get_attributes(key, consistent_read=False)
            return result[VALUE]
        except KeyError:
            # The eventually consistent read failed. Try a strongly consistent
            # read.
            result = self.domain.get_attributes(key, consistent_read=True)
            return result[VALUE]
        finally:
            sampler.end()
        

    def incRefCount(self, key):
        # Not implemented.
        pass
        
        
    def decRefCount(self, key):
        # Not implemented.
        pass
    
    
    def nuke(self):
        # Delete and re-create the table.
        self.connection.delete_domain(TABLE_NAME)
        self.domain = self.connection.create_domain(TABLE_NAME)
        
                    
    def flush(self):
        pass # No-op.

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def submit():
    if len(sys.argv) < 3:
        print "Usage:"
        print "  %s -seed <seed> [<spawn coords>]" % sys.argv[0]
        return

    # TODO use less crappy command line parsing
    seed = sys.argv[2]

    if len(sys.argv) == 4:
        spawn = [int(x) for x in sys.argv[3].split(",")]
        assert(len(spawn) == 3)
        print "Generate world with this seed (\"%s\") with spawn %r [y/N]?" % (seed, spawn)
    else:
        spawn = None
        print "Generate world with this seed (\"%s\") [y/N]?" % seed
    if raw_input().lower() == 'y':
        uid = uuid.uuid4()

        print "Submitting job %s to queue..." % uid
        sqs = SQSConnection()
        sdb = SDBConnection()
        queue = sqs.get_queue("overviewer-genfromseed")
        db = sdb.get_domain("overviewerdb")
        print queue
        print db

        data = dict()
        data['uuid'] = str(uid)
        data['seed'] = seed
        data['generated'] = False
        if spawn:
            data['target_spawn'] = spawn
        if not db.put_attributes(uid, data):
            print "***Error: Failed to update the db"
            return 1
        
        msg = Message()
        msg.set_body(str(uid))
        if not queue.write(msg):
            print "***Error: Failed to enqueue"
            return 1

        print "Ok, job enqueued"


    else:
        print "Ok, not submitting.  Bye"
        return

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def submit():
    if len(sys.argv) < 3:
        print "Usage:"
        print "  %s -submit <world uuid>" % sys.argv[0]
        return

    sdb = SDBConnection()
    db = sdb.get_domain("overviewerdb")

    # TODO use less crappy command line parsing
    world_uuid = uuid.UUID(sys.argv[2])

    world_item = db.get_item(world_uuid)
    if not world_item:
        print "Can't find that world!"
        return 1

    print "Submit this world for rendering? [y/N]"
    if raw_input().lower() != 'y':
        return "Ok, nevermind."
        return 0

    from boto.sqs.connection import SQSConnection 
    sqs = SQSConnection()
    
    queue = sqs.get_queue("overviewer-render")
    
    render_uuid = uuid.uuid4()
    print "Render UUID:", render_uuid
    data = dict()
    data['uuid'] = str(render_uuid)
    data['rendered'] = False
    data['world_uuid'] = str(world_uuid)

    if not db.put_attributes(str(render_uuid), data):
        print "***Error: Failed to update the db"
        return 1

    msg = Message()
    msg.set_body(str(render_uuid))

    if not queue.write(msg):
        print "***Error: Failed to enqueue"
        return 1
    print "Ok, job enqueued"
    return 0

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def add_from_url():
    if len(sys.argv) < 3:
        print "Usage:"
        print "  %s -url <url>" % sys.argv[0]
        return

    url = sys.argv[2]
    print "Generate world with this url (\"%s\") [y/N]?" % url
    if raw_input().lower() != 'y':
        print "Ok, nevermind"
        return
    
    uid = uuid.uuid4()
    data = dict()
    data['uuid'] = uid
    data['world_url'] = url

    sdb = SDBConnection()
    db = sdb.get_domain("overviewerdb")
    if not db.put_attributes(uid, data):
        print "***Error: Failed to update the db"
        return 1
    print "Ok. DB updated"
    print uid

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class statWriter(object):
    
    def __init__(self,domain=None):
        self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        self.domain_name = domain
        
    def write(self,stat,key=None,domain=None):
        if domain is None:
            if self.domain_name is None:
                return False
            else:
                domain = self.domain_name
        
        if key is None:
            key = uuid.uuid1()
            
        if len(stat) > 0:
            try:
                domain = self.conn.get_domain(domain)
            except SDBResponseError:
                domain = self.conn.create_domain(domain)
            domain.put_attributes(key, stat, replace=False)
        
        return True

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
#!/usr/bin/python2

import sys
import os
from pprint import pprint

try:
    import boto
except ImportError:
    sys.path.append(os.path.expanduser("~/devel/boto"))
    import boto


from boto.sdb.connection import SDBConnection 

uid = sys.argv[1]
print "Looking up", uid

sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
data = db.get_item(uid)
pprint(data)

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class SimpleDB(object):
    
    def __init__(self, domain=None):
        self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        self.domain_name = None
        self.domains = {}

        if domain is None and is_ec2():
            stack = get_stack()
            stack_name = str(stack['instance']['stack'])
            self.domain_name = stack_name
        elif domain is not None:
            self.domain_name = domain

    def addStat(self, stat):
        if self.domain_name is None:
            return

        try:
            # Only add specific parameters
            data = {}

            if 'user_id' in stat:
                data['uid'] = stat['user_id']

            if 'path' in stat:
                data['uri'] = stat['path']

            if 'method' in stat:
                data['mtd'] = stat['method']

            if 'form' in stat:
                try:
                    for k, v in stat['form'].items():
                        try:
                            if not isinstance(v, basestring):
                                v = str(v)
                            if len(v.encode('utf-8')) > 1024:
                                v = '<INPUT TOO LONG>'
                            data['frm_%s' % k] = v
                        except Exception as e:
                            print e
                except Exception as e:
                    print e

            if 'result' in stat:
                data['cde'] = str(stat['result'])

            if 'begin' in stat:
                data['bgn'] = stat['begin'].isoformat()

            if 'finish' in stat:
                data['end'] = stat['finish'].isoformat()

            if 'node' in stat:
                data['nde'] = stat['node']
                
            if 'client_id' in stat:
                data['cid'] = stat['client_id']

            if 'duration' in stat:
                data['dur'] = "{0:10d}".format(stat['duration'])

            if len(data) > 0:
                statId = str(ObjectId())
                if data['uri'] != '/v1/ping.json' and data['uri'] != '/v1/temp/ping.json':
                    suffix = '0%s' % (sha1(statId).hexdigest()[0])
                    if suffix in self.domains:
                        domain = self.domains[suffix]
                    else:
                        try:
                            domain = self.conn.get_domain('%s_%s' % (self.domain_name, suffix))
                        except SDBResponseError:
                            domain = self.conn.create_domain('%s_%s' % (self.domain_name, suffix))
                        self.domains[suffix] = domain
                    domain.put_attributes(statId, data, replace=False)


        except Exception as e:
            print e
            raise

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def add_from_path():
    if len(sys.argv) < 3:
        print "Usage:"
        print "  %s -path <path>" % sys.argv[0]
        return
    
    path = sys.argv[2]

    if os.path.isdir(path):
        print "You've specified a directory.  I'll tar it up before uploading"
        print "OK? [y/N] ", 
        if raw_input().lower() != 'y':
            print "Ok, nevermind."
            return

        tmpdir = tempfile.mkdtemp(prefix="mc_gen")
        print "tmpdir is", tmpdir
        print "Making tarball..."
        p = subprocess.Popen(["tar", "-cf", os.path.join(tmpdir, "world.tar"), "."],
                cwd=path)
        p.wait()
        if p.returncode != 0:
            print "***Error: tar failed"
            return

        print "OK."
        print "Compressing..."
        p = subprocess.Popen(["bzip2", "world.tar"],
                shell=False,
                cwd=tmpdir)
        p.wait()
        if p.returncode != 0:
            print "***Error: compress failed"
            return
        print "OK."

        print "Checking filesize..."
        s = os.stat(os.path.join(tmpdir, "world.tar.bz2"))
        if s.st_size > 10*1024*1024:
            print "***Error: Compressed world is too big"
            return 1
        print "OK."


        uid = uuid.uuid4()
        print uid
        s3 = S3Connection()
        bucket = s3.get_bucket("overviewer-worlds")
        k = Key(bucket)
        k.key = "%s.tar.bz2" % uid
        print "Uploading to S3..."
        k.set_contents_from_filename(os.path.join(tmpdir, "world.tar.bz2"), reduced_redundancy=True)
        print "OK."
        k.make_public()

        urlbase = "https://s3.amazonaws.com/overviewer-worlds/"
        url = urlbase + k.key
        print "World is now available at:", url
   
        data = dict()
        data['uuid'] = uid
        data['world_url'] = url
        sdb = SDBConnection()
        db = sdb.get_domain("overviewerdb")
        if not db.put_attributes(uid, data):
            print "***Error: Failed to update the db"
            return 1
        print "Ok. DB updated"
    elif os.path.isfile(path):
        print "You've specified a file. I'll upload it without modification"
        print "OK? [y/N] ", 
        if raw_input().lower() != 'y':
            print "Ok, nevermind."
            return
        
        uid = uuid.uuid4()
        print uid
        s3 = S3Connection()
        bucket = s3.get_bucket("overviewer-worlds")
        k = Key(bucket)
        k.key = "%s.tar.bz2" % uid
        print "Uploading to S3..."
        k.set_contents_from_filename(path, reduced_redundancy=True)
        print "OK."
        k.make_public()

        urlbase = "https://s3.amazonaws.com/overviewer-worlds/"
        url = urlbase + k.key
        print "World is now available at:", url
   
        data = dict()
        data['uuid'] = uid
        data['world_url'] = url
        sdb = SDBConnection()
        db = sdb.get_domain("overviewerdb")
        if not db.put_attributes(uid, data):
            print "***Error: Failed to update the db"
            return 1
        print "Ok. DB updated"
    else:
#.........这里部分代码省略.........

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def generate():
    pass
    from boto.sqs.connection import SQSConnection 
    sqs = SQSConnection()

    queue = sqs.get_queue("overviewer-genfromseed")
    sdb = SDBConnection()
    db = sdb.get_domain("overviewerdb")

    message = queue.read(visibility_timeout=15)
    if not message:
        print "Nothing in the queue.  Please try again later"
        return 0
    uid = message.get_body()
    print "Got a job for %r" % uid
    data = db.get_item(uid)

    if 'target_spawn' in data:
        data['target_spawn'] = map(int, data['target_spawn'])
    print data

    # this script generate maps from seeds
    # if this map is already generated, then upate the db
    if data['generated'] == 'True':
        print "---Warning: I was asked to generate this map, but it's already generated"
        print "            I'm going to update the db, but not re-generate"
        data['generated'] = True
        data.save()
        queue.delete_message(message)
        return 


    # check config options
    if not os.path.isfile(config.minecraft_server):
        raise Exception("minecraft_server isn't configured")

    # with a generous amount of rounding up, assume it'll take 5 minutes to generate the map
    message.change_visibility(5*60)

    tmpdir = tempfile.mkdtemp(prefix="mc_gen")

    # create a settings.properties file with our seed
    with open(os.path.join(tmpdir, "server.properties"), "w") as f:
        f.write("level-seed=%s" % data['seed'])

    p = subprocess.Popen(["java", "-jar",
        config.minecraft_server, "-noGUI"],
        shell=False,
        stdin=subprocess.PIPE,
        cwd=tmpdir)

    p.stdin.write("stop\n")
    p.stdin.close()
    p.wait()
    print ""
    print "Minecraft server exited with %r" % p.returncode
    print "World resided in %r" % tmpdir

    # if we want a specific spawn, we need to rewrite the level.dat file,
    # remove the old region files, and restart the server
    if 'target_spawn' in data:
        s = data['target_spawn']
        leveldat = redstone.NBT.parse_from_file(os.path.join(tmpdir, "world", "level.dat"))
        root = leveldat.root
        root['Data']['SpawnX'].set_integer(int(s[0]))
        root['Data']['SpawnY'].set_integer(int(s[1]))
        root['Data']['SpawnZ'].set_integer(int(s[2]))
        leveldat.write_to_file(os.path.join(tmpdir, "world", "level.dat"))

        shutil.rmtree(os.path.join(tmpdir,"world","region"))
        p = subprocess.Popen(["java", "-jar",
            config.minecraft_server, "-noGUI"],
            shell=False,
            stdin=subprocess.PIPE,
            cwd=tmpdir)

        p.stdin.write("stop\n")
        p.stdin.close()
        p.wait()
        print "Minecraft server exited with %r" % p.returncode

    message.change_visibility(5*60)
    print "Making tarball..."
    p = subprocess.Popen(["tar", "-cf", "world.tar", "world/"],
        cwd=tmpdir)
    p.wait()
    if p.returncode != 0:
        print "***Error: tar failed"
        return
    print "OK."
    print "Compressing..."
    p = subprocess.Popen(["bzip2", "world.tar"],
            shell=False,
            cwd=tmpdir)
    p.wait()
    if p.returncode != 0:
        print "***Error: compress failed"
        return
    print "OK."

#.........这里部分代码省略.........

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class weeklyScore(object):
    
    def __init__(self,api):
        self.api = api
        self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        self.domains = {}
        self.statDict = {}

        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            self.domains[suffix] = self.conn.get_domain('stats_dev_%s' % (suffix))
    
    
    
    def scoreQuery(self,domain,t0,t1,uri,points):
        
        query = 'select uri,uid from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (domain.name,uri,t0.isoformat(),t1.isoformat())
        stats = domain.select(query)
        
        for stat in stats:
            
            if 'uid' not in stat or 'uri' not in stat or 'cde' in stat:
                continue
            key = stat['uid']
            try:
                self.statDict[key] += points
            except KeyError:
                self.statDict[key] = points

    
    def segmentationReport(self,t0,t1,bMonth,percentage=False):
        self.statDict = {}
        
        scores = {
            #2 points for likes
            '/v1/stamps/likes/create.json': 2,
            '/v1/stamps/likes/remove.json': -2,
            
            #2 points for todos
            '/v1/todos/create.json': 2,
            '/v1/todos/remove.json': -2,
            
            #3 points for comments
            '/v1/comments/create.json': 3,
            '/v1/comments/remove.json': -3,
            
            #3 points for adding a friend
            '/v1/friendships/create.json': 3,
            '/v1/friendships/remove.json': -3,
            
            #3 points for taking an action
            '/v1/actions/complete.json': 2,
            
            #5 points for a stamp
            '/v1/stamps/create.json': 5,
            '/v1/stamps/remove.json': -5,
            
            #5 points for an invite
            '/v1/frienships/invite.json': 5,
            
            }
        
        pool = Pool(30)
        
        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            for uri in scores:
                pool.spawn(self.scoreQuery,self.domains[suffix],t0,t1,uri,scores[uri])
            
        pool.join()
        
        power = 0
        active = 0
        irregular = 0
        dormant = 0
        
        agg_score = 0
        
        if bMonth:
            POWER = 100
            AVERAGE = 10
        else: #Looking at a week
            POWER = 25
            AVERAGE = 5
            
        for uid in self.statDict:
            agg_score += self.statDict[uid]
            
            if self.statDict[uid] >= POWER:
                power += 1
            elif self.statDict[uid] >= AVERAGE:
                active += 1
            elif self.statDict[uid] > 0:
                irregular += 1
            else:
                dormant +=1

           
        users = len(self.statDict)
        
#.........这里部分代码省略.........

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class SimpleDBConnection(object):
    def __init__(self, stack):
        try:
            self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        except SDBResponseError:
            print "SimpleDB Connection Refused"
            raise

        self._stack = stack

    @lazyProperty
    def domains(self):
        result = []
        for i in range(16):
            suffix = "0" + hex(i)[2]
            name = "%s_%s" % (self._stack, suffix)
            result.append(self.conn.get_domain(name))
        return result

    def _queryParams(self, params, bgn, end):
        queryParams = []
        if params is not None:
            for k, v in params.items():
                queryParams.append("%s='%s'" % (k, v))

        if bgn is not None:
            queryParams.append("end >= '%s'" % bgn.isoformat())

        if end is not None:
            queryParams.append("end < '%s'" % end.isoformat())

        return queryParams

    def _deserialize(self, item):
        def isoparse(s):
            # Credit: http://blog.client9.com/2010/02/fast-iso-datetime-format-parsing-in.html
            try:
                return datetime(int(s[0:4]), int(s[5:7]), int(s[8:10]), int(s[11:13]), int(s[14:16]), int(s[17:19]))
            except:
                return None

        # Convert datetimes
        if "bgn" in item:
            item["bgn"] = isoparse(item["bgn"])

        if "end" in item:
            item["end"] = isoparse(item["end"])

        # Convert numbers
        if "dur" in item:
            item["dur"] = int(item["dur"])

        return item

    def query(self, params=None, fields=None, bgn=None, end=None):

        queryFields = "*"
        if fields is not None:
            queryFields = ",".join(fields)

        queryParams = self._queryParams(params, bgn, end)

        pool = Pool(len(self.domains))

        data = []

        def run(domain, query):
            data.extend(domain.select(query))

        for domain in self.domains:

            # Build query
            query = "select %s from `%s`" % (queryFields, domain.name)
            if len(queryParams) > 0:
                query += " where %s" % (" and ".join(queryParams))

            # Run it
            pool.spawn(run, domain, query)

        pool.join()

        results = []
        for item in data:
            results.append(self._deserialize(item))

        return results

    def count(self, params=None, bgn=None, end=None):

        queryParams = self._queryParams(params, bgn, end)

        pool = Pool(len(self.domains))

        data = []

        def run(domain, query):
            data.extend(domain.select(query))

        for domain in self.domains:

#.........这里部分代码省略.........

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
def render():
    
    # check config options
    if not os.path.isdir(config.overviewer_root):
        raise Exception("overviewer_root isn't configured")
    if not os.path.isfile(config.upload_ssh_key):
        raise Exception("upload_ssh_key isn't configured")

    sqs = SQSConnection()
    queue = sqs.get_queue("overviewer-render")
    sdb = SDBConnection()
    db = sdb.get_domain("overviewerdb")
    
    message = queue.read(visibility_timeout=15)
    if not message:
        print "Nothing in the queue.  Please try again later"
        return 0

    render_uuid = message.get_body()
    print "render uuid:", render_uuid
    
    render_item = db.get_item(str(render_uuid))
    if not render_item:
        print "***Error can't find a world with that UUID"
        return 1
    if render_item.get("rendered") != "False":
        print "***Error: this render has already been started"
        print "          state", render_item.get("rendered")
        return 1


    world_uuid = render_item.get("world_uuid")
    world_item = db.get_item(str(world_uuid))
    print "world uuid:", world_uuid


    url = world_item.get("world_url", None)
    if not url:
        print "***Error: can't find worldurl"
        return 1

    message.change_visibility(3*60)
    render_item['rendered'] = "inprogress"
    render_item.save()

    print "Getting map..." 
    map_url = urllib2.urlopen(url)
    print "OK."

    tmpdir = tempfile.mkdtemp(prefix="mc_gen")
    fobj = open(os.path.join(tmpdir, "world.tar.bz2"), "w")
    print "Downloading map to %s..." % tmpdir
    shutil.copyfileobj(map_url, fobj)
    fobj.close()
    print "OK."

    print "Uncompressing..."
    os.mkdir(os.path.join(tmpdir, "world"))
    p = subprocess.Popen(["tar", "-jxf", os.path.join(tmpdir,"world.tar.bz2")],
            cwd=os.path.join(tmpdir,"world"))
    p.wait()
    if p.returncode != 0:
        print "***Error: decompressing"
        return 1

    # find the exact directory containing level.dat
    def findLevel(start):
        for root, dirs, files in os.walk(start):
            if "level.dat" in files: return root
            for d in dirs:
                findLevel(os.path.join(root, d))
        raise Exception("Failed to find level.dat")

    real_world_dir = findLevel(os.path.join(tmpdir, "world"))

    # TODO message.change_visibility(10*60)
    p = subprocess.Popen(["python2", 
        os.path.join(config.overviewer_root, "overviewer.py"),
        real_world_dir,
        os.path.join(tmpdir, "output_dir"),
        "--rendermode=smooth-lighting"])
    p.wait()
    if p.returncode != 0:
        print "***Error: rendering"
        return 1

    print "Making tarball..."
    p = subprocess.Popen(["tar", "-cf", 
        os.path.join(tmpdir, "render.tar"),
        "."],
        cwd=os.path.join(tmpdir, "output_dir"))

    p.wait()
    if p.returncode != 0:
        print "***Error: tar failed"
        return
    print "OK."
    print "Compressing..."
    p = subprocess.Popen(["bzip2", "render.tar"],
            shell=False,
#.........这里部分代码省略.........

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class SimpleDBConnection(object):
    
    def __init__(self):
        try:
            self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        except SDBResponseError:
            print "SimpleDB Connection Refused"
        
        self.statList = []
        self.statCount = 0
        
        self.v1_prod_domains = {}
        self.v2_prod_domains = {}
        self.v2_dev_domains = {}
        self.v2_stress_domains = {}
        
        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            
            self.v1_prod_domains[suffix] = self.conn.get_domain('stats_prod_%s' % suffix)
            self.v2_prod_domains[suffix] = self.conn.get_domain('bowser_%s' % suffix)
            self.v2_dev_domains[suffix] = self.conn.get_domain('stats_dev_%s' % suffix)
            self.v2_stress_domains[suffix] = self.conn.get_domain('stress_%s' % suffix)
    
    # Perform a query and write to an optional list, set or dict (key is first field specified)
    def execute(self, domain, query_dict, fields, bgn=None, end=None, destination=None, limit=None, duration=None):

        query = 'select %s from `%s`' % (fields,domain.name)
        transition = 'where'

        for key,value in query_dict.items():
            if "!=" in value:
                query = '%s %s %s != "%s"' %  (query, transition, key, value[2:])
            else:
                query = '%s %s %s="%s"' % (query, transition, key, value)
            transition = 'and'
        
        if bgn is not None: 
            query = '%s %s bgn > "%s"' % (query, transition, bgn.isoformat())
            transition = 'and'
        
        if end is not None:
            query = '%s %s bgn > "%s"' % (query, transition, bgn.isoformat())
            transition = 'and'
            
        init_results = domain.select(query)
        
        results = reveal(init_results,limit=limit,duration=duration)
        
        if destination != None:
            if fields == 'count(*)':
                destination += results[0]
            else:
                destination.extend(results)
        
        return results
    
    def query (self, stack, query_dict, fields='*', bgn=None, end=None, limit=None, duration=None):
        
        if fields == 'count(*)':
            self.statCount = 0
            destination = self.statCount
        else:
            self.statList = []
            destination = self.statList
            
        if stack == 'bowser':
            domains = self.v2_prod_domains
        else:
            domains = self.v2_dev_domains
        
        pool = Pool(16)
        
        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            
            pool.spawn(self.execute, domains[suffix], query_dict, fields=fields, bgn=bgn,end=end, destination=destination, limit=limit/16 + 1, duration=duration)

        pool.join()
        
        if fields == 'count(*)':
            return self.statCount
        else:
            return self.statList[:limit]
        
        
            
            
            
            
            
            
            
            
            

# 需要导入模块: from boto.sdb.connection import SDBConnection [as 别名]
# 或者: from boto.sdb.connection.SDBConnection import get_domain [as 别名]
class logsQuery(object):
    
    def __init__(self,domain_name=None):
        self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
        self.domains = {}
        self.statSet  = set()
        self.statDict = {}
        self.errDict = {}
        self.statCount = 0
        self.statCountByNode = {}
        self.statTimeByNode = {}

        if domain_name is None:
            domain_name = 'stress'
            
        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            self.domains[suffix] = self.conn.get_domain('%s_%s' % (domain_name,suffix))
            
            

    def performQuery(self,domain,fields,uri,t0,t1,byNode=False):
        
        if uri is not None:
            query = 'select %s from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (fields, domain.name, uri, t0.isoformat(), t1.isoformat())
        else:
            query = 'select %s from `%s` where bgn >= "%s" and bgn <= "%s"' % (fields, domain.name, t0.isoformat(), t1.isoformat())

        stats = domain.select(query)
        
        for stat in stats:
            
            try:
                if fields == 'count(*)':
                    self.statCount += int(stat['Count'])
                elif byNode:
                    bgn = stat['bgn'].split('T')
                    end = stat['end'].split('T')
                    if end[0] == bgn[0]:
                        bgn = bgn[1].split(':')
                        end = end[1].split(':')
                        hours = float(end[0]) - float(bgn[0])
                        minutes = float(end[1]) - float(bgn[1])
                        seconds = float(end[2]) - float(bgn[2])
                        diff = seconds + 60*(minutes + 60*hours)
                    
                    try:
                        self.statCountByNode[stat['nde']] += 1
                        self.statTimeByNode[stat['nde']] += diff
                    except KeyError:
                        self.statCountByNode[stat['nde']] = 1
                        self.statTimeByNode[stat['nde']] = diff
                else:
                    self.statSet.add(stat[fields])
            except KeyError:
                pass
        
        
    def activeUsers(self,t0, t1):
        self.statSet = set()
        
        pool = Pool(16)
        
        for i in range (0,16):
            suffix = '0'+hex(i)[2]
            #Just use collections inbox for speed
            pool.spawn(self.performQuery,self.domains[suffix],'uid',"/v1/activity/unread.json",t0,t1)
        
        pool.join()
        
        return len(self.statSet)
    
    def latencyQuery(self,domain,t0,t1,uri,blacklist,whitelist):
        if uri is None:
            query = 'select uri,frm_scope,bgn,end,cde,uid from `%s` where uri like "/v1/%%" and bgn >= "%s" and bgn <= "%s"' % (domain.name,t0.isoformat(),t1.isoformat())
        else:
            query = 'select uri,frm_scope,bgn,end,cde,uid from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (domain.name,uri,t0.isoformat(),t1.isoformat())
        stats = domain.select(query)
        
        for stat in stats:
            if 'uid' in stat and stat['uid'] in blacklist:
                continue
            elif len(blacklist) == 0 and len(whitelist) > 0 and 'uid' in stat and stat['uid'] not in whitelist:
                continue
            bgn = stat['bgn'].split('T')
            end = stat['end'].split('T')
            if end[0] == bgn[0]:
                bgn = bgn[1].split(':')
                end = end[1].split(':')
                hours = float(end[0]) - float(bgn[0])
                minutes = float(end[1]) - float(bgn[1])
                seconds = float(end[2]) - float(bgn[2])
                diff = seconds + 60*(minutes + 60*hours)
                key = stat['uri']

                if 'frm_scope' in stat:
                    key = "%s?scope=%s" % (stat['uri'], stat['frm_scope'])
                
                if 'cde' in stat:
                    errType = stat['cde'][0]
#.........这里部分代码省略.........