Python mypass.getConn函数代码示例

    def get_status(self, id, getUser=False, toDB=True):
	time_db = 0
	time_db_u = 0
	start_time_api = time.time()
	api_misses = 0
	while api_misses < self.max_api_misses:
	    try:
		status = self.api.get_status(id=id)
		break
	    except weibopy.error.WeibopError as e:
		print e.reason
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": e.reason }
		if e.reason.find("Error: target weibo does not exist!") >= 0:
		    try:
			rps = self.getRangePartitionByIds([id])
		    	for x in rps:
			    yw = x.split(",")
			    year = int(yw[0])
			    week = int(yw[1])
			    sql_deleted = "UPDATE rp_sinaweibo_y%(year)dw%(week)d SET deleted = NOW() WHERE id = %(id)d AND deleted IS NULL " % { "year": year, "week": week, "id": id }
			    if self.pgconn is None:
				self.pgconn = mypass.getConn()
			    res = self.pgconn.query(sql_deleted)
		    except pg.ProgrammingError, pg.InternalError:
			print self.pgconn.error
		    return { "msg": e.reason }
		time.sleep(self.api_wait_secs)

    def comments(self, status_id, count=200, page=1, toDB=True, toBeginning=True):
	already_exists_count = 0
	start_time_api = time.time()
	api_misses = 0
	while api_misses < self.max_api_misses:
	    try:
		comments = self.api.comments(id=status_id, count=count, page=page)
		break
	    except weibopy.error.WeibopError as e:
		print e.reason
		api_misses += 1
		if api_misses == self.max_api_misses:
		    return { "msg": e.reason }
		if e.reason.find("Error: target weibo does not exist!") > 0:
		    try:
			rps = self.getRangePartitionByIds([status_id])
		    	for x in rps:
			    yw = x.split(",")
			    year = int(yw[0])
			    week = int(yw[1])
			    sql_deleted = "UPDATE rp_sinaweibo_y%(year)dw%(week)d SET deleted = NOW() WHERE id = %(id)d AND deleted IS NULL " % { "year": year, "week": week, "id": status_id }
			    if self.pgconn is None:
				self.pgconn = mypass.getConn()
			    res = self.pgconn.query(sql_deleted)
		    except pg.ProgrammingError, pg.InternalError:
			print self.pgconn.error
		    return { "msg": e.reason }
		time.sleep(self.api_wait_secs)
		if e.reason.find("requests out of rate limit") >= 0:
		    if e.reason.find("IP") >= 0 and api_misses <= self.max_api_misses_half:
			time.sleep(60) # to consider rolling IPs
		    else:
			self.waitRateLimit()

    def __init__(self, geo=None):
	self.pgconn = mypass.getConn()
	if geo is None:
	    self.key = None
	    self.geotable = None
	else:
	    self.geotable = geo
    	    if geo in self.GEO_KEYS:
    		self.key = self.GEO_KEYS[geo]
		self.geokey = self.GEO_KEYS[geo]
    	    else:
    		self.key = geo
		self.geokey = geo
	self.datatable = None
	self.geokml = dict()
	self.data = dict()
	self.factory = kmldom.KmlFactory_GetFactory()
	self.kml = self.factory.CreateElementById(kmldom.Type_kml)
	self.geo = None
	self.geokey = None
	self.geo_tolerance = None
	self.geo_mapping = dict() # for TPU to TPU_(LARGE|SMALL)
	self.output = None
	self.verbose = False
	self.outputformat = "csv"
	self.year = None

    def __init__(self):
	smartcn = lucene.SmartChineseAnalyzer(lucene.Version.LUCENE_33)
	self.analyzers = { "smartcn": smartcn }
	directory = lucene.SimpleFSDirectory(lucene.File(self.STORE_DIR))
	self.searcher = lucene.IndexSearcher(directory, True)
	self.pgconn = mypass.getConn()
	self.sw = sinaweibooauth.SinaWeiboOauth()

    def __init__(self, forumname):
	if not forumname in self.supported_forums:
	    sys.exit()
	else:
	    self.forum = forumname
	self.STORE_DIR = self.STORE_BASE_DIR + forumname
	smartcn = lucene.SmartChineseAnalyzer(lucene.Version.LUCENE_33)
	self.analyzers = { "smartcn": smartcn }
	directory = lucene.SimpleFSDirectory(lucene.File(self.STORE_DIR))
	self.searcher = lucene.IndexSearcher(directory, True)
	self.pgconn = mypass.getConn()

 def __init__(self):
     twitterOAuth = mypass.getTwitterOauth()
     api = twitter.Api(
         consumer_key=twitterOAuth["consumer_key"],
         consumer_secret=twitterOAuth["consumer_secret"],
         access_token_key=twitterOAuth["oauth_token"],
         access_token_secret=twitterOAuth["oauth_token_secret"],
         cache=None,
     )
     self._api = api
     self._pgconn = mypass.getConn()
     socket.setdefaulttimeout(150)

    def __init__(self, network):
	self.network = network	
	smartcn = lucene.SmartChineseAnalyzer(lucene.Version.LUCENE_33)
	#analyzer = lucene.StandardAnalyzer(lucene.Version.LUCENE_33)
	analyzers = { "smartcn": smartcn }
	self.pgconn = mypass.getConn()
	writerconfig = lucene.IndexWriterConfig(lucene.Version.LUCENE_33, analyzers["smartcn"])
	writerconfig.setWriteLockTimeout(600000L)
	writerconfig.setMaxThreadStates(50)
	writerconfig.setRAMBufferSizeMB(128.0)
	self.storeDir = self.storeDirBase + self.network
	store = lucene.SimpleFSDirectory(lucene.File(self.storeDir))
	self.writer = lucene.IndexWriter(store, writerconfig)

def getCensusData(req, geotype, geoid):
    pgconn = mypass.getConn()
    if geotype is None:
	return json.dumps(None);
    if geoid is None:
	return json.dumps(None);
    if geotype in GEO_TABLES:
	geotable = GEO_TABLES[geotype]
    else:
	geotable = geotype
    if geotype == "dcca":
	parentgeotype = "dc"
	parentgeoid = geoid[0:2]
	'''

    def __init__(self):
	smartcn = lucene.SmartChineseAnalyzer(lucene.Version.LUCENE_33)
	#analyzer = lucene.StandardAnalyzer(lucene.Version.LUCENE_33)
	analyzers = { "smartcn": smartcn }
	self.pgconn = mypass.getConn()
	self.sw = sinaweibooauth.SinaWeiboOauth()
	if not os.path.exists(self.storeDir):
	    os.mkdir(self.storeDir)
	store = lucene.SimpleFSDirectory(lucene.File(self.storeDir))
	writerconfig = lucene.IndexWriterConfig(lucene.Version.LUCENE_33, analyzers["smartcn"])
	writerconfig.setWriteLockTimeout(600000L)
	writerconfig.setMaxThreadStates(50)
	writerconfig.setRAMBufferSizeMB(128.0)
	self.writer = lucene.IndexWriter(store, writerconfig)

    def toDB(self, tablename, data, doupdate=False, updatefirst=False):
	if self.pgconn is None:
	    self.pgconn = mypass.getConn()
	resp = { "success": False, "already_exists": False }
	#r = self.pgconn.insert(tablename, data)
	#r = self.pgconn.update(tablename, data)
	if updatefirst:
	    if doupdate:
		try:
		    r = self.pgconn.update(tablename, data)
		    resp["success"] = True
		except pg.DatabaseError:#, pg.ProgrammingError, pg.InternalError:
		    if self.pgconn.error.find('No such record in') > 0:
			try:
			    r = self.pgconn.insert(tablename, data)
			    resp["success"] = True
			except:
			    if self.pgconn.error.find('duplicate key value violates unique constraint') > 0:
				resp["already_exists"] = True
	    else:
		try:
		    #print data
		    r = self.pgconn.insert(tablename, data)
		    resp["success"] = True
		except:
		    if self.pgconn.error.find('duplicate key value violates unique constraint') > 0:
			resp["already_exists"] = True
	else:
	    try:
		#print data
		r = self.pgconn.insert(tablename, data)
    		resp["success"] = True
		if r is not None and "id" in r:
		    resp["id"] = r["id"]
	    except pg.ProgrammingError, pg.InternalError:
		resp["reason"] = self.pgconn.error
		if self.pgconn.error.find('duplicate key value violates unique constraint') > 0:
		    resp["already_exists"] = True
		    try:
			if doupdate:
			    r = self.pgconn.update(tablename, data)
			    resp["success"] = True
			    if r is not None and "id" in r:
				resp["id"] = r["id"]
		    except:
			resp["reason"] = self.pgconn.error
			pass

def getBounds(req, geotype, geoid, year=2001):
    pgconn = mypass.getConn()
    if geotype is None:
	return json.dumps(None);
    if geoid is None:
	return json.dumps(None);
    #geoid = geoid.strip()
    if geotype in GEO_TABLES:
	geotable = GEO_TABLES[geotype]
    else:
	geotable = geotype
    if geotype in GEO_HASYEARS:
	geotable += str(year)
    if geotype in GEO_KEYS:
	geokey = GEO_KEYS[geotype]
    else:
	geokey = geotype
    if geotype in GEO_KEYS:
	geogeom = GEO_GEOM[geotype]
    else:
	geogeom = "the_geom"
    if geotype in GEO_TOLERANCE:
	tolerance = GEO_TOLERANCE[geotype]
	if geotype == "dc":
	    try:
		geoid_int = int(geoid)
		if geoid_int >= 32:
		    tolerance = tolerance * 5
	    except:
		pass
    else:
	tolerance = 1.0
    if geotype == "sb":
	m = re.match(r"([\d/]+)", geoid)
	if m is not None:
	    x_geo = m.group(0) # x_geo is numeric
	    if len(geoid) > len(x_geo):
		x_geo += "S"
		geoid = x_geo
    sql = "SELECT ST_AsGeoJSON(ST_Transform(ST_SimplifyPreserveTopology(%(geom)s, %(tolerance)f),4326)) AS bounds FROM hkcensus.%(geotable)s \
WHERE %(geokey)s = '%(geoid)s'" % { "geotable": geotable, "geokey": geokey, "geoid": geoid, "geom": geokey, "geom": geogeom, "tolerance": tolerance }
    rows = pgconn.query(sql)
    results = rows.getresult()
    if len(results) <= 0:
	return json.dumps({})
    return results[0][0]

    def user_timeline(self, user_id, count=200, page=1):
	start_time_api = time.time()
	api_misses = 0
	while api_misses < self.max_api_misses:
	    try:
		timeline = self.api.user_timeline(count=count, page=page, user_id=user_id)
		break
	    except httplib.IncompleteRead as h:
		print h
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": h }
		time.sleep(self.api_wait_secs)
	    except weibopy.error.WeibopError as e:
		print e.reason
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": e.reason }
		time.sleep(self.api_wait_secs)
		if string.find(e.reason, "requests out of rate limit") >= 0:
		    self.waitRateLimit()
	    except socket.error as e:
		print e
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": e.message }
		time.sleep(self.api_wait_secs)
	    '''
	    except ValueError as e:
		print user_id
		print e
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": e.message }
		time.sleep(self.api_wait_secs)
	    '''
	time_api = time.time() - start_time_api
	r = self.status_timeline(timeline, toBeginning=False)
	if "count" in r and r["count"] == 0:
	    if self.pgconn is None:
    		self.pgconn = mypass.getConn()
	    #self.pgconn.query("UPDATE sinaweibo_users SET posts_updated = NOW() WHERE id = %d" % user_id)
	r["time_api"] = time_api
	r["page"] = page
	return r

    def get_status(self):
        time_db = 0
        time_db_u = 0
        start_time_api = time.time()
        api_misses = 0
        while api_misses < self.max_api_misses:
            try:
                status = self.api2.statuses.show.get(id=self.id)
                break
            except weibo.APIError as e: ## Need more exception handling, and warned by > Python 2.6.
		if self.verbose > 0:
		    print e
		if e is not None and ("out of rate limit" in str(e).lower()):
		    self.changeToken()			
		    api_misses += 1
                if api_misses >= self.max_api_misses:
                    return { "id": self.id, "err_msg": e } ## aka toxicbar
                if e is not None and ("target weibo does not exist" in str(e).lower() or "permission denied" in str(e).lower()):
		    out = { 'id': self.id, "error_msg": str(e).lower(), "deleted": True, "permission_denied": False }
                    permission_denied = False
                    if ("permission denied" in str(e).lower()):
                        permission_denied = True
		    try:
			if self.pgconn is None:
		    	    self.pgconn = mypass.getConn()
			permission_sql = ""
			if "permission denied" in str(e).lower():
			    permission_sql = ", permission_denied = true"
			    out["permission_denied"] = True
			sql_deleted = "UPDATE rp_sinaweibo SET deleted = NOW() %(permission)s WHERE id = %(id)d AND deleted IS NULL " % { "id": self.id, "permission": permission_sql }
			if self.verbose > 0:
			    print "deleted %d " % self.id
		    	res = self.pgconn.query(sql_deleted)
			sql_status = "SELECT * FROM rp_sinaweibo WHERE id = %(id)d " % { "id": self.id }
			res_status = self.pgconn.query(sql_status).dictresult()
			out["deleted"] = True
			if len(res_status) > 0:
			    out["user_id"] = res_status[0]["user_id"]
			if self.verbose > 1:
			    out["status"] = res_status[0]
			    out["sql"] = sql_deleted
			return out
		    except pg.ProgrammingError, pg.InternalError:
			print self.pgconn.error
                time.sleep(self.api_wait_secs * 1)

    def user(self, user_id, screen_name=None, toDB=True):
	start_time_api = time.time()
	try:
	    if screen_name is not None:
		user = self.api.get_user(screen_name=screen_name)
	    else:
		user = self.api.get_user(user_id=user_id)
	except weibopy.error.WeibopError as e:
	    if e.reason.find("User does not exists") >= 0:
		if self.pgconn is None:
		    self.pgconn = mypass.getConn()
		try:
		    if not self.force_screenname and not user_id is None:
			sql_deleted = "UPDATE sinaweibo_users SET deleted = NOW() WHERE id = %d AND deleted IS NULL " % user_id
			res = self.pgconn.query(sql_deleted)
		except pg.ProgrammingError, pg.InternalError:
		    print self.pgconn.error
	    return { "msg": e.reason }

    def get_status(self, id, getUser=False, toDB=True):
	time_db = 0
	time_db_u = 0
	start_time_api = time.time()
	api_misses = 0
	while api_misses < self.max_api_misses:
	    try:
		status = self.api.get_status(id=id)
		break
	    except weibopy2.error.WeibopError as e:
		print e.reason
		api_misses += 1
		if api_misses >= self.max_api_misses:
		    return { "msg": e.reason }
		if e.reason is not None and ("target weibo does not exist" in e.reason.lower() or "permission denied" in e.reason.lower()):
		    out = { "msg": e.reason }
		    try:
			'''
			rps = self.getRangePartitionByIds([id])
		    	for x in rps:
			    yw = x.split(",")
			    year = int(yw[0])
			    week = int(yw[1])
			    sql_deleted = "UPDATE rp_sinaweibo_y%(year)dw%(week)d SET deleted = NOW() WHERE id = %(id)d AND deleted IS NULL " % { "year": year, "week": week, "id": id }
			    if self.pgconn is None:
				self.pgconn = mypass.getConn()
			'''
			if self.pgconn is None:
		    	    self.pgconn = mypass.getConn()
			permission_sql = ""
			if "permission denied" in e.reason.lower():
			    permission_sql = ", permission_denied = true"
			sql_deleted = "UPDATE rp_sinaweibo SET deleted = NOW() %(permission)s WHERE id = %(id)d AND deleted IS NULL " % { "id": id, "permission": permission_sql }
			print "deleted %d " % id
		    	res = self.pgconn.query(sql_deleted)
			sql_status = "SELECT * FROM rp_sinaweibo WHERE id = %(id)d " % { "id": id }
			res_status = self.pgconn.query(sql_status).dictresult()
			out["deleted"] = True
			if len(res_status) > 0:
			    out["user_id"] = res_status[0]["user_id"]
			return out
		    except pg.ProgrammingError, pg.InternalError:
			print self.pgconn.error
		time.sleep(self.api_wait_secs * 1)