Python config.Config类代码示例

 def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None):
     Config.__init__(self, config_file)
     self.aws_access_key_id = aws_access_key_id
     self.aws_secret_access_key = aws_secret_access_key
     script = Config.get(self, 'Pyami', 'scripts')
     if script:
         self.name = script.split('.')[-1]
     else:
         self.name = None

    def parse_aws_credentials():
        path = os.getenv('AWS_SHARED_CREDENTIALS_FILE', "~/.aws/credentials")
        if not os.path.exists(os.path.expanduser(path)):
            return None

        conf = Config(os.path.expanduser(path))

        if access_key_id == conf.get('default', 'aws_access_key_id'):
            return (access_key_id, conf.get('default', 'aws_secret_access_key'))
        return (conf.get(access_key_id, 'aws_access_key_id'),
                conf.get(access_key_id, 'aws_secret_access_key'))

 def create(cls, config_file=None, logical_volume = None, cfg = None, **params):
     if config_file:
         cfg = Config(path=config_file)
     if cfg.has_section('EC2'):
         # include any EC2 configuration values that aren't specified in params:
         for option in cfg.options('EC2'):
             if option not in params:
                 params[option] = cfg.get('EC2', option)
     getter = CommandLineGetter()
     getter.get(cls, params)
     region = params.get('region')
     ec2 = region.connect()
     cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key)
     ami = params.get('ami')
     kp = params.get('keypair')
     group = params.get('group')
     zone = params.get('zone')
     # deal with possibly passed in logical volume:
     if logical_volume != None:
        cfg.set('EBS', 'logical_volume_name', logical_volume.name) 
     cfg_fp = StringIO.StringIO()
     cfg.write(cfg_fp)
     # deal with the possibility that zone and/or keypair are strings read from the config file:
     if isinstance(zone, Zone):
         zone = zone.name
     if isinstance(kp, KeyPair):
         kp = kp.name
     reservation = ami.run(min_count=1,
                           max_count=params.get('quantity', 1),
                           key_name=kp,
                           security_groups=[group],
                           instance_type=params.get('instance_type'),
                           placement = zone,
                           user_data = cfg_fp.getvalue())
     l = []
     i = 0
     elastic_ip = params.get('elastic_ip')
     instances = reservation.instances
     if elastic_ip != None and instances.__len__() > 0:
         instance = instances[0]
         print 'Waiting for instance to start so we can set its elastic IP address...'
         while instance.update() != 'running':
             time.sleep(1)
         instance.use_ip(elastic_ip)
         print 'set the elastic IP of the first instance to %s' % elastic_ip
     for instance in instances:
         s = cls()
         s.ec2 = ec2
         s.name = params.get('name') + '' if i==0 else str(i)
         s.description = params.get('description')
         s.region_name = region.name
         s.instance_id = instance.id
         if elastic_ip and i == 0:
             s.elastic_ip = elastic_ip
         s.put()
         l.append(s)
         i += 1
     return l

 def getConfig(self):
     if not self._config:
         remote_file = BotoConfigPath
         local_file = '%s.ini' % self.instance.id
         self.get_file(remote_file, local_file)
         self._config = Config(local_file)
     return self._config

 def getint(self, option, default=0):
     try:
         val = Config.get(self, self.name, option)
         val = int(val)
     except:
         val = int(default)
     return val

    def __init__(self, name, access_key=None, secret_key=None,
                 security_token=None, profile_name=None):
        self.host = None
        self.port = None
        self.host_header = None
        self.access_key = access_key
        self.secret_key = secret_key
        self.security_token = security_token
        self.profile_name = profile_name
        self.name = name
        self.acl_class = self.AclClassMap[self.name]
        self.canned_acls = self.CannedAclsMap[self.name]
        self._credential_expiry_time = None

        # Load shared credentials file if it exists
        shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
        self.shared_credentials = Config(do_load=False)
        if os.path.isfile(shared_path):
            self.shared_credentials.load_from_path(shared_path)

        self.get_credentials(access_key, secret_key, security_token, profile_name)
        self.configure_headers()
        self.configure_errors()

        # Allow config file to override default host and port.
        host_opt_name = '%s_host' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', host_opt_name):
            self.host = config.get('Credentials', host_opt_name)
        port_opt_name = '%s_port' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', port_opt_name):
            self.port = config.getint('Credentials', port_opt_name)
        host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', host_header_opt_name):
            self.host_header = config.get('Credentials', host_header_opt_name)

 def getbool(self, option, default=False):
     try:
         val = Config.get(self, self.name, option)
         if val.lower() == 'true':
             val = True
         else:
             val = False
     except:
         val = default
     return val

def get_aws_credentials(config_file):
    # Try to read .boto configuration from several places (later ones take precedence)
    try: # user home directory (~/.boto)
        boto_cfg = Config(os.path.join(os.path.expanduser('~'), '.boto'))
    except: pass
    try: # current directory (./.boto)
        boto_cfg = Config('.boto')
    except: pass
    try: # command line option (--config <file>)
        if config_file: boto_cfg = Config(config_file)
    except: pass

    # Load the AWS key credentials
    try:
        access_key = boto_cfg.get('Credentials', 'aws_access_key_id')
        secret_key = boto_cfg.get('Credentials', 'aws_secret_access_key')
    except:
        print >> sys.stderr, 'Could not find .boto config file'
        sys.exit(1)
        
    return (access_key, secret_key)

    def setUp(self):
        self.config = Config()

        # Enable https_validate_certificates.
        self.config.add_section('Boto')
        self.config.setbool('Boto', 'https_validate_certificates', True)

        # Set up bogus credentials so that the auth module is willing to go
        # ahead and make a request; the request should fail with a service-level
        # error if it does get to the service (S3 or GS).
        self.config.add_section('Credentials')
        self.config.set('Credentials', 'gs_access_key_id', 'xyz')
        self.config.set('Credentials', 'gs_secret_access_key', 'xyz')
        self.config.set('Credentials', 'aws_access_key_id', 'xyz')
        self.config.set('Credentials', 'aws_secret_access_key', 'xyz')

        self._config_patch = mock.patch('boto.config', self.config)
        self._config_patch.start()

 def get(self, name, default=None):
     return Config.get(self, self.name, name, default)

class CertValidationTest(unittest.TestCase):
    def setUp(self):
        self.config = Config()

        # Enable https_validate_certificates.
        self.config.add_section('Boto')
        self.config.setbool('Boto', 'https_validate_certificates', True)

        # Set up bogus credentials so that the auth module is willing to go
        # ahead and make a request; the request should fail with a service-level
        # error if it does get to the service (S3 or GS).
        self.config.add_section('Credentials')
        self.config.set('Credentials', 'gs_access_key_id', 'xyz')
        self.config.set('Credentials', 'gs_secret_access_key', 'xyz')
        self.config.set('Credentials', 'aws_access_key_id', 'xyz')
        self.config.set('Credentials', 'aws_secret_access_key', 'xyz')

        self._config_patch = mock.patch('boto.config', self.config)
        self._config_patch.start()

    def tearDown(self):
        self._config_patch.stop()

    def enableProxy(self):
        self.config.set('Boto', 'proxy', PROXY_HOST)
        self.config.set('Boto', 'proxy_port', PROXY_PORT)

    def assertConnectionThrows(self, connection_class, error):
        conn = connection_class('fake_id', 'fake_secret')
        self.assertRaises(error, conn.get_all_buckets)

    def do_test_valid_cert(self):
        # When connecting to actual servers with bundled root certificates, no
        # cert errors should be thrown; instead we will get "invalid
        # credentials" errors since the config used does not contain any
        # credentials.
        self.assertConnectionThrows(S3Connection, exception.S3ResponseError)
        self.assertConnectionThrows(GSConnection, exception.GSResponseError)

    def test_valid_cert(self):
        self.do_test_valid_cert()

    def test_valid_cert_with_proxy(self):
        self.enableProxy()
        self.do_test_valid_cert()

    def do_test_invalid_signature(self):
        self.config.set('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
        self.assertConnectionThrows(S3Connection, ssl.SSLError)
        self.assertConnectionThrows(GSConnection, ssl.SSLError)

    def test_invalid_signature(self):
        self.do_test_invalid_signature()

    def test_invalid_signature_with_proxy(self):
        self.enableProxy()
        self.do_test_invalid_signature()

    def do_test_invalid_host(self):
        self.config.set('Credentials', 'gs_host', INVALID_HOSTNAME_HOST)
        self.config.set('Credentials', 's3_host', INVALID_HOSTNAME_HOST)
        self.assertConnectionThrows(S3Connection, ssl.SSLError)
        self.assertConnectionThrows(GSConnection, ssl.SSLError)

    def do_test_invalid_host(self):
        self.config.set('Credentials', 'gs_host', INVALID_HOSTNAME_HOST)
        self.config.set('Credentials', 's3_host', INVALID_HOSTNAME_HOST)
        self.assertConnectionThrows(
                S3Connection, https_connection.InvalidCertificateException)
        self.assertConnectionThrows(
                GSConnection, https_connection.InvalidCertificateException)

    def test_invalid_host(self):
        self.do_test_invalid_host()

    def test_invalid_host_with_proxy(self):
        self.enableProxy()
        self.do_test_invalid_host()

#.........这里部分代码省略.........
    ErrorMap = {
        'aws': {
            STORAGE_COPY_ERROR: boto.exception.S3CopyError,
            STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
            STORAGE_DATA_ERROR: boto.exception.S3DataError,
            STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
            STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
        },
        'google': {
            STORAGE_COPY_ERROR: boto.exception.GSCopyError,
            STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
            STORAGE_DATA_ERROR: boto.exception.GSDataError,
            STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
            STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
        }
    }

    def __init__(self, name, access_key=None, secret_key=None,
                 security_token=None, profile_name=None):
        self.host = None
        self.port = None
        self.host_header = None
        self.access_key = access_key
        self.secret_key = secret_key
        self.security_token = security_token
        self.profile_name = profile_name
        self.name = name
        self.acl_class = self.AclClassMap[self.name]
        self.canned_acls = self.CannedAclsMap[self.name]
        self._credential_expiry_time = None

        # Load shared credentials file if it exists
        shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
        self.shared_credentials = Config(do_load=False)
        if os.path.isfile(shared_path):
            self.shared_credentials.load_from_path(shared_path)

        self.get_credentials(access_key, secret_key, security_token, profile_name)
        self.configure_headers()
        self.configure_errors()

        # Allow config file to override default host and port.
        host_opt_name = '%s_host' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', host_opt_name):
            self.host = config.get('Credentials', host_opt_name)
        port_opt_name = '%s_port' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', port_opt_name):
            self.port = config.getint('Credentials', port_opt_name)
        host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
        if config.has_option('Credentials', host_header_opt_name):
            self.host_header = config.get('Credentials', host_header_opt_name)

    def get_access_key(self):
        if self._credentials_need_refresh():
            self._populate_keys_from_metadata_server()
        return self._access_key

    def set_access_key(self, value):
        self._access_key = value

    access_key = property(get_access_key, set_access_key)

    def get_secret_key(self):
        if self._credentials_need_refresh():
            self._populate_keys_from_metadata_server()
        return self._secret_key

def main(argv):
    parser = argparse.ArgumentParser(description='Upload assets to Amazon')
    parser.add_argument('--config',
                        dest='config_filename',
                        action='store',
                        default=CONFIG_FILE,
                        help='optional custom configuration filename')
    parser.add_argument('--node',
                        dest='node_name_override',
                        action='store',
                        default=False,
                        help='optional override for the pid-id specified in the config file')
    parameters = parser.parse_args()

    current_defaults_filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), parameters.config_filename)
    config = Config(path=current_defaults_filename)
    global access_key_id
    global secret_access_key
    access_key_id = config.get('Amazon', 'aws_access_key_id')
    secret_access_key = config.get('Amazon', 'aws_secret_access_key')
    log_file_path = config.get('General', 'log_file_path', '/var/log/s3ingest.log')
    log_level = config.getint('General', 'log_level', 20)
    target_bucket_name = config.get('Amazon', 's3_bucket_name')
    monitored_dir_name = config.get('General', 'monitored_directory')
    worker_threads = config.getint('General', 'worker_threads', 5)
    pid_file_path = config.get('General', 'pid_file_path', './s3ingest.semaphore')
    if not parameters.node_name_override:
        pid_id = config.get('General', 'pid_id').rstrip()
    else:
        pid_id = parameters.node_name_override.rstrip()
    HEART_BEAT_TIME_SECS = config.getint('General', 'heart_beat_time_secs', 300)
    MIN_MODIFIED_INTERVAL_SECS = 3600 # 3600 secs = 1 hr. Keep high to allow time for large files to upload and reduce false positives

    if not os.path.exists(monitored_dir_name):
        print "The directory to be monitored '{0}' does not exist".format(monitored_dir_name)
        sys.exit(1)

    logging.basicConfig(filename=log_file_path, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=log_level)
    mailhost = config.get('Mail', 'mailhost')
    fromaddr = config.get('Mail', 'fromaddr')
    toaddrs = config.get('Mail', 'toaddrs')
    smtp_handler = handlers.SMTPHandler(mailhost, fromaddr, toaddrs, 'S3Util error occurred')
    smtp_handler.setLevel(logging.ERROR)
    logging.getLogger().addHandler(smtp_handler)
    
    s3_util = S3Util(access_key_id, secret_access_key)

    s3_util.set_target_bucket_name(target_bucket_name)
    signal.signal(signal.SIGINT, s3_util.signal_handler)
    signal.signal(signal.SIGTERM, s3_util.signal_handler)

    # Check for pid file and create if not found
    if not os.path.exists(pid_file_path):
        pid_file = open(pid_file_path, "w+")
        fcntl.flock(pid_file.fileno(), fcntl.LOCK_EX)
        pid_file.write(str(pid_id))
        fcntl.flock(pid_file.fileno(), fcntl.LOCK_UN)
        pid_file.close()

    s3_util.start_monitoring(monitored_dir_name)

    logging.debug("Starting worker threads")
    for i in range(worker_threads):
        t = S3Uploader(s3_util)
        t.setDaemon(True)
        t.start()

    logging.debug("Worker threads started")

    while True:
        pid_file = open(pid_file_path, "r+")
        logging.debug("Waiting for lock")
        fcntl.flock(pid_file.fileno(), fcntl.LOCK_SH)
        logging.debug("Acquired lock")
        current_pid = pid_file.readline().rstrip()
        st = os.stat(pid_file_path)
        now = time.time()
        pid_modified_time = st[stat.ST_MTIME]
        logging.debug("pid file: {0}, current_host: {1}".format(current_pid, pid_id))
        if pid_id == current_pid:
            logging.debug("State - Active")
            os.utime(pid_file_path, None)
            s3_util.set_active(True)
            # Find files have been unmodified for a defined threshold and assume that they need to be queued
            for dirpath, dirnames, filenames in os.walk(monitored_dir_name):
                for name in filenames:
                    file_path = os.path.normpath(os.path.join(dirpath, name))
                    last_modifed_time = os.path.getmtime(file_path)
                    if ((now - last_modifed_time) > MIN_MODIFIED_INTERVAL_SECS and not
                        (s3_util.is_queued(file_path) or s3_util.is_currently_processing(file_path))):
                        logging.info("Directory scan found file '{0}' older than {1} seconds and added to queue".format(file_path, (now - last_modifed_time)))
                        s3_util.add_to_queue(file_path)
        else:
            if now - pid_modified_time > HEART_BEAT_TIME_SECS:
                logging.debug("Stale pid file found, setting state - Active")
                pid_file.truncate(0)
                pid_file.seek(0)
                pid_file.write(str(pid_id))
                s3_util.set_active(True)
            else:
#.........这里部分代码省略.........

   exit()

# Get Command Line Arguments
PHRASE        = sys.argv[1]
JOB_ID        = sys.argv[2]
ITERATION     = sys.argv[3]
PARENT_HIT_ID = sys.argv[4]
BRANCHES      = sys.argv[5]


# Connect to the HIT database
database = sqlite3.connect('crowdstorming.db', isolation_level='DEFERRED')
db       = database.cursor()

# BOTO Configuration
config = Config()
AWS_ID = config.get('Credentials', 'aws_access_key_id', None)
SECRET_ID = config.get('Credentials', 'aws_secret_access_key_id', None)
HOST = 'mechanicalturk.amazonaws.com'

mt = MTurkConnection(
   aws_access_key_id=AWS_ID, 
   aws_secret_access_key=SECRET_ID, 
   host=HOST
   )

# HIT Configuration - global title, description, keywords, qualifications
TITLE    = 'Provide Related Terms'
DESC     = 'Given a word or phrase, provide another (different) word that relates to the given one.'
KEYWORDS = 'opinions, relations, idea, brainstorm, crowdstorm'
QUAL     = Qualifications()

 def load_config(self):
     self._config = Config(do_load=False)
     self._config.load_from_sdb("botoConfigs", self.id)