Python report_utils.reporter函数代码示例

    def object_lister(self, url, container, object_count=None, last_obj=None):
        """Builds a long list of objects found in a container.

        NOTE: This could be millions of Objects.

        :param url:
        :param container:
        :param object_count:
        :param last_obj:
        :return None | list:
        """

        for retry in basic.retryloop(attempts=ARGS.get("error_retry"), obj="Object List"):
            fheaders = self.payload["headers"]
            fpath = http.quoter(url=url.path, cont=container)
            with meth.operation(retry, obj="%s %s" % (fheaders, fpath)):
                resp = self._header_getter(url=url, rpath=fpath, fheaders=fheaders)
                if resp.status_code == 404:
                    report.reporter(msg="Not found. %s | %s" % (resp.status_code, resp.request))
                    return None, None, None
                else:
                    if object_count is None:
                        object_count = resp.headers.get("x-container-object-count")
                        if object_count:
                            object_count = int(object_count)
                            if not object_count > 0:
                                return None, None, None
                        else:
                            return None, None, None

                    # Set the number of loops that we are going to do
                    return self._list_getter(url=url, filepath=fpath, fheaders=fheaders, last_obj=last_obj)

def authenticate():
    """Authentication For Openstack API.

    Pulls the full Openstack Service Catalog Credentials are the Users API
    Username and Key/Password "osauth" has a Built in Rackspace Method for
    Authentication

    Set a DC Endpoint and Authentication URL for the OpenStack environment
    """

    # Setup the request variables
    url = auth.parse_region()
    a_url = http.parse_url(url=url, auth=True)
    auth_json = auth.parse_reqtype()

    # remove the prefix for the Authentication URL if Found
    LOG.debug("POST == REQUEST DICT > JSON DUMP %s", auth_json)
    auth_json_req = json.dumps(auth_json)
    headers = {"Content-Type": "application/json"}

    # Send Request
    request = ("POST", a_url.path, auth_json_req, headers)
    resp_read = auth.request_process(aurl=a_url, req=request)
    LOG.debug("POST Authentication Response %s", resp_read)
    try:
        auth_resp = json.loads(resp_read)
    except ValueError as exp:
        LOG.error("Authentication Failure %s\n%s", exp, traceback.format_exc())
        raise turbo.SystemProblem("JSON Decode Failure. ERROR: %s - RESP %s" % (exp, resp_read))
    else:
        auth_info = auth.parse_auth_response(auth_resp)
        token, tenant, user, inet, enet, cnet, acfep = auth_info
        report.reporter(msg=("API Access Granted. TenantID: %s Username: %s" % (tenant, user)), prt=False, log=True)
        return token, tenant, user, inet, enet, cnet, a_url, acfep

    def start(self):
        """Retrieve a long list of all files in a container."""

        # Package up the Payload
        payload = http.prep_payload(auth=self.auth, container=None, source=None, args=ARGS)

        # Prep Actions.
        self.go = actions.CloudActions(payload=payload)

        report.reporter(msg='PAYLOAD\t: "%s"' % payload, log=True, lvl="debug", prt=False)

        with multi.spinner():
            if ARGS.get("cdn_info"):
                url = payload["cnet"]
            else:
                url = payload["url"]
            message = self.go.detail_show(url=url)

        try:
            if message.status_code != 404:
                report.reporter(msg="Object Found...")
                report.reporter(msg=report.print_virt_table(dict(message.headers)))
            else:
                report.reporter(msg="Nothing Found...")
        except ValueError as exp:
            report.reporter(msg=("Non-hashable Type, Likley Item is not found." " Additional Data: %s" % exp))

def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
                  opt=None):
    """Process all jobs in batches.

    :param num_jobs:
    :param objects:
    :param job_action:
    :param concur:
    :param payload:
    """

    count = 0
    batch_size = basic.batcher(num_files=num_jobs)
    for work in basic.batch_gen(data=objects,
                                batch_size=batch_size,
                                count=num_jobs):
        count += 1
        report.reporter(msg='Job Count %s' % count)
        work_q = basic_queue(work)
        with spinner(work_q=work_q):
            worker_proc(job_action=job_action,
                        concurrency=concur,
                        queue=work_q,
                        opt=opt,
                        kwargs=kwargs)
            basic.stupid_hack(wait=1)
        work_q.close()

def set_concurrency(args, file_count):
    """Concurrency is a user specified variable when the arguments parsed.

    :param args:

    However if the number of things Turbo lift has to do is less than the
    desired concurency, then turbolift will lower the concurency rate to
    the number of operations.
    """

    def verbose(ccr):
        report.reporter(
            msg='MESSAGE: We are creating %s Processes' % ccr,
            prt=False
        )
        return ccr

    _cc = args.get('cc')

    if _cc > file_count:
        report.reporter(
            msg=('MESSAGE: There are less things to do than the number of'
                 ' concurrent processes specified by either an override'
                 ' or the system defaults. I am leveling the number of'
                 ' concurrent processes to the number of jobs to perform.'),
            lvl='warn'
        )
        return verbose(ccr=file_count)
    else:
        return verbose(ccr=_cc)

def job_processer(num_jobs, objects, job_action, concur, kwargs=None,
                  opt=None):
    """Process all jobs in batches.

    :param num_jobs:
    :param objects:
    :param job_action:
    :param concur:
    :param payload:
    """

    count = 0
    batch_size = basic.batcher(num_files=num_jobs)
    while objects:
        count += 1
        report.reporter(msg='Job Count %s' % count)
        work = [
            objects.pop(objects.index(obj)) for obj in objects[0:batch_size]
        ]
        work_q = basic_queue(work)
        with spinner(work_q=work_q):
            worker_proc(
                job_action=job_action,
                concurrency=concur,
                queue=work_q,
                opt=opt,
                kwargs=kwargs
            )
            basic.stupid_hack(wait=.2)
        work_q.close()

    def _putter(self, conn, fpath, rpath, fheaders, retry, skip=False):
        """Place  object into the container.

        :param conn:
        :param fpath:
        :param rpath:
        :param fheaders:
        :param retry:
        """

        if self._checker(conn, rpath, fpath, fheaders, retry, skip) is True:
            report.reporter(
                msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
                prt=False,
                lvl='debug'
            )

            if basic.file_exists(fpath) is False:
                return None
            else:
                with open(fpath, 'rb') as f_open:
                    conn.request('PUT', rpath, body=f_open, headers=fheaders)
                resp, read = http.response_get(conn=conn, retry=retry)
                self.resp_exception(resp=resp, rty=retry)

                report.reporter(
                    msg=('MESSAGE %s %s %s'
                         % (resp.status, resp.reason, resp.msg)),
                    prt=False,
                    lvl='debug'
                )

    def container_cdn_command(self, url, container, sfile=None):
        """Command your CDN enabled Container.

        :param url:
        :param container:
        """
        rty_count = ARGS.get('error_retry')
        for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile):
            # Open Connection
            conn = http.open_connection(url=url)
            with meth.operation(retry, conn):
                cheaders = self.payload['headers']
                if sfile is not None:
                    rpath = http.quoter(url=url.path,
                                        cont=container,
                                        ufile=sfile)
                    # perform CDN Object DELETE
                    conn.request('DELETE', rpath, headers=cheaders)
                    resp, read = http.response_get(conn=conn, retry=retry)
                    self.resp_exception(resp=resp, rty=retry)
                else:
                    rpath = http.quoter(url=url.path,
                                        cont=container)
                    http.cdn_toggle(headers=cheaders)
                    # perform CDN Enable POST
                    conn.request('PUT', rpath, headers=cheaders)
                    resp, read = http.response_get(conn=conn, retry=retry)
                    self.resp_exception(resp=resp, rty=retry)

                report.reporter(
                    msg=('OBJECT %s MESSAGE %s %s %s'
                         % (rpath, resp.status, resp.reason, resp.msg)),
                    prt=False,
                    lvl='debug'
                )

    def _putter(self, url, fpath, rpath, fheaders, skip=False):
        """Place  object into the container.

        :param url:
        :param fpath:
        :param rpath:
        :param fheaders:
        """

        if self._checker(url, rpath, fpath, fheaders, skip) is True:
            report.reporter(
                msg='OBJECT ORIGIN %s RPATH %s' % (fpath, rpath),
                prt=False,
                lvl='debug'
            )

            if basic.file_exists(fpath) is False:
                return None
            else:
                with open(fpath, 'rb') as f_open:
                    resp = http.put_request(
                        url=url, rpath=rpath, body=f_open, headers=fheaders
                    )
                    self.resp_exception(resp=resp)

                report.reporter(
                    msg=('MESSAGE %s %s %s'
                         % (resp.status_code, resp.reason, resp.request)),
                    prt=False,
                    lvl='debug'
                )

    def _downloader(self, url, rpath, fheaders, lfile, source,
                    skip=False):
        """Download a specified object in the container.

        :param url:
        :param rpath:
        :param fheaders:
        :param lfile:
        :param skip:
        """

        resp = None

        if source is None:
            local_f = lfile
        else:
            local_f = basic.jpath(root=source, inode=lfile)

        if self._checker(url, rpath, local_f, fheaders, skip) is True:
            report.reporter(
                msg='Downloading remote %s to local file %s' % (rpath, lfile),
                prt=False,
                lvl='debug',
            )

            # Perform Object GET
            resp = http.get_request(
                url=url, rpath=rpath, headers=fheaders, stream=True
            )
            self.resp_exception(resp=resp)
            local_f = basic.collision_rename(file_name=local_f)

            # Open our source file and write it
            with open(local_f, 'wb') as f_name:
                for chunk in resp.iter_content(chunk_size=2048):
                    if chunk:
                        f_name.write(chunk)
                        f_name.flush()
            resp.close()

        if ARGS.get('restore_perms') is not None:
            # Make a connection
            if resp is None:
                resp = self._header_getter(
                    url=url, rpath=rpath, fheaders=fheaders
                )

            all_headers = resp.headers

            if all(['x-object-meta-group' in all_headers,
                    'x-object-meta-owner' in all_headers,
                    'x-object-meta-perms' in all_headers]):
                basic.restor_perms(local_file=local_f, headers=all_headers)
            else:
                report.reporter(
                    msg=('No Permissions were restored, because none were'
                         ' saved on the object "%s"' % rpath),
                    lvl='warn',
                    log=True
                )

    def container_cdn_command(self, url, container, sfile=None):
        """Command your CDN enabled Container.

        :param url:
        :param container:
        """

        rty_count = ARGS.get("error_retry")
        for retry in basic.retryloop(attempts=rty_count, delay=2, obj=sfile):
            cheaders = self.payload["headers"]
            if sfile is not None:
                rpath = http.quoter(url=url.path, cont=container, ufile=sfile)
                # perform CDN Object DELETE
                adddata = "%s %s" % (cheaders, container)
                with meth.operation(retry, obj=adddata):
                    resp = http.delete_request(url=url, rpath=rpath, headers=cheaders)
                    self.resp_exception(resp=resp)
            else:
                rpath = http.quoter(url=url.path, cont=container)
                http.cdn_toggle(headers=cheaders)

                # perform CDN Enable PUT
                adddata = "%s %s" % (cheaders, container)
                with meth.operation(retry, obj=adddata):
                    resp = http.put_request(url=url, rpath=rpath, headers=cheaders)
                    self.resp_exception(resp=resp)

            report.reporter(
                msg="OBJECT %s MESSAGE %s %s %s" % (rpath, resp.status_code, resp.reason, resp.request),
                prt=False,
                lvl="debug",
            )

        def _deleterator(payload):
            """Multipass Object Delete."""

            report.reporter(msg='Getting file list')
            with multi.spinner():
                # Get all objects in a Container
                objects, list_count, last_obj = self.action(
                    url=payload['url'],
                    container=payload['c_name']
                )

                if ARGS.get('pattern_match'):
                    objects = basic.match_filter(
                        idx_list=objects,
                        pattern=ARGS['pattern_match'],
                        dict_type=True
                    )

                # Count the number of objects returned.
                if objects is False:
                    report.reporter(msg='No Container found.')
                    return
                elif objects is not None:
                    # Load the queue
                    obj_list = [obj['name'] for obj in objects]
                    num_files = len(obj_list)
                    if num_files < 1:
                        report.reporter(msg='No Objects found.')
                        return
                else:
                    report.reporter(msg='Nothing found.')
                    return

                # Get The rate of concurrency
                concurrency = multi.set_concurrency(args=ARGS,
                                                    file_count=num_files)

                if ARGS.get('object'):
                    obj_names = ARGS.get('object')
                    obj_list = [obj for obj in obj_list if obj in obj_names]
                    if not obj_list:
                        return 'Nothing Found to Delete.'
                    num_files = len(obj_list)
                report.reporter(
                    msg=('Performing Object Delete for "%s" object(s)...'
                         % num_files)
                )
                kwargs = {'url': payload['url'],
                          'container': payload['c_name'],
                          'cf_job': getattr(self.go, 'object_deleter')}
            multi.job_processer(
                num_jobs=num_files,
                objects=obj_list,
                job_action=multi.doerator,
                concur=concurrency,
                kwargs=kwargs
            )
            _deleterator(payload=payload)

def authenticate():
    """Authentication For Openstack API.

    Pulls the full Openstack Service Catalog Credentials are the Users API
    Username and Key/Password "osauth" has a Built in Rackspace Method for
    Authentication

    Set a DC Endpoint and Authentication URL for the OpenStack environment
    """

    # Setup the request variables
    a_url = "https://zebra.zerovm.org/auth/v1.0"
    #a_url = http.parse_url(url=url, auth=True)
    auth_json = auth.parse_reqtype()
    print auth_json
    # remove the prefix for the Authentication URL if Found
#    LOG.debug('POST == REQUEST DICT > JSON DUMP %s', auth_json)
#    auth_json_req = json.dumps(auth_json)
    headers = {
        'Content-Type': 'application/json',
        "X-Auth-User": auth_json['auth']['passwordCredentials']['username'],
        "X-Auth-Key": auth_json['auth']['passwordCredentials']['password']}

    # Send Request
    try:
        auth_resp = requests.get(
            url=a_url, headers=headers
        )
        if auth_resp.status_code >= 300:
            raise SystemExit(
                'Authentication Failure, %s %s' % (auth_resp.status_code,
                                                   auth_resp.reason)
            )
    except ValueError as exp:
        LOG.error('Authentication Failure %s\n%s', exp, traceback.format_exc())
        raise turbo.SystemProblem('JSON Decode Failure. ERROR: %s' % exp)
    else:
        LOG.debug('POST Authentication Response %s', auth_resp.json())
        #auth_info = auth.parse_auth_response(auth_resp.json())
        #token, tenant, user, inet, enet, cnet, acfep = auth_info
        token = auth_resp.headers['x-auth-token']
        tenant, user = auth_json['auth']['passwordCredentials']['username'].split(":")
        inet = urlparse.urlparse(auth_resp.headers['x-storage-url'])
        enet = inet
        cnet = None
        acfep = inet
        report.reporter(
            msg=('API Access Granted. TenantID: %s Username: %s'
                 % (tenant, user)),
            prt=False,
            log=True
        )
        return token, tenant, user, inet, enet, cnet, urlparse.urlparse(a_url), acfep

def batcher(num_files):
    """Check the batch size and return it.

    :param num_files:
    :return int:
    """

    batch_size = turbo.ARGS.get("batch_size")
    report.reporter(msg='Job process MAX Batch Size is "%s"' % batch_size, lvl="debug", log=True, prt=False)
    if num_files > batch_size:
        ops = num_files / batch_size + 1
        report.reporter(msg='This will take "%s" operations to complete.' % ops, lvl="warn", log=True, prt=True)
    return batch_size

    def object_lister(self, url, container, object_count=None, last_obj=None):
        """Builds a long list of objects found in a container.

        NOTE: This could be millions of Objects.

        :param url:
        :param container:
        :param object_count:
        :param last_obj:
        :return None | list:
        """

        for retry in basic.retryloop(attempts=ARGS.get('error_retry'),
                                     obj='Object List'):
            # Open Connection
            conn = http.open_connection(url=url)

            # Open connection and perform operation
            with meth.operation(retry, conn):
                # Determine how many files are in the container
                fpath = http.quoter(url=url.path,
                                    cont=container)
                # Make a connection
                resp = self._header_getter(conn=conn,
                                           rpath=fpath,
                                           fheaders=self.payload['headers'],
                                           retry=retry)
                if resp.status == 404:
                    report.reporter(
                        msg='Not found. %s | %s' % (resp.status, resp.msg)
                    )
                    return None, None, None
                else:
                    if object_count is None:
                        head_check = dict(resp.getheaders())
                        object_count = head_check.get(
                            'x-container-object-count'
                        )
                        if object_count:
                            object_count = int(object_count)
                            if not object_count > 0:
                                return None, None, None
                        else:
                            return None, None, None

                    # Set the number of loops that we are going to do
                    return self._list_getter(conn=conn,
                                             count=object_count,
                                             filepath=fpath,
                                             fheaders=self.payload['headers'],
                                             last_obj=last_obj)