Python fingerprint_404.is_404函数代码示例

    def _setup_404_detection(self):
        #
        #    NOTE: I need to perform this test here in order to avoid some weird
        #    thread locking that happens when the webspider calls is_404, and
        #    because I want to initialize the is_404 database in a controlled
        #    try/except block.
        #
        from w3af.core.controllers.core_helpers.fingerprint_404 import is_404

        for url in cf.cf.get('targets'):
            try:
                response = self._w3af_core.uri_opener.GET(url, cache=True)
            except ScanMustStopByUserRequest:
                raise
            except Exception, e:
                msg = ('Failed to send HTTP request to the configured target'
                       ' URL "%s", the original exception was: "%s" (%s).')
                args = (url, e, e.__class__.__name__)
                raise ScanMustStopException(msg % args)

            try:
                is_404(response)
            except ScanMustStopByUserRequest:
                raise
            except Exception, e:
                msg = ('Failed to initialize the 404 detection using HTTP'
                       ' response from "%s", the original exception was: "%s"'
                       ' (%s).')
                args = (url, e, e.__class__.__name__)
                raise ScanMustStopException(msg % args)

    def grep(self, request, response):
        """
        Plugin entry point. Get responses, analyze words, create dictionary.

        :param request: The HTTP request object.
        :param response: The HTTP response object
        :return: None.
        """
        if not self.got_lang():
            return

        # I added the 404 code here to avoid doing some is_404 lookups
        if response.get_code() not in self.BANNED_STATUS \
        and not is_404(response) \
        and request.get_method() in self.ALLOWED_METHODS:

            # Run the plugins
            data = self._run_plugins(response)

            with self._plugin_lock:
                old_data = kb.kb.raw_read(self.get_name(), self.get_name())

                new_data = self.merge_maps(old_data, data, request,
                                           self.captured_lang)

                new_data = self._trim_data(new_data)

                # save the updated map
                kb.kb.raw_write(self, self.get_name(), new_data)

    def _extract_urls(self, fuzzable_request, response):
        """
        Extract information from the server-status page and send FuzzableRequest
        instances to the core.
        """
        self.output_queue.put(FuzzableRequest(response.get_url()))

        # Now really parse the file and create custom made fuzzable requests
        regex = "<td>.*?<td nowrap>(.*?)</td><td nowrap>.*? (.*?) HTTP/1"
        for domain, path in re.findall(regex, response.get_body()):

            if "unavailable" in domain:
                domain = response.get_url().get_domain()

            # Check if the requested domain and the found one are equal.
            if domain == response.get_url().get_domain():
                proto = response.get_url().get_protocol()
                found_url = proto + "://" + domain + path
                found_url = URL(found_url)

                # They are equal, request the URL and create the fuzzable
                # requests
                tmp_res = self._uri_opener.GET(found_url, cache=True)
                if not is_404(tmp_res):
                    self.output_queue.put(FuzzableRequest(found_url))
            else:
                # This is a shared hosting server
                self._shared_hosting_hosts.append(domain)

    def _confirm_file_upload(self, path, mutant, http_response):
        """
        Confirms if the file was uploaded to path

        :param path: The URL where we suspect that a file was uploaded to.
        :param mutant: The mutant that originated the file on the remote end
        :param http_response: The HTTP response asociated with sending mutant
        """
        get_response = self._uri_opener.GET(path, cache=False)

        if not is_404(get_response) and self._has_no_bug(mutant):
            desc = "A file upload to a directory inside the webroot" " was found at: %s" % mutant.found_at()

            v = Vuln.from_mutant(
                "Insecure file upload",
                desc,
                severity.HIGH,
                [http_response.id, get_response.id],
                self.get_name(),
                mutant,
            )

            v["file_dest"] = get_response.get_url()
            v["file_vars"] = mutant.get_file_vars()

            self.kb_append_uniq(self, "file_upload", v)

    def _exists_in_target(self, url):
        """
        Check if a resource still exists in the target web site.

        :param url: The resource to verify.
        :return: None, the result is stored in self.output_queue
        """
        if url in self._already_verified:
            return

        self._already_verified.add(url)

        response = self._uri_opener.GET(url, cache=True)

        if not is_404(response):
            msg = 'The URL: "%s" was found at archive.org and is'\
                  ' STILL AVAILABLE in the target site.'
            om.out.debug(msg % url)

            fr = FuzzableRequest(response.get_uri())
            self.output_queue.put(fr)
        else:
            msg = 'The URL: "%s" was found at archive.org and was'\
                  ' DELETED from the target site.'
            om.out.debug(msg % url)

    def _confirm_file_upload(self, path, mutant, http_response):
        """
        Confirms if the file was uploaded to path

        :param path: The URL where we suspect that a file was uploaded to.
        :param mutant: The mutant that originated the file on the remote end
        :param http_response: The HTTP response asociated with sending mutant
        """
        get_response = self._uri_opener.GET(path, cache=False)

        if not is_404(get_response) and self._has_no_bug(mutant):
            # This is necessary, if I don't do this, the session
            # saver will break cause REAL file objects can't
            # be picked
            mutant.set_mod_value('<file_object>')

            desc = 'A file upload to a directory inside the webroot' \
                   ' was found at: %s' % mutant.found_at()
            
            v = Vuln.from_mutant('Insecure file upload', desc, severity.HIGH,
                                 [http_response.id, get_response.id],
                                 self.get_name(), mutant)
            
            v['file_dest'] = get_response.get_url()
            v['file_vars'] = mutant.get_file_vars()

            self.kb_append_uniq(self, 'file_upload', v)

    def audit(self, freq, orig_response):
        """
        Searches for file upload vulns using a POST to author.dll.

        :param freq: A FuzzableRequest
        """
        domain_path = freq.get_url().get_domain_path()

        if kb.kb.get(self, 'frontpage'):
            # Nothing to do, I have found vuln(s) and I should stop on first
            msg = 'Not verifying if I can upload files to: "%s" using'\
                  ' author.dll. Because I already found a vulnerability.'
            om.out.debug(msg)
            return

        # I haven't found any vulns yet, OR i'm trying to find every
        # directory where I can write a file.
        if domain_path not in self._already_tested:
            self._already_tested.add(domain_path)

            # Find a file that doesn't exist and then try to upload it
            for _ in xrange(3):
                rand_file = rand_alpha(5) + '.html'
                rand_path_file = domain_path.url_join(rand_file)
                res = self._uri_opener.GET(rand_path_file)
                if is_404(res):
                    upload_id = self._upload_file(domain_path, rand_file)
                    self._verify_upload(domain_path, rand_file, upload_id)
                    break
            else:
                msg = 'frontpage plugin failed to find a 404 page. This is'\
                      ' mostly because of an error in 404 page detection.'
                om.out.error(msg)

    def crawl(self, fuzzable_request):
        """
        Get the sitemap.xml file and parse it.

        :param fuzzable_request: A fuzzable_request instance that contains
                                   (among other things) the URL to test.
        """
        base_url = fuzzable_request.get_url().base_url()
        sitemap_url = base_url.url_join('sitemap.xml')
        response = self._uri_opener.GET(sitemap_url, cache=True)

        if '</urlset>' in response and not is_404(response):
            # Send response to core
            fr = FuzzableRequest.from_http_response(response)
            self.output_queue.put(fr)

            om.out.debug('Parsing xml file with xml.dom.minidom.')
            try:
                dom = xml.dom.minidom.parseString(response.get_body())
            except:
                raise BaseFrameworkException('Error while parsing sitemap.xml')
            else:
                raw_url_list = dom.getElementsByTagName("loc")
                parsed_url_list = []
                for url in raw_url_list:
                    try:
                        url = url.childNodes[0].data
                        url = URL(url)
                    except ValueError, ve:
                        msg = 'Sitemap file had an invalid URL: "%s"'
                        om.out.debug(msg % ve)
                    except:
                        om.out.debug('Sitemap file had an invalid format')

    def _is_possible_backdoor(self, response):
        """
        Heuristic to infer if the content of <response> has the pattern of a
        backdoor response.

        :param response: HTTPResponse object
        :return: A bool value
        """
        if not is_404(response):
            body_text = response.get_body()
            dom = response.get_dom()
            if dom is not None:
                for ele, attrs in BACKDOOR_COLLECTION.iteritems():
                    for attrname, attr_vals in attrs.iteritems():
                        # Set of lowered attribute values
                        dom_attr_vals = \
                            set(n.get(attrname).lower() for n in
                                (dom.xpath('//%s[@%s]' % (ele, attrname))))
                        # If at least one elem in intersection return True
                        if (dom_attr_vals and set(attr_vals)):
                            return True

            # If no regex matched then try with keywords. At least 2 should be
            # contained in 'body_text' to succeed.
            times = 0
            for back_kw in KNOWN_OFFENSIVE_WORDS:
                if re.search(back_kw, body_text, re.I):
                    times += 1
                    if times == 2:
                        return True
        return False

    def crawl(self, fuzzable_req):
        """
        Searches for links on the html.

        :param fuzzable_req: A fuzzable_req instance that contains
                             (among other things) the URL to test.
        """
        self._handle_first_run()

        #
        # If it is a form, then smart_fill the parameters to send something that
        # makes sense and will allow us to cover more code.
        #
        data_container = fuzzable_req.get_raw_data()
        if isinstance(data_container, Form):

            if fuzzable_req.get_url() in self._already_filled_form:
                return

            self._already_filled_form.add(fuzzable_req.get_url())
            data_container.smart_fill()

        # Send the HTTP request
        resp = self._uri_opener.send_mutant(fuzzable_req)

        # Nothing to do here...
        if resp.get_code() == http_constants.UNAUTHORIZED:
            return

        # And we don't trust what comes from the core, check if 404
        if is_404(resp):
            return

        self._extract_html_forms(resp, fuzzable_req)
        self._extract_links_and_verify(resp, fuzzable_req)

    def _classic_worker(self, gh, search_term):
        """
        Perform the searches and store the results in the kb.
        """
        google_list = self._google_se.get_n_results(search_term, 9)

        for result in google_list:
            # I found a vuln in the site!
            response = self._uri_opener.GET(result.URL, cache=True)
            if not is_404(response):
                desc = 'ghdb plugin found a vulnerability at URL: "%s".' \
                      ' According to GHDB the vulnerability description'\
                      ' is "%s".'
                desc = desc % (response.get_url(), gh.desc)
                
                v = Vuln('Google hack database match', desc,
                         severity.MEDIUM, response.id, self.get_name())
                v.set_url(response.get_url())
                v.set_method('GET')

                kb.kb.append(self, 'vuln', v)
                om.out.vulnerability(v.get_desc(), severity=severity.LOW)

                # Create the fuzzable requests
                for fr in self._create_fuzzable_requests(response):
                    self.output_queue.put(fr)

    def _send_and_check(self, repo_url, repo_get_files, repo, domain_path):
        """
        Check if a repository index exists in the domain_path.

        :return: None, everything is saved to the self.out_queue.
        """
        http_response = self.http_get_and_parse(repo_url)

        if not is_404(http_response):

            filenames = repo_get_files(http_response.get_body())

            parsed_url_set = set()

            for filename in self._clean_filenames(filenames):
                test_url = domain_path.url_join(filename)
                if test_url not in self._analyzed_filenames:
                    parsed_url_set.add(test_url)
                    self._analyzed_filenames.add(filename)

            self.worker_pool.map(self.http_get_and_parse, parsed_url_set)

            if parsed_url_set:
                desc = 'A %s was found at: "%s"; this could indicate that'\
                       ' a %s is accessible. You might be able to download'\
                       ' the Web application source code.'
                desc = desc % (repo, http_response.get_url(), repo)
                
                v = Vuln('Source code repository', desc, severity.MEDIUM,
                         http_response.id, self.get_name())
                v.set_url(http_response.get_url())
                
                kb.kb.append(self, repo, v)
                om.out.vulnerability(v.get_desc(), severity=v.get_severity())

    def discover(self, fuzzable_request):
        """
        Checks if JBoss Interesting Directories exist in the target server.
        Also verifies some vulnerabilities.
        """
        base_url = fuzzable_request.get_url().base_url()

        args_iter = izip(repeat(base_url), self.JBOSS_VULNS)
        otm_send_request = one_to_many(self.send_request)
        response_pool = self.worker_pool.imap_unordered(otm_send_request,
                                                        args_iter)

        for vuln_db_instance, response in response_pool:

            if is_404(response):
                continue

            vuln_url = base_url.url_join(vuln_db_instance['url'])
            name = vuln_db_instance['name']
            desc = vuln_db_instance['desc']

            if vuln_db_instance['type'] == 'info':
                o = Info(name, desc, response.id, self.get_name())
            else:
                o = Vuln(name, desc, severity.LOW, response.id, self.get_name())

            o.set_url(vuln_url)
            kb.kb.append(self, 'find_jboss', o)

            for fr in self._create_fuzzable_requests(response):
                self.output_queue.put(fr)

    def crawl(self, fuzzable_request):
        """
        Finds the version of a WordPress installation.
        :param fuzzable_request: A fuzzable_request instance that contains
        (among other things) the URL to test.
        """
        if not self._exec:
            # This will remove the plugin from the crawl plugins to be run.
            raise RunOnce()

        #
        # Check if the server is running wp
        #
        domain_path = fuzzable_request.get_url().get_domain_path()

        # Main scan URL passed from w3af + unique wp file
        wp_unique_url = domain_path.url_join('wp-login.php')
        response = self._uri_opener.GET(wp_unique_url, cache=True)

        # If wp_unique_url is not 404, wordpress = true
        if not is_404(response):
            # It was possible to analyze wp-login.php, don't run again
            self._exec = False

            # Analyze the identified wordpress installation
            self._fingerprint_wordpress(domain_path, wp_unique_url,
                                        response)

            # Extract the links
            for fr in self._create_fuzzable_requests(response):
                self.output_queue.put(fr)

    def _do_request(self, url, mutant):
        """
        Perform a simple GET to see if the result is an error or not, and then
        run the actual fuzzing.
        """
        response = self._uri_opener.GET(
            mutant, cache=True, headers=self._headers)

        if not (is_404(response) or
        response.get_code() in (403, 401) or
        self._return_without_eval(mutant)):

            # Create the fuzzable request and send it to the core
            fr = FuzzableRequest.from_http_response(response)
            self.output_queue.put(fr)
            
            #
            #   Save it to the kb (if new)!
            #
            if response.get_url() not in self._seen and response.get_url().get_file_name():
                desc = 'A potentially interesting file was found at: "%s".'
                desc = desc % response.get_url()

                i = Info('Potentially interesting file', desc, response.id,
                         self.get_name())
                i.set_url(response.get_url())
                
                kb.kb.append(self, 'files', i)
                om.out.information(i.get_desc())

                # Report only once
                self._seen.add(response.get_url())