Python mozprofile.FirefoxProfile类代码示例

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        profile = FirefoxProfile(locations=locations, proxy=True, preferences=preferences)
        profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                 "marionette.defaultPrefs.port": self.marionette_port,
                                 "dom.disable_open_during_load": False})

        self.runner = FirefoxRunner(profile=profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")

    def start(self):
        profile = FirefoxProfile()

        profile.set_preferences({"dom.disable_open_during_load": False,
                                 "marionette.defaultPrefs.enabled": True})

        self.logger.debug("Creating device runner")
        self.runner = mozrunner.B2GDeviceRunner(profile=profile)
        self.logger.debug("Starting device runner")
        self.runner.start()
        self.logger.debug("Device runner started")

def install_greasemonkey(profile_path):
    profile = FirefoxProfile(str(profile_path))
    gm_amo_url = 'https://services.addons.mozilla.org/fr/firefox/api/1.5/addon/748'
    xpi_url = profile.addon_manager.get_amo_install_path(gm_amo_url)
    print("Installing Greasemonkey from:", xpi_url)
    profile.addon_manager.install_from_path(xpi_url)
    # avoid showing Greasemonkey appearing in anyway in the UI
    gm_prefs = {
        'extensions.greasemonkey.haveInsertedToolbarbutton': True,
        'extensions.greasemonkey.stats.prompted': True
    }
    profile.set_persistent_preferences(gm_prefs)
    import ipdb; ipdb.set_trace()

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames)})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
        self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.enabled": True,
                                      "marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def run(self):
        '''
        Starts Firefox thread with Marionette turned on.
        '''
        
        self.profile = FirefoxProfile()
        self.profile.set_preferences({"marionette.defaultPrefs.enabled" : True,
                                      "marionette.defaultPrefs.port": 2828,
                                      "browser.startup.page": 0,
                                      "browser.startup.homepage": "about:blank",
                                      })
        self.runner = FirefoxRunner(profile = self.profile,
                                    binary = self.binary,
                                    kp_kwargs = {'processOutputLine' : [self.logger]})

        self.runner.start()
        self._firefoxRunningEvent.set()
        self.runner.wait()

class FirefoxBrowser(Browser):
    used_ports = set()
    init_timeout = 60

    def __init__(self, logger, binary, prefs_root, debug_info=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None, e10s=False):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
        self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
#.........这里部分代码省略.........

    def start(self, group_metadata=None, **kwargs):
        if group_metadata is None:
            group_metadata = {}

        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        if self.asan:
            print "Setting up LSAN"
            self.lsan_handler = mozleak.LSANLeaks(self.logger,
                                                  scope=group_metadata.get("scope", "/"),
                                                  allowed=self.lsan_allowed,
                                                  maxNumRecordedFrames=self.lsan_max_stack_depth)

        env = test_environment(xrePath=os.path.dirname(self.binary),
                               debugger=self.debug_info is not None,
                               log=self.logger,
                               lsanPath=self.prefs_root)

        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.headless:
            env["MOZ_HEADLESS"] = "1"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port": self.marionette_port,
            "network.dns.localDomains": ",".join(self.config.domains_set),
            "dom.file.createInChild": True,
            # TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
            "network.proxy.type": 0,
            "places.history.enabled": False,
            "network.preload": True,
        })
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check:
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        args = self.binary_args[:] if self.binary_args else []
        args += [cmd_arg("marionette"), "about:blank"]

        debug_args, cmd = browser_command(self.binary,
                                          args,
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

class FennecBrowser(FirefoxBrowser):
    used_ports = set()
    init_timeout = 300
    shutdown_timeout = 60

    def __init__(self, logger, prefs_root, test_type, package_name=None,
                 device_serial="emulator-5444", **kwargs):
        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
        self._package_name = package_name
        self.device_serial = device_serial
        self.tests_root = kwargs["tests_root"]
        self.install_fonts = kwargs["install_fonts"]

    @property
    def package_name(self):
        """
        Name of app to run on emulator.
        """
        if self._package_name is None:
            self._package_name = "org.mozilla.fennec"
            user = os.getenv("USER")
            if user:
                self._package_name += "_" + user
        return self._package_name

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True,
                                      "network.preload": True})
        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
                                          # Disable high DPI
                                          "layout.css.devPixelsPerPx": "1.0",
                                          # Ensure that the full browser element
                                          # appears in the screenshot
                                          "apz.allow_zooming": False,
                                          "android.widget_paints_background": False,
                                          # Ensure that scrollbars are always painted
                                          "ui.scrollbarFadeBeginDelay": 100000})

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.package_name,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FennecEmulatorRunner(app=self.package_name,
                                           profile=self.profile,
                                           cmdargs=cmd[1:],
                                           env=env,
                                           symbols_path=self.symbols_path,
                                           serial=self.device_serial,
                                           # TODO - choose appropriate log dir
                                           logdir=os.getcwd(),
                                           process_class=ProcessHandler,
                                           process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        write_hosts_file(self.config, self.runner.device.device)

#.........这里部分代码省略.........

class FirefoxBrowser(Browser):
    used_ports = set()

    def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
                 symbols_path=None, stackwalk_binary=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.used_ports.add(self.marionette_port)
        self.runner = None
        self.debug_args = debug_args
        self.interactive = interactive
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        ports = {"http": "8000",
                 "https": "8443",
                 "ws": "8888"}

        self.profile = FirefoxProfile(locations=locations,
                                      proxy=ports,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False})

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")
#.........这里部分代码省略.........

def run():
    parser = argparse.ArgumentParser(description="Run crawler")
    parser.add_argument("-b", "--binary", type=str, help="path to the Firefox binary")
    parser.add_argument("-a", "--abpdir", type=str, help="path to the Adblock Plus repository")
    parser.add_argument(
        "-f",
        "--filters",
        metavar="url",
        type=str,
        nargs="+",
        default=[
            "https://easylist-downloads.adblockplus.org/easylist.txt",
            "https://easylist-downloads.adblockplus.org/exceptionrules.txt",
        ],
        help="filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.",
    )
    parser.add_argument("-t", "--timeout", type=int, default=300, help="Load timeout (seconds)")
    parser.add_argument("-x", "--maxtabs", type=int, default=15, help="Maximal number of tabs to open in parallel")
    parser.add_argument("list", type=str, help="URL list to process")
    parser.add_argument("outdir", type=str, help="directory to write data into")
    parameters = parser.parse_args()

    import buildtools.packagerGecko as packager

    cleanup = []
    try:
        base_dir = os.path.dirname(os.path.abspath(__file__))
        handle, crawlerxpi = tempfile.mkstemp(suffix=".xpi")
        os.close(handle)
        cleanup.append(crawlerxpi)
        packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)

        abpxpi = "https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi"
        if parameters.abpdir:
            handle, abpxpi = tempfile.mkstemp(suffix=".xpi")
            os.close(handle)
            cleanup.append(abpxpi)
            packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)

        profile = FirefoxProfile(
            addons=[crawlerxpi, abpxpi],
            preferences={
                "browser.startup.homepage": "about:blank",
                "browser.tabs.warnOnCloseOtherTabs": False,
                "browser.uitour.enabled": False,
                "prompts.tab_modal.enabled": False,
                "startup.homepage_welcome_url": "about:blank",
                "startup.homepage_welcome_url.additional": "about:blank",
                "xpinstall.signatures.required": False,
            },
        )

        abpsettings = os.path.join(profile.profile, "adblockplus")
        os.makedirs(abpsettings)
        with open(os.path.join(abpsettings, "patterns.ini"), "w") as handle:
            print >> handle, "# Adblock Plus preferences"
            print >> handle, "version=4"
            for url in parameters.filters:
                if "=" in url:
                    path, url = url.split("=", 1)
                    with open(path, "r") as source:
                        data = source.read()
                else:
                    data = urllib.urlopen(url).read()
                print >> handle, "[Subscription]"
                print >> handle, "url=%s" % url
                print >> handle, "[Subscription filters]"
                print >> handle, "\n".join(data.splitlines()[1:])
    finally:
        for path in cleanup:
            os.unlink(path)

    server = None
    try:
        port = random.randrange(2000, 60000)
        print "Communicating with client on port %i" % port

        app = CrawlerApp(parameters)
        server = make_server("localhost", port, app)
        app.server = server
        threading.Thread(target=lambda: server.serve_forever()).start()

        runner = FirefoxRunner(
            profile=profile,
            binary=parameters.binary,
            cmdargs=["--crawler-port", str(port)],
            env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE="1"),
        )
        while app.urls:
            runner.start()
            runner.wait()
    finally:
        if server:
            server.shutdown()
        profile.cleanup()

def run():
  parser = argparse.ArgumentParser(description='Run crawler')
  parser.add_argument(
    '-b', '--binary', type=str,
    help='path to the Firefox binary'
  )
  parser.add_argument(
    '-a', '--abpdir', type=str,
    help='path to the Adblock Plus repository'
  )
  parser.add_argument(
    '-f', '--filters', metavar='url', type=str, nargs='+',
    default=["https://easylist-downloads.adblockplus.org/easylist.txt", "https://easylist-downloads.adblockplus.org/exceptionrules.txt"],
    help='filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
  )
  parser.add_argument(
    '-t', '--timeout', type=int, default=300,
    help='Load timeout (seconds)'
  )
  parser.add_argument(
    '-x', '--maxtabs', type=int, default=15,
    help='Maximal number of tabs to open in parallel'
  )
  parser.add_argument(
    'list', type=str,
    help='URL list to process'
  )
  parser.add_argument(
    'outdir', type=str,
    help='directory to write data into'
  )
  parameters = parser.parse_args()

  import buildtools.packagerGecko as packager
  cleanup = []
  try:
    base_dir = os.path.dirname(__file__)
    handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi')
    os.close(handle)
    cleanup.append(crawlerxpi)
    packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)

    abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi'
    if parameters.abpdir:
      handle, abpxpi = tempfile.mkstemp(suffix='.xpi')
      os.close(handle)
      cleanup.append(abpxpi)
      packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)

    profile = FirefoxProfile(
      addons=[
        crawlerxpi,
        abpxpi,
      ],
      preferences={
        'browser.uitour.enabled': False,
        'prompts.tab_modal.enabled': False,
      }
    )

    abpsettings = os.path.join(profile.profile, 'adblockplus')
    os.makedirs(abpsettings)
    with open(os.path.join(abpsettings, 'patterns.ini'), 'w') as handle:
      print >>handle, '# Adblock Plus preferences'
      print >>handle, 'version=4'
      for url in parameters.filters:
        if '=' in url:
          path, url = url.split('=', 1)
          with open(path, 'r') as source:
            data = source.read()
        else:
          data = urllib.urlopen(url).read()
        print >>handle, '[Subscription]'
        print >>handle, 'url=%s' % url
        print >>handle, '[Subscription filters]'
        print >>handle, '\n'.join(data.splitlines()[1:])
  finally:
    for path in cleanup:
      os.unlink(path)

  server = None
  try:
    port = random.randrange(2000, 60000)
    print "Communicating with client on port %i" % port

    app = CrawlerApp(parameters)
    server = make_server('localhost', port, app)
    app.server = server
    threading.Thread(target=lambda: server.serve_forever()).start()

    runner = FirefoxRunner(
      profile=profile,
      binary=parameters.binary,
      cmdargs=['--crawler-port', str(port)],
      env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE='1'),
    )
    while app.urls:
      runner.start()
      runner.wait()
  finally:
#.........这里部分代码省略.........

class FirefoxBrowser(Browser):
    used_ports = set()

    def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.used_ports.add(self.marionette_port)
        self.runner = None
        self.debug_args = debug_args
        self.interactive = interactive
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        ports = {"http": "8000",
                 "https": "8443",
                 "ws": "8888"}

        self.profile = FirefoxProfile(locations=locations,
                                      proxy=ports,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()

#.........这里部分代码省略.........

class FirefoxBrowser(Browser):
    init_timeout = 70
    shutdown_timeout = 70

    def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None, e10s=False, lsan_dir=None, stackfix_dir=None,
                 binary_args=None, timeout_multiplier=None, leak_check=False, asan=False,
                 stylo_threads=1, chaos_mode_flags=None, config=None, browser_channel="nightly", headless=None, **kwargs):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.test_type = test_type
        self.extra_prefs = extra_prefs
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s
        self.binary_args = binary_args
        self.config = config
        if stackfix_dir:
            self.stack_fixer = get_stack_fixer_function(stackfix_dir,
                                                        self.symbols_path)
        else:
            self.stack_fixer = None

        if timeout_multiplier:
            self.init_timeout = self.init_timeout * timeout_multiplier

        self.asan = asan
        self.lsan_dir = lsan_dir
        self.lsan_allowed = None
        self.lsan_max_stack_depth = None
        self.mozleak_allowed = None
        self.mozleak_thresholds = None
        self.leak_check = leak_check
        self.leak_report_file = None
        self.lsan_handler = None
        self.stylo_threads = stylo_threads
        self.chaos_mode_flags = chaos_mode_flags
        self.browser_channel = browser_channel
        self.headless = headless

    def settings(self, test):
        return {"check_leaks": self.leak_check and not test.leaks,
                "lsan_allowed": test.lsan_allowed,
                "lsan_max_stack_depth": test.lsan_max_stack_depth,
                "mozleak_allowed": self.leak_check and test.mozleak_allowed,
                "mozleak_thresholds": self.leak_check and test.mozleak_threshold}

    def start(self, group_metadata=None, **kwargs):
        if group_metadata is None:
            group_metadata = {}

        self.group_metadata = group_metadata
        self.lsan_allowed = kwargs.get("lsan_allowed")
        self.lsan_max_stack_depth = kwargs.get("lsan_max_stack_depth")
        self.mozleak_allowed = kwargs.get("mozleak_allowed")
        self.mozleak_thresholds = kwargs.get("mozleak_thresholds")

        if self.marionette_port is None:
            self.marionette_port = get_free_port()

        if self.asan:
            self.lsan_handler = mozleak.LSANLeaks(self.logger,
                                                  scope=group_metadata.get("scope", "/"),
                                                  allowed=self.lsan_allowed,
                                                  maxNumRecordedFrames=self.lsan_max_stack_depth)

        env = test_environment(xrePath=os.path.dirname(self.binary),
                               debugger=self.debug_info is not None,
                               log=self.logger,
                               lsanPath=self.lsan_dir)

        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.headless:
            env["MOZ_HEADLESS"] = "1"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port": self.marionette_port,
            "network.dns.localDomains": ",".join(self.config.domains_set),
            "dom.file.createInChild": True,
            # TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
            "network.proxy.type": 0,
            "places.history.enabled": False,
            "network.preload": True,
        })
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

#.........这里部分代码省略.........