Python master.BuildMaster类代码示例

def upgradeDatabase(config, master_cfg):
    if not config['quiet']:
        print("upgrading database (%s)"
              % (stripUrlPassword(master_cfg.db['db_url'])))
        print("Warning: Stopping this process might cause data loss")

    def sighandler(signum, frame):
        msg = " ".join("""
        WARNING: ignoring signal %s.
        This process should not be interrupted to avoid database corruption.
        If you really need to terminate it, use SIGKILL.
        """.split())
        print(msg % signum)

    prev_handlers = {}
    try:
        for signame in ("SIGTERM", "SIGINT", "SIGQUIT", "SIGHUP",
                        "SIGUSR1", "SIGUSR2", "SIGBREAK"):
            if hasattr(signal, signame):
                signum = getattr(signal, signame)
                prev_handlers[signum] = signal.signal(signum, sighandler)

        master = BuildMaster(config['basedir'])
        master.config = master_cfg
        master.db.disownServiceParent()
        db = connector.DBConnector(basedir=config['basedir'])
        db.setServiceParent(master)
        yield db.setup(check_version=False, verbose=not config['quiet'])
        yield db.model.upgrade()
        yield db.masters.setAllMastersActiveLongTimeAgo()

    finally:
        # restore previous signal handlers
        for signum, handler in prev_handlers.items():
            signal.signal(signum, handler)

def getMaster(case, reactor, config_dict):
    """
    Create a started ``BuildMaster`` with the given configuration.
    """
    basedir = FilePath(case.mktemp())
    basedir.createDirectory()
    config_dict['buildbotNetUsageData'] = None
    master = BuildMaster(
        basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))

    if 'db_url' not in config_dict:
        config_dict['db_url'] = 'sqlite://'

    # TODO: Allow BuildMaster to transparently upgrade the database, at least
    # for tests.
    master.config.db['db_url'] = config_dict['db_url']
    yield master.db.setup(check_version=False)
    yield master.db.model.upgrade()
    master.db.setup = lambda: None

    yield master.startService()
    # and shutdown the db threadpool, as is normally done at reactor stop
    case.addCleanup(master.db.pool.shutdown)
    case.addCleanup(master.stopService)

    defer.returnValue(master)

    def do_test_master(self):
        # create the master and set its config
        m = BuildMaster(self.basedir, self.configfile)

        # update the DB
        yield m.db.setup(check_version=False)
        yield m.db.model.upgrade()

        # stub out m.db.setup since it was already called above
        m.db.setup = lambda: None

        # mock reactor.stop (which trial *really* doesn't
        # like test code to call!)
        mock_reactor = mock.Mock(spec=reactor)
        mock_reactor.callWhenRunning = reactor.callWhenRunning

        # start the service
        yield m.startService(_reactor=mock_reactor)
        self.failIf(mock_reactor.stop.called,
                    "startService tried to stop the reactor; check logs")

        # hang out for a fraction of a second, to let startup processes run
        d = defer.Deferred()
        reactor.callLater(0.01, d.callback, None)
        yield d

        # stop the service
        yield m.stopService()

        # and shutdown the db threadpool, as is normally done at reactor stop
        m.db.pool.shutdown()

def doCleanupDatabase(config, master_cfg):
    if not config["quiet"]:
        print("cleaning database (%s)" % (master_cfg.db["db_url"]))

    master = BuildMaster(config["basedir"])
    master.config = master_cfg
    db = master.db
    yield db.setup(check_version=False, verbose=not config["quiet"])
    res = yield db.logs.getLogs()
    i = 0
    percent = 0
    saved = 0
    for log in res:
        saved += yield db.logs.compressLog(log["id"], force=config["force"])
        i += 1
        if not config["quiet"] and percent != i * 100 / len(res):
            percent = i * 100 / len(res)
            print(" {0}%  {1} saved".format(percent, saved))
            saved = 0
            sys.stdout.flush()

    if master_cfg.db["db_url"].startswith("sqlite"):
        if not config["quiet"]:
            print("executing sqlite vacuum function...")

        # sqlite vacuum function rebuild the whole database to claim
        # free disk space back
        def thd(engine):
            r = engine.execute("vacuum;")
            r.close()

        yield db.pool.do(thd)

    def do_test_master(self):
        # create the master and set its config
        m = BuildMaster(self.basedir, self.configfile)
        m.config = config.MasterConfig.loadConfig(
                                    self.basedir, self.configfile)

        # update the DB
        yield m.db.setup(check_version=False)
        yield m.db.model.upgrade()

        # stub out m.db.setup since it was already called above
        m.db.setup = lambda : None

        # mock reactor.stop (which trial *really* doesn't
        # like test code to call!)
        mock_reactor = mock.Mock(spec=reactor)
        mock_reactor.callWhenRunning = reactor.callWhenRunning

        # start the service
        yield m.startService(_reactor=mock_reactor)
        self.failIf(mock_reactor.stop.called,
            "startService tried to stop the reactor; check logs")

        # stop the service
        yield m.stopService()

        # and shutdown the db threadpool, as is normally done at reactor stop
        m.db.pool.shutdown()

    def setupConfig(self, configFunc):
        """
        Setup and start a master configured
        by the function configFunc defined in the test module.
        @type configFunc: string
        @param configFunc: name of a function
        without argument defined in the test module
        that returns a BuildmasterConfig object.
        """
        self.basedir = os.path.abspath('basdir')
        self.setUpDirs(self.basedir)
        self.configfile = os.path.join(self.basedir, 'master.cfg')
        if self.proto == 'pb':
            proto = '{"pb": {"port": "tcp:0:interface=127.0.0.1"}}'
        elif self.proto == 'null':
            proto = '{"null": {}}'
        # We create a master.cfg, which loads the configuration from the
        # test module. Only the slave config is kept there, as it should not
        # be changed
        open(self.configfile, "w").write(textwrap.dedent("""
            from buildbot.buildslave import BuildSlave
            from %s import %s
            c = BuildmasterConfig = %s()
            c['slaves'] = [BuildSlave("local1", "localpw")]
            c['protocols'] = %s
            """ % (self.__class__.__module__,
                   configFunc, configFunc,
                   proto)))
        # create the master and set its config
        m = BuildMaster(self.basedir, self.configfile)
        self.master = m

        # update the DB
        yield m.db.setup(check_version=False)
        yield m.db.model.upgrade()

        # stub out m.db.setup since it was already called above
        m.db.setup = lambda: None

        # mock reactor.stop (which trial *really* doesn't
        # like test code to call!)
        mock_reactor = mock.Mock(spec=reactor)
        mock_reactor.callWhenRunning = reactor.callWhenRunning

        # start the service
        yield m.startService(_reactor=mock_reactor)
        self.failIf(mock_reactor.stop.called,
                    "startService tried to stop the reactor; check logs")

        if self.proto == 'pb':
            # We find out the slave port automatically
            slavePort = list(itervalues(m.pbmanager.dispatchers))[0].port.getHost().port

            # create a slave, and attach it to the master, it will be started, and stopped
            # along with the master
            s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
        elif self.proto == 'null':
            s = LocalBuildSlave("local1", self.basedir, False)
        s.setServiceParent(m)

def upgradeDatabase(config, master_cfg):
    if not config['quiet']:
        print "upgrading database (%s)" % (master_cfg.db['db_url'])

    master = BuildMaster(config['basedir'])
    master.config = master_cfg
    db = connector.DBConnector(master, basedir=config['basedir'])

    yield db.setup(check_version=False, verbose=not config['quiet'])
    yield db.model.upgrade()

 def getMaster(self, config_dict):
     """
     Create a started ``BuildMaster`` with the given configuration.
     """
     basedir = FilePath(self.mktemp())
     basedir.createDirectory()
     master = BuildMaster(
         basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))
     master.config = master.config_loader.loadConfig()
     return master

    def testSteps(self):
        m = BuildMaster(".")
        m.loadConfig(cfg1)
        b = m.botmaster.builders["builder1"]
        steps = b.buildFactory.steps
        self.failUnlessEqual(len(steps), 4)

        self.failUnlessExpectedShell(steps[0], command="echo yes")
        self.failUnlessExpectedShell(steps[1], defaults=False, command="old-style")
        self.failUnlessExpectedDarcs(steps[2], repourl="http://buildbot.net/repos/trunk")
        self.failUnlessExpectedShell(steps[3], defaults=False, command="echo old-style")

def upgradeDatabase(config, master_cfg):
    if not config['quiet']:
        print("upgrading database (%s)" % (master_cfg.db['db_url']))

    master = BuildMaster(config['basedir'])
    master.config = master_cfg
    master.db.disownServiceParent()
    db = connector.DBConnector(basedir=config['basedir'])
    db.setServiceParent(master)
    yield db.setup(check_version=False, verbose=not config['quiet'])
    yield db.model.upgrade()
    yield db.masters.setAllMastersActiveLongTimeAgo()

    def check_master_cfg(self):
        from buildbot.master import BuildMaster
        from twisted.python import log, failure

        master_cfg = os.path.join(self.basedir, "master.cfg")
        if not os.path.exists(master_cfg):
            if not self.quiet:
                print "No master.cfg found"
            return 1

        # side-effects of loading the config file:

        #  for each Builder defined in c['builders'], if the status directory
        #  didn't already exist, it will be created, and the
        #  $BUILDERNAME/builder pickle might be created (with a single
        #  "builder created" event).

        # we put basedir in front of sys.path, because that's how the
        # buildmaster itself will run, and it is quite common to have the
        # buildmaster import helper classes from other .py files in its
        # basedir.

        if sys.path[0] != self.basedir:
            sys.path.insert(0, self.basedir)

        m = BuildMaster(self.basedir)
        # we need to route log.msg to stdout, so any problems can be seen
        # there. But if everything goes well, I'd rather not clutter stdout
        # with log messages. So instead we add a logObserver which gathers
        # messages and only displays them if something goes wrong.
        messages = []
        log.addObserver(messages.append)
        try:
            # this will raise an exception if there's something wrong with
            # the config file. Note that this BuildMaster instance is never
            # started, so it won't actually do anything with the
            # configuration.
            m.loadConfig(open(master_cfg, "r"))
        except:
            f = failure.Failure()
            if not self.quiet:
                print
                for m in messages:
                    print "".join(m['message'])
                print f
                print
                print "An error was detected in the master.cfg file."
                print "Please correct the problem and run 'buildbot upgrade-master' again."
                print
            return 1
        return 0

def createDB(config, _noMonkey=False):
    # apply the db monkeypatches (and others - no harm)
    if not _noMonkey:  # pragma: no cover
        monkeypatches.patch_all()

    # create a master with the default configuration, but with db_url
    # overridden
    master_cfg = config_module.MasterConfig()
    master_cfg.db['db_url'] = config['db']
    master = BuildMaster(config['basedir'])
    master.config = master_cfg
    db = connector.DBConnector(master, config['basedir'])
    yield db.setup(check_version=False, verbose=not config['quiet'])
    if not config['quiet']:
        print "creating database (%s)" % (master_cfg.db['db_url'],)
    yield db.model.upgrade()

    def setUp(self):
        self.basedir = os.path.abspath('basdir')
        self.setUpDirs(self.basedir)
        self.configfile = os.path.join(self.basedir, 'master.cfg')

        # We create a master.cfg, which loads the configuration from the
        # test module. Only the slave config is kept there, as it should not
        # be changed
        open(self.configfile, "w").write(textwrap.dedent("""
            from buildbot.buildslave import BuildSlave
            from %s import masterConfig
            c = BuildmasterConfig = masterConfig()
            c['slaves'] = [BuildSlave("local1", "localpw")]
            c['protocols'] = {"pb": {"port": "tcp:0:interface=127.0.0.1"}}
            """ % self.__class__.__module__))
        # create the master and set its config
        m = BuildMaster(self.basedir, self.configfile)
        self.master = m

        # update the DB
        yield m.db.setup(check_version=False)
        yield m.db.model.upgrade()

        # stub out m.db.setup since it was already called above
        m.db.setup = lambda: None

        # mock reactor.stop (which trial *really* doesn't
        # like test code to call!)
        mock_reactor = mock.Mock(spec=reactor)
        mock_reactor.callWhenRunning = reactor.callWhenRunning

        # start the service
        yield m.startService(_reactor=mock_reactor)
        self.failIf(mock_reactor.stop.called,
                    "startService tried to stop the reactor; check logs")

        # We find out the slave port automatically
        slavePort = m.pbmanager.dispatchers.values()[0].port.getHost().port

        # create a slave, and attach it to the master, it will be started, and stopped
        # along with the master
        s = BuildSlave("127.0.0.1", slavePort, "local1", "localpw", self.basedir, False, False)
        s.setServiceParent(m)

def getMaster(case, reactor, config_dict):
    """
    Create a started ``BuildMaster`` with the given configuration.
    """
    basedir = FilePath(case.mktemp())
    basedir.createDirectory()
    master = BuildMaster(basedir.path, reactor=reactor, config_loader=DictLoader(config_dict))

    if 'db_url' not in config_dict:
        config_dict['db_url'] = 'sqlite://'

    # TODO: Allow BuildMaster to transparently upgrade the database, at least for tests.
    master.config.db['db_url'] = config_dict['db_url']
    yield master.db.setup(check_version=False)
    yield master.db.model.upgrade()
    master.db.setup = lambda: None

    yield master.startService()

    defer.returnValue(master)

    def _run_master(self, loaded_config):
        # create the master
        m = BuildMaster(self.basedir, self.configfile)

        # update the DB
        yield m.db.setup(check_version=False)
        yield m.db.model.upgrade()

        # stub out m.db.setup since it was already called above
        m.db.setup = lambda: None

        # mock reactor.stop (which trial *really* doesn't
        # like test code to call!)
        mock_reactor = mock.Mock(spec=reactor)
        mock_reactor.callWhenRunning = reactor.callWhenRunning
        mock_reactor.getThreadPool = reactor.getThreadPool
        mock_reactor.callFromThread = reactor.callFromThread

        # mock configuration loading
        @classmethod
        def loadConfig(cls, basedir, filename):
            return loaded_config

        with mock.patch('buildbot.config.MasterConfig.loadConfig', loadConfig):
            # start the service
            yield m.startService(_reactor=mock_reactor)
        self.failIf(mock_reactor.stop.called,
                    "startService tried to stop the reactor; check logs")

        # hang out for a fraction of a second, to let startup processes run
        d = defer.Deferred()
        reactor.callLater(0.01, d.callback, None)
        yield d

        # stop the service
        yield m.stopService()

        # and shutdown the db threadpool, as is normally done at reactor stop
        m.db.pool.shutdown()