Python app.CronTabber类代码示例

    def test_run_drop_old_partitions(self):
        cur = self.conn.cursor()

        # Ensure test table is present.
        statement = """
        SELECT COUNT(*) FROM information_schema.tables
        WHERE table_name = 'phrotest_20120102';
        """
        cur.execute(statement)
        result = cur.fetchone()
        eq_(result[0], 1)

        # Run the crontabber job to remove the test table.
        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

        # Basic assertion test of stored procedure.
        information = self._load_structure()
        assert information['drop-old-partitions']
        assert not information['drop-old-partitions']['last_error']
        assert information['drop-old-partitions']['last_success']

        # Ensure test table was removed.
        statement = """
        SELECT COUNT(*) FROM information_schema.tables
        WHERE table_name = 'phrotest_20120102';
        """
        cur.execute(statement)
        result = cur.fetchone()
        eq_(result[0], 0)

    def test_run_job_with_mocked_data_with_wrong_products(self):
        config_manager = self._setup_config_manager(
            product='Thunderbird,SeaMonkey',
            version='1.0,2.0',
            public_output_path=False
        )
        self._insert_waterwolf_mock_data()

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['daily-url']
            assert not information['daily-url']['last_error']
            assert information['daily-url']['last_success']

            # this should have created two .csv.gz files
            now = datetime.datetime.utcnow() - datetime.timedelta(days=1)

            private = now.strftime('%Y%m%d-crashdata.csv.gz')
            public = now.strftime('%Y%m%d-pub-crashdata.csv.gz')
            ok_(private in os.listdir(self.tempdir))
            ok_(public not in os.listdir(self.tempdir))

            private_path = os.path.join(self.tempdir, private)
            f = gzip.open(private_path)
            try:
                eq_(f.read(), '')
            finally:
                f.close()

    def test_run(self):
        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['suspicious-crashes']
            assert not information['suspicious-crashes']['last_error']
            assert information['suspicious-crashes']['last_success']

            cursor = self.conn.cursor()

            cursor.execute("""
                SELECT signatures.signature, scs.report_date
                FROM suspicious_crash_signatures scs
                JOIN signatures ON scs.signature_id=signatures.signature_id
            """)

            count = 0
            today = (utc_now() - datetime.timedelta(1)).date()
            for row in cursor.fetchall():
                eq_('sig', row[0])
                eq_(today, row[1].date())
                count += 1

            eq_(1, count)

    def test_other_indices_are_not_deleted(self):
        """Verify that non-week-based indices are not removed. For example,
        the socorro_email index should not be deleted by the cron job.
        """
        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            # clear the indices cache so the index is created on every test
            self.storage.indices_cache = set()

            es = self.storage.es

            # Create the socorro emails index.
            self.storage.create_emails_index()

            # This will raise an error if the index was not correctly created.
            es.status('socorro_emails')

            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['elasticsearch-cleanup']
            assert not information['elasticsearch-cleanup']['last_error']
            assert information['elasticsearch-cleanup']['last_success']

            # Verify the email index is still there. This will raise an error
            # if the index does not exist.
            es.status('socorro_emails')

    def test_basic_run_job_without_reports(self):
        config_manager = self._setup_config_manager(3)

        cursor = self.conn.cursor()
        cursor.execute('select count(*) from reports')
        count, = cursor.fetchone()
        assert count == 0, "reports table not cleaned"
        cursor.execute('select count(*) from bugs')
        count, = cursor.fetchone()
        assert count == 0, "'bugs' table not cleaned"
        cursor.execute('select count(*) from bug_associations')
        count, = cursor.fetchone()
        assert count == 0, "'bug_associations' table not cleaned"

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['bugzilla-associations']
            assert not information['bugzilla-associations']['last_error']
            assert information['bugzilla-associations']['last_success']

        # now, because there we no matching signatures in the reports table
        # it means that all bugs are rejected
        cursor.execute('select count(*) from bugs')
        count, = cursor.fetchone()
        ok_(not count)
        cursor.execute('select count(*) from bug_associations')
        count, = cursor.fetchone()
        ok_(not count)

    def test_run_job_with_no_data_with_ssh_errors(self):
        config_manager = self._setup_config_manager(
          public_output_path='',
          private_user='peter',
          private_server='secure.mozilla.org',
          private_location='/var/data/',
          private_ssh_command='chmod 0640 /var/data/*',
        )
        self._insert_waterwolf_mock_data()

        # any mutable so we can keep track of the number of times
        # the side_effect function is called
        calls = []

        def comm():
            if calls:
                # some errors
                return '', "CRAP!"
            else:
                calls.append(1)
                return '', ''

        self.Popen().communicate.side_effect = comm

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['daily-url']
            assert not information['daily-url']['last_error']
            assert information['daily-url']['last_success']

            ok_(config.logger.warn.called)

    def test_basic_run(self):
        cur = self.conn.cursor()
        # Ensure test table is present.
        statement = """
            INSERT INTO missing_symbols
            (date_processed, debug_file, debug_id, code_file, code_id)
            VALUES
            (%(first)s, 'foo.pdb', '0420', 'foo.py', '123'),
            (%(second)s, 'bar.pdb', '65EA9', 'bar.py', null)
        """
        second = utc_now().date()
        first = second - datetime.timedelta(days=1)
        cur.execute(statement, {'first': first, 'second': second})
        self.conn.commit()

        # Run the crontabber job to remove the test table.
        config_manager = self._setup_config_manager(days_to_keep=1)
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

        # Basic assertion test of stored procedure.
        information = self._load_structure()
        assert information['clean-missing-symbols']
        assert not information['clean-missing-symbols']['last_error']
        assert information['clean-missing-symbols']['last_success']

        # Ensure expected test row was removed
        cur.execute("""
            SELECT date_processed FROM missing_symbols
        """)
        first, = cur.fetchall()
        date_processed = first[0]
        eq_(date_processed, second)

    def test_run_job_with_reports_with_existing_bugs_different(self, requests_mocker):
        """Verify that an association to a signature that no longer is part
        of the crash signatures list gets removed.
        """
        requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
        config_manager = self._setup_config_manager(3)
        cursor = self.conn.cursor()

        cursor.execute("""
            insert into bug_associations (bug_id, signature)
            values (8, '@different');
        """)
        self.conn.commit()

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['bugzilla-associations']
            assert not information['bugzilla-associations']['last_error']
            assert information['bugzilla-associations']['last_success']

        cursor.execute(
            'select signature from bug_associations where bug_id = 8'
        )
        associations = cursor.fetchall()
        # The previous association, to signature '@different' that is not in
        # crash signatures, is now missing.
        assert ('@different',) not in associations

    def test_reprocessing_exception(self):
        config_manager = self._setup_config_manager()

        cursor = self.conn.cursor()

        # Test exception handling
        cursor.execute("""
            alter table reprocessing_jobs RENAME TO test_reprocessing_jobs
        """)
        # Need to commit this in order to test the exception handling
        # because the crontabber invocation happens in a different Pg
        # transaction.
        self.conn.commit()

        try:
            with config_manager.context() as config:
                tab = CronTabber(config)
                tab.run_all()

            state = tab.job_state_database['reprocessing-jobs']
            res_expected = "<class 'psycopg2.ProgrammingError'>"
            res = state['last_error']['type']
            eq_(res, res_expected)

        finally:
            # Change table name back
            cursor.execute("""
                alter table test_reprocessing_jobs RENAME TO reprocessing_jobs
            """)
            self.conn.commit()

    def test_basic_run_job(self, requests_mocker):
        requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
        config_manager = self._setup_config_manager(3)

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['bugzilla-associations']
            assert not information['bugzilla-associations']['last_error']
            assert information['bugzilla-associations']['last_success']

        cursor = self.conn.cursor()
        cursor.execute('select bug_id from bug_associations order by bug_id')
        associations = cursor.fetchall()

        # Verify we have the expected number of associations.
        assert len(associations) == 8
        bug_ids = [x[0] for x in associations]

        # Verify bugs with no crash signatures are missing.
        assert 6 not in bug_ids

        cursor.execute(
            'select signature from bug_associations where bug_id = 8'
        )
        associations = cursor.fetchall()
        # New signatures have correctly been inserted.
        assert len(associations) == 2
        assert ('another::legitimate(sig)',) in associations
        assert ('legitimate(sig)',) in associations

    def test_run_job_with_reports_with_existing_bugs_same(self, requests_mocker):
        requests_mocker.get(BUGZILLA_BASE_URL, json=SAMPLE_BUGZILLA_RESULTS)
        config_manager = self._setup_config_manager(3)
        cursor = self.conn.cursor()

        cursor.execute("""
            insert into bug_associations (bug_id, signature)
            values (8, 'legitimate(sig)');
        """)
        self.conn.commit()

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['bugzilla-associations']
            assert not information['bugzilla-associations']['last_error']
            assert information['bugzilla-associations']['last_success']

        cursor.execute(
            'select signature from bug_associations where bug_id = 8'
        )
        associations = cursor.fetchall()
        # New signatures have correctly been inserted.
        assert len(associations) == 2
        assert ('another::legitimate(sig)',) in associations
        assert ('legitimate(sig)',) in associations

    def test_reprocessing(self):
        """ Simple test of reprocessing"""
        config_manager = self._setup_config_manager()
        c = config_manager.get_config()

        cursor = self.conn.cursor()

        # Create partitions to support the status query
        # Load report_partition_info data
        cursor.execute("""
            INSERT into reprocessing_jobs
              (crash_id)
            VALUES
             ('13c4a348-5d04-11e3-8118-d231feb1dc81')
        """)

        # We have to do this here to accommodate separate crontabber processes
        self.conn.commit()

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = tab.job_state_database['reprocessing-jobs']
            assert not information['last_error']
            assert information['last_success']

        cursor = self.conn.cursor()
        cursor.execute('select count(*) from reprocessing_jobs')

        res_expected = 0
        res, = cursor.fetchone()
        eq_(res, res_expected)

    def test_symbols_unpack_zip_file(self):

        source_file = os.path.join(
            self.temp_source_directory,
            os.path.basename(ZIP_FILE)
        )
        shutil.copy(ZIP_FILE, source_file)

        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

        information = self._load_structure()
        assert information['symbols-unpack']
        assert not information['symbols-unpack']['last_error']
        assert information['symbols-unpack']['last_success']

        ok_(os.listdir(self.temp_destination_directory), 'empty')
        # there should now be a directory named `sample-<todays date>
        destination_dir = self.temp_destination_directory
        ok_(os.path.isdir(destination_dir))
        # and it should contain files
        ok_(os.listdir(destination_dir))
        # and the original should now have been deleted
        ok_(not os.path.isfile(source_file))

    def test_failing_pig_job(self):
        # a mutable where commands sent are stored
        commands_sent = []
        self.Popen.side_effect = functools.partial(
            mocked_Popen,
            _commands_sent=commands_sent,
            _exit_code=1,
            _stdout='',
            _stderr='First command failed :(',
        )

        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['modulelist']
            assert information['modulelist']['last_error']
            _traceback = information['modulelist']['last_error']['traceback']
            ok_('pig run failed' in _traceback)
            # the other two where cancelled
            eq_(len(commands_sent), 1)
            config.logger.error.has_calls([
                mock.call('First command failed :(')
            ])

    def test_run_drop_old_partitions(self):
        cur = self.conn.cursor()

        # Ensure test table is present.
        statement = """
        SELECT COUNT(*) FROM raw_crashes_20120102;
        """
        cur.execute(statement)
        result = cur.fetchone()
        eq_(result[0], 2L)
        # need to get out of this transaction to
        # allow crontabber to acquire a lock
        self.conn.commit()

        # Run the crontabber job to remove the test table.
        config_manager = self._setup_config_manager()
        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

        # Basic assertion test of stored procedure.
        information = self._load_structure()
        print information['truncate-partitions']['last_error']
        assert information['truncate-partitions']
        assert not information['truncate-partitions']['last_error']
        assert information['truncate-partitions']['last_success']

        # Ensure test table was removed.
        statement = """
        SELECT COUNT(*) FROM raw_crashes_20120102;
        """
        cur.execute(statement)
        result = cur.fetchone()
        eq_(result[0], 0)