Python models.ReportStore类代码示例

def push_student_responses_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
    """
    For a given `course_id`, generate a responses CSV file for students that
    have submitted problem responses, and store using a `ReportStore`. Once
    created, the files can be accessed by instantiating another `ReportStore` (via
    `ReportStore.from_config()`) and calling `link_for()` on it. Writes are
    buffered, so we'll never write part of a CSV file to S3 -- i.e. any files
    that are visible in ReportStore will be complete ones.
    """
    start_time = datetime.now(UTC)

    try:
        course = get_course_by_id(course_id)
    except ValueError as e:
        TASK_LOG.error(e.message)
        return "failed"

    rows = student_response_rows(course)

    # Generate parts of the file name
    timestamp_str = start_time.strftime("%Y-%m-%d-%H%M")
    course_id_prefix = urllib.quote(course_id.to_deprecated_string().replace("/", "_"))

    # Perform the actual upload
    report_store = ReportStore.from_config()
    report_store.store_rows(
        course_id,
        u"{}_responses_report_{}.csv".format(course_id_prefix, timestamp_str),
        rows
    )

    return "succeeded"

    def verify_rows_in_csv(self, expected_rows, file_index=0, verify_order=True, ignore_other_columns=False):
        """
        Verify that the last ReportStore CSV contains the expected content.

        Arguments:
            expected_rows (iterable): An iterable of dictionaries,
                where each dict represents a row of data in the last
                ReportStore CSV.  Each dict maps keys from the CSV
                header to values in that row's corresponding cell.
            file_index (int): Describes which report store file to
                open.  Files are ordered by last modified date, and 0
                corresponds to the most recently modified file.
            verify_order (boolean): When True, we verify that both the
                content and order of `expected_rows` matches the
                actual csv rows.  When False (default), we only verify
                that the content matches.
            ignore_other_columns (boolean): When True, we verify that `expected_rows`
                contain data which is the subset of actual csv rows.
        """
        report_store = ReportStore.from_config(config_name="GRADES_DOWNLOAD")
        report_csv_filename = report_store.links_for(self.course.id)[file_index][0]
        report_path = report_store.path_to(self.course.id, report_csv_filename)
        with report_store.storage.open(report_path) as csv_file:
            # Expand the dict reader generator so we don't lose it's content
            csv_rows = [row for row in unicodecsv.DictReader(csv_file)]

            if ignore_other_columns:
                csv_rows = [
                    {key: row.get(key) for key in expected_rows[index].keys()} for index, row in enumerate(csv_rows)
                ]

            if verify_order:
                self.assertEqual(csv_rows, expected_rows)
            else:
                self.assertItemsEqual(csv_rows, expected_rows)

def upload_csv_to_report_store(rows, csv_name, course_id, timestamp):
    """
    Upload data as a CSV using ReportStore.

    Arguments:
        rows: CSV data in the following format (first column may be a
            header):
            [
                [row1_colum1, row1_colum2, ...],
                ...
            ]
        csv_name: Name of the resulting CSV
        course_id: ID of the course
    """
    report_store = ReportStore.from_config()
    report_store.store_rows(
        course_id,
        u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format(
            course_prefix=course_filename_prefix_generator(course_id),
            csv_name=csv_name,
            timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")
        ),
        rows
    )
    tracker.emit(REPORT_REQUESTED_EVENT_NAME, {"report_type": csv_name, })

    def test_grading_failure(self, error_message, mock_iterate_grades_for, _mock_current_task):
        """
        Test that any grading errors are properly reported in the progress
        dict and uploaded to the report store.
        """
        # mock an error response from `iterate_grades_for`
        student = self.create_student(u'username', u'[email protected]')
        mock_iterate_grades_for.return_value = [
            (student, {}, error_message)
        ]
        result = upload_problem_grade_report(None, None, self.course.id, None,
                                             'graded')
        self.assertDictContainsSubset(
            {'attempted': 1, 'succeeded': 0, 'failed': 1}, result)

        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
        self.assertTrue(any('grade_report_err' in item[0] for item in
                            report_store.links_for(self.course.id)))
        self.verify_rows_in_csv([
            {
                u'Student ID': unicode(student.id),
                u'Email': student.email,
                u'Username': student.username,
                u'error_msg': error_message if error_message else "Unknown error"
            }
        ])

 def create_report_store(self):
     """
     Create and return a DjangoStorageReportStore using the old
     S3ReportStore configuration.
     """
     connection = boto.connect_s3()
     connection.create_bucket(settings.GRADES_DOWNLOAD['BUCKET'])
     return ReportStore.from_config(config_name='GRADES_DOWNLOAD')

 def get_csv_row_with_headers(self):
     """
     Helper function to return list with the column names from the CSV file (the first row)
     """
     report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
     report_csv_filename = report_store.links_for(self.course.id)[0][0]
     with open(report_store.path_to(self.course.id, report_csv_filename)) as csv_file:
         rows = unicodecsv.reader(csv_file, encoding='utf-8')
         return rows.next()

 def download_url_for_last_report(self):
     """ Get the URL for the last report, if any """
     # Unfortunately this is a bit inefficient due to the ReportStore API
     if not self.last_export_result or self.last_export_result['error'] is not None:
         return None
     from instructor_task.models import ReportStore
     report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
     course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
     return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])

 def tearDown(self):
     report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
     try:
         reports_download_path = report_store.storage.path('')
     except NotImplementedError:
         pass  # storage backend does not use the local filesystem
     else:
         if os.path.exists(reports_download_path):
             shutil.rmtree(reports_download_path)

    def test_success(self):
        self.create_student('student', '[email protected]')
        task_input = {'features': []}
        with patch('instructor_task.tasks_helper._get_current_task'):
            result = upload_students_csv(None, None, self.course.id, task_input, 'calculated')
        report_store = ReportStore.from_config()
        links = report_store.links_for(self.course.id)

        self.assertEquals(len(links), 1)
        self.assertDictContainsSubset({'attempted': 1, 'succeeded': 1, 'failed': 0}, result)

 def _verify_cell_data_in_csv(self, username, column_header, expected_cell_content):
     """
     Verify that the last ReportStore CSV contains the expected content.
     """
     report_store = ReportStore.from_config(config_name='FINANCIAL_REPORTS')
     report_csv_filename = report_store.links_for(self.course.id)[0][0]
     with open(report_store.path_to(self.course.id, report_csv_filename)) as csv_file:
         # Expand the dict reader generator so we don't lose it's content
         for row in unicodecsv.DictReader(csv_file):
             if row.get('Username') == username:
                 self.assertEqual(row[column_header], expected_cell_content)

 def _verify_csv_data(self, username, expected_data):
     """
     Verify grade report data.
     """
     with patch('instructor_task.tasks_helper._get_current_task'):
         upload_grades_csv(None, None, self.course.id, None, 'graded')
         report_store = ReportStore.from_config()
         report_csv_filename = report_store.links_for(self.course.id)[0][0]
         with open(report_store.path_to(self.course.id, report_csv_filename)) as csv_file:
             for row in unicodecsv.DictReader(csv_file):
                 if row.get('username') == username:
                     csv_row_data = [row[column] for column in self.columns_to_check]
                     self.assertEqual(csv_row_data, expected_data)

def list_report_downloads(_request, course_id):
    """
    List grade CSV files that are available for download for this course.
    """
    report_store = ReportStore.from_config()

    response_payload = {
        'downloads': [
            dict(name=name, url=url, link='<a href="{}">{}</a>'.format(url, name))
            for name, url in report_store.links_for(course_id)
        ]
    }
    return JsonResponse(response_payload)

 def _verify_cell_data_for_user(self, username, course_id, column_header, expected_cell_content):
     """
     Verify cell data in the grades CSV for a particular user.
     """
     with patch('instructor_task.tasks_helper._get_current_task'):
         result = upload_grades_csv(None, None, course_id, None, 'graded')
         self.assertDictContainsSubset({'attempted': 2, 'succeeded': 2, 'failed': 0}, result)
         report_store = ReportStore.from_config()
         report_csv_filename = report_store.links_for(course_id)[0][0]
         with open(report_store.path_to(course_id, report_csv_filename)) as csv_file:
             for row in unicodecsv.DictReader(csv_file):
                 if row.get('username') == username:
                     self.assertEqual(row[column_header], expected_cell_content)

def export_data(course_id, source_block_id_str):
    """
    Exports all answers to all questions by all students to a CSV file.
    """
    start_timestamp = time.time()
    response = {}

    logger.debug("Beginning data export")
    try:
        course_key = CourseKey.from_string(course_id)
        block = modulestore().get_items(course_key, qualifiers={'name': source_block_id_str}, depth=0)[0]
    except IndexError:
        raise ValueError("Could not find the specified Block ID.")
    course_key_str = unicode(course_key)

    # Define the header row of our CSV:
    rows = []

    header = ["Course ID", "Block ID", "Student ID", "Quiz Title", "Final Result"]
    for order in range(len(block.questions)):
        header.append("Question {}".format(order + 1))
        header.append("Answer")

    rows.append(header)

    results = _extract_data(course_key_str, block)
    rows += results

    # Generate the CSV:
    try:
        from instructor_task.models import ReportStore
        filename = u"diagnostic-data-export-{}.csv".format(
            time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp)))
        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
        report_store.store_rows(course_key, filename, rows)

        generation_time_s = time.time() - start_timestamp
        logger.debug("Done data export - took {} seconds".format(generation_time_s))

        response = {
            "error": None,
            "report_filename": filename,
            "start_timestamp": start_timestamp,
            "generation_time_s": generation_time_s,
            "display_data": [] if len(rows) == 1 else rows
        }

    except Exception:
        pass

    return response

    def test_grading_failure(self, mock_iterate_grades_for, _mock_current_task):
        """
        Test that any grading errors are properly reported in the
        progress dict and uploaded to the report store.
        """
        # mock an error response from `iterate_grades_for`
        mock_iterate_grades_for.return_value = [
            (self.create_student('username', '[email protected]'), {}, 'Cannot grade student')
        ]
        result = upload_grades_csv(None, None, self.course.id, None, 'graded')
        self.assertDictContainsSubset({'attempted': 1, 'succeeded': 0, 'failed': 1}, result)

        report_store = ReportStore.from_config()
        self.assertTrue(any('grade_report_err' in item[0] for item in report_store.links_for(self.course.id)))