Python App.door43_s3_handler方法代码示例

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def test_obs_deploy_revision_to_door43(self):
     self.mock_s3_obs_project()
     build_log_key = '{0}/build_log.json'.format(self.project_key)
     ret = self.deployer.deploy_revision_to_door43(build_log_key)
     self.assertTrue(ret)
     self.assertTrue(App.door43_s3_handler().key_exists(build_log_key))
     self.assertTrue(App.door43_s3_handler().key_exists('{0}/50.html'.format(self.project_key)))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def multipart_master_merge(self, s3_commit_key, resource_type, download_key, output_dir, source_dir, start,
                            template_file):
     prefix = download_key + '/'
     App.door43_s3_handler().download_dir(prefix, source_dir)  # get previous templated files
     source_dir = os.path.join(source_dir, download_key)
     files = sorted(glob(os.path.join(source_dir, '*.*')))
     for f in files:
         App.logger.debug("Downloaded: " + f)
     fname = os.path.join(source_dir, 'index.html')
     if os.path.isfile(fname):
         os.remove(fname)  # remove index if already exists
     elapsed_seconds = int(time.time() - start)
     App.logger.debug("deploy download completed in " + str(elapsed_seconds) + " seconds")
     templater = init_template(resource_type, source_dir, output_dir, template_file)
     # restore index from previous passes
     index_json = self.get_templater_index(s3_commit_key, 'index.json')
     templater.titles = index_json['titles']
     templater.chapters = index_json['chapters']
     templater.book_codes = index_json['book_codes']
     templater.already_converted = templater.files  # do not reconvert files
     # merge the source files with the template
     try:
         self.run_templater(templater)
         success = True
     except Exception as e:
         App.logger.error("Error multi-part applying template {0} to resource type {1}".format(template_file,
                                                                                               resource_type))
         self.close()
         success = False
     return source_dir, success

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
    def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False):
        converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects')
        test_file_base = test_file_name.split('.zip')[0]
        zip_file = os.path.join(converted_proj_dir, test_file_name)
        out_dir = os.path.join(self.temp_dir, test_file_base)
        unzip(zip_file, out_dir)
        project_dir = os.path.join(out_dir, test_file_base) + os.path.sep
        self.project_files = file_utils.get_files(out_dir)
        self.project_key = project_key
        for filename in self.project_files:
            sub_path = filename.split(project_dir)[1].replace(os.path.sep, '/')  # Make sure it is a bucket path
            App.cdn_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, sub_path))

            if multi_part:  # copy files from cdn to door43
                base_name = os.path.basename(filename)
                if '.html' in base_name:
                    with codecs.open(filename, 'r', 'utf-8-sig') as f:
                        soup = BeautifulSoup(f, 'html.parser')

                    # add nav tag
                    new_tag = soup.new_tag('div', id='right-sidebar')
                    soup.body.append(new_tag)
                    html = unicode(soup)
                    file_utils.write_file(filename, html.encode('ascii', 'xmlcharrefreplace'))

                App.door43_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, base_name))

        # u, user, repo = project_key
        App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
                                          'templates/project-page.html')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def test_tw_deploy_revision_to_door43(self):
     self.mock_s3_tw_project()
     build_log_key = '{0}/build_log.json'.format(self.project_key)
     ret = self.deployer.deploy_revision_to_door43(build_log_key)
     self.assertTrue(ret)
     self.assertTrue(App.door43_s3_handler().key_exists(build_log_key))
     for file_name in ['index.html', 'kt.html', 'names.html', 'other.html', 'build_log.json', 'manifest.yaml']:
         key = '{0}/{1}'.format(self.project_key, file_name)
         self.assertTrue(App.door43_s3_handler().key_exists(key), "Key not found: {0}".format(key))
     parent_key = '/'.join(self.project_key.split('/')[:-1])
     for file_name in ['project.json']:
         key = '{0}/{1}'.format(parent_key, file_name)
         self.assertTrue(App.door43_s3_handler().key_exists(key), "Key not found: {0}".format(key))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def mock_s3_obs_project(self):
     zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip')
     out_dir = os.path.join(self.temp_dir, 'en-obs-complete')
     unzip(zip_file, out_dir)
     project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678')
     self.project_files = [f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f))]
     self.project_key = 'u/door43/en-obs/12345678'
     for filename in self.project_files:
         App.cdn_s3_handler().upload_file(os.path.join(project_dir, filename), '{0}/{1}'.format(self.project_key,
                                                                                                filename))
     App.cdn_s3_handler().upload_file(os.path.join(out_dir, 'door43', 'en-obs', 'project.json'),
                                      'u/door43/en-obs/project.json')
     App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
                                         'templates/project-page.html')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def setUp(self):
     """Runs before each test."""
     App(prefix='{0}-'.format(self._testMethodName))
     App.cdn_s3_handler().create_bucket()
     App.door43_s3_handler().create_bucket()
     self.temp_dir = tempfile.mkdtemp(prefix="test_project_deployer")
     self.deployer = ProjectDeployer()
     TdLanguage.language_list = {
         'aa': TdLanguage({'gw': False, 'ld': 'ltr', 'ang': 'Afar', 'lc': 'aa', 'ln': 'Afaraf', 'lr': 'Africa',
                           'pk': 6}),
         'en': TdLanguage({'gw': True, 'ld': 'ltr', 'ang': 'English', 'lc': 'en', 'ln': 'English',
                           'lr': 'Europe', 'pk': 1747}),
         'es': TdLanguage({'gw': True, 'ld': 'ltr', 'ang': 'Spanish', 'lc': 'es', 'ln': 'espa\xf1ol',
                           'lr': 'Europe', 'pk': 1776}),
         'fr': TdLanguage({'gw': True, 'ld': 'ltr', 'ang': 'French', 'lc': 'fr',
                           'ln': 'fran\xe7ais, langue fran\xe7aise', 'lr': 'Europe', 'pk': 1868})
     }

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
    def test_tn_deploy_revision_to_door43(self):
        # given
        part = '1'
        self.mock_s3_tn_project(part)
        build_log_key = '{0}/{1}/build_log.json'.format(self.project_key, part)

        # when
        ret = self.deployer.deploy_revision_to_door43(build_log_key)

        # then
        self.assertTrue(ret)
        self.assertTrue(App.door43_s3_handler().key_exists('{0}/build_log.json'.format(self.project_key)))
        files_to_verify = ['01-GEN.html', 'index.json']

        for file_name in files_to_verify:
            key = '{0}/{1}'.format(self.project_key, file_name)
            self.assertTrue(App.door43_s3_handler().key_exists(key), "Key not found: {0}".format(key))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
    def validate_conversion(self, user, repo, success, build_log_json, commit_id, commit_sha, commit_path,
                            expected_output_names, job, chapter_count=-1, file_ext=""):
        self.assertTrue(len(build_log_json) > 0)
        self.assertIsNotNone(job)
        self.temp_dir = tempfile.mkdtemp(prefix='testing_')

        if not (type(expected_output_names) is list):
            expected_output_names = [expected_output_names]  # put string in list

        # check pre-convert files
        self.download_and_check_zip_file(self.preconvert_handler, expected_output_names,
                                         self.preprocessor_output_extension, self.get_preconvert_s3_key(commit_sha),
                                         "preconvert", success, chapter_count, file_ext)

        # check converted files
        destination_key = self.get_destination_s3_key(commit_sha, repo, user)
        converted_build_log = self.check_destination_files(self.cdn_handler, expected_output_names, "html",
                                                           destination_key, chapter_count)

        # check required fields
        App.logger.debug(converted_build_log)
        saved_build_json = json.loads(converted_build_log)
        self.assertTrue('commit_id' in saved_build_json)
        self.assertTrue('repo_owner' in saved_build_json)
        self.assertTrue('repo_name' in saved_build_json)
        self.assertTrue('created_at' in saved_build_json)
        self.assertTrue('source' in saved_build_json)
        self.assertTrue('errors' in saved_build_json)
        self.assertTrue('warnings' in saved_build_json)
        self.assertTrue('message' in saved_build_json)
        self.assertTrue('status' in saved_build_json)

        self.assertEqual(len(commit_id), COMMIT_LENGTH)
        self.assertIsNotNone(commit_sha)
        self.assertIsNotNone(commit_path)
        if len(job.errors) > 0:
            self.warn("WARNING: Found job errors: " + str(job.errors))

        if len(build_log_json['errors']) > 0:
            self.warn("WARNING: Found build_log errors: " + str(build_log_json['errors']))

        door43_handler = App.door43_s3_handler()
        deployed_build_log = self.check_deployed_files(door43_handler, expected_output_names, "html",
                                                       destination_key, chapter_count)

        self.compare_build_logs(converted_build_log, deployed_build_log, destination_key)

        if len(self.warnings):
            App.logger.debug("\n#######\nHave warnings:\n#######\n" + '\n'.join(self.warnings))

        self.assertTrue(success)

        # Test that repo is in manifest table
        tx_manifest = TxManifest.get(repo_name=repo, user_name=user)
        # Giving TxManifest above just the composite keys will cause it to load all the data from the App.
        self.assertIsNotNone(tx_manifest)
        self.assertEqual(tx_manifest.repo_name, repo)
        self.assertEqual(tx_manifest.user_name, user)

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def mock_s3_tn_project(self, part):
     zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip')
     out_dir = os.path.join(self.temp_dir, 'en_tn_converted')
     unzip(zip_file, out_dir)
     src_dir = os.path.join(out_dir, 'en_tn_converted')
     self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))]
     self.project_key = 'u/door43/en_tn/12345678'
     build_log = file_utils.load_json_object(os.path.join(src_dir, 'build_log.json'))
     build_log['part'] = part
     file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log)
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
                                      '{0}/{1}/build_log.json'.format(self.project_key, part))
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'index.json'),
                                      '{0}/{1}/index.json'.format(self.project_key, part))
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
                                      '{0}/{1}/finished'.format(self.project_key, part))
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, '01-GEN.html'),
                                      '{0}/{1}/01-GEN.html'.format(self.project_key, part))
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
                                      'u/door43/en_tq/project.json')
     App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
                                         'templates/project-page.html')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
    def test_tq_deploy_revision_to_door43(self):
        # given
        self.mock_s3_tq_project()
        build_log_key = '{0}/build_log.json'.format(self.project_key)

        # when
        ret = self.deployer.deploy_revision_to_door43(build_log_key)

        # then
        self.assertTrue(ret)
        self.assertTrue(App.door43_s3_handler().key_exists(build_log_key))
        files_to_verify = ['manifest.yaml']
        for book in BOOK_NUMBERS:
            html_file = '{0}-{1}.html'.format(BOOK_NUMBERS[book], book.upper())
            files_to_verify.append(html_file)

        for file_name in files_to_verify:
            key = '{0}/{1}'.format(self.project_key, file_name)
            self.assertTrue(App.door43_s3_handler().key_exists(key), "Key not found: {0}".format(key))
        parent_key = '/'.join(self.project_key.split('/')[:-1])
        for file_name in ['project.json']:
            key = '{0}/{1}'.format(parent_key, file_name)
            self.assertTrue(App.door43_s3_handler().key_exists(key), "Key not found: {0}".format(key))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
 def validate_bible_results(self, ret, build_log_key, expect_success, output_key):
     self.assertEqual(ret, expect_success)
     if expect_success:
         if output_key:
             self.assertTrue(App.door43_s3_handler().key_exists(output_key))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import door43_s3_handler [as 别名]
    def deploy_revision_to_door43(self, build_log_key):
        """
        Deploys a single revision of a project to door43.org
        :param string build_log_key:
        :return bool:
        """
        build_log = None
        try:
            build_log = App.cdn_s3_handler().get_json(build_log_key, catch_exception=False)
        except Exception as e:
            App.logger.debug("Deploying error could not access {0}: {1}".format(build_log_key, str(e)))
            pass

        if not build_log or 'commit_id' not in build_log or 'repo_owner' not in build_log \
                or 'repo_name' not in build_log:
            App.logger.debug("Exiting, Invalid build log at {0}: {1}".format(build_log_key, build_log))
            return False

        start = time.time()
        App.logger.debug("Deploying, build log: " + json.dumps(build_log)[:256])

        user = build_log['repo_owner']
        repo_name = build_log['repo_name']
        commit_id = build_log['commit_id'][:10]

        s3_commit_key = 'u/{0}/{1}/{2}'.format(user, repo_name, commit_id)
        s3_repo_key = 'u/{0}/{1}'.format(user, repo_name)
        download_key = s3_commit_key

        do_part_template_only = False
        do_multipart_merge = False
        if 'multiple' in build_log:
            do_multipart_merge = build_log['multiple']
            App.logger.debug("Found multi-part merge: {0}".format(download_key))

            prefix = download_key + '/'
            undeployed = self.get_undeployed_parts(prefix)
            if len(undeployed) > 0:
                App.logger.debug("Exiting, Parts not yet deployed: {0}".format(undeployed))
                return False

            key_deployed_ = download_key + '/final_deployed'
            if App.cdn_s3_handler().key_exists(key_deployed_):
                App.logger.debug("Exiting, Already merged parts: {0}".format(download_key))
                return False
            self.write_data_to_file(self.temp_dir, key_deployed_, 'final_deployed', ' ')  # flag that deploy has begun
            App.logger.debug("Continuing with merge: {0}".format(download_key))

        elif 'part' in build_log:
            part = build_log['part']
            download_key += '/' + part
            do_part_template_only = True
            App.logger.debug("Found partial: {0}".format(download_key))

            if not App.cdn_s3_handler().key_exists(download_key + '/finished'):
                App.logger.debug("Exiting, Not ready to process partial")
                return False

        source_dir = tempfile.mkdtemp(prefix='source_', dir=self.temp_dir)
        output_dir = tempfile.mkdtemp(prefix='output_', dir=self.temp_dir)
        template_dir = tempfile.mkdtemp(prefix='template_', dir=self.temp_dir)

        resource_type = build_log['resource_type']
        template_key = 'templates/project-page.html'
        template_file = os.path.join(template_dir, 'project-page.html')
        App.logger.debug("Downloading {0} to {1}...".format(template_key, template_file))
        App.door43_s3_handler().download_file(template_key, template_file)

        if not do_multipart_merge:
            source_dir, success = self.template_converted_files(build_log, download_key, output_dir, repo_name,
                                                                resource_type, s3_commit_key, source_dir, start,
                                                                template_file)
            if not success:
                return False
        else:
            source_dir, success = self.multipart_master_merge(s3_commit_key, resource_type, download_key, output_dir,
                                                              source_dir, start, template_file)
            if not success:
                return False

        #######################
        #
        #  Now do the deploy
        #
        #######################

        if not do_part_template_only or do_multipart_merge:
            # Copy first HTML file to index.html if index.html doesn't exist
            html_files = sorted(glob(os.path.join(output_dir, '*.html')))
            index_file = os.path.join(output_dir, 'index.html')
            if len(html_files) > 0 and not os.path.isfile(index_file):
                copyfile(os.path.join(output_dir, html_files[0]), index_file)

        # Copy all other files over that don't already exist in output_dir, like css files
        for filename in sorted(glob(os.path.join(source_dir, '*'))):
            output_file = os.path.join(output_dir, os.path.basename(filename))
            if not os.path.exists(output_file) and not os.path.isdir(filename):
                copyfile(filename, output_file)

            if do_part_template_only:  # move files to common area
#.........这里部分代码省略.........