Python App.cdn_s3_handler方法代码示例

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
    def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False):
        converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects')
        test_file_base = test_file_name.split('.zip')[0]
        zip_file = os.path.join(converted_proj_dir, test_file_name)
        out_dir = os.path.join(self.temp_dir, test_file_base)
        unzip(zip_file, out_dir)
        project_dir = os.path.join(out_dir, test_file_base) + os.path.sep
        self.project_files = file_utils.get_files(out_dir)
        self.project_key = project_key
        for filename in self.project_files:
            sub_path = filename.split(project_dir)[1].replace(os.path.sep, '/')  # Make sure it is a bucket path
            App.cdn_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, sub_path))

            if multi_part:  # copy files from cdn to door43
                base_name = os.path.basename(filename)
                if '.html' in base_name:
                    with codecs.open(filename, 'r', 'utf-8-sig') as f:
                        soup = BeautifulSoup(f, 'html.parser')

                    # add nav tag
                    new_tag = soup.new_tag('div', id='right-sidebar')
                    soup.body.append(new_tag)
                    html = unicode(soup)
                    file_utils.write_file(filename, html.encode('ascii', 'xmlcharrefreplace'))

                App.door43_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, base_name))

        # u, user, repo = project_key
        App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
                                          'templates/project-page.html')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def update_project_file(build_log, output_dir):
     commit_id = build_log['commit_id']
     user_name = build_log['repo_owner']
     repo_name = build_log['repo_name']
     project_json_key = 'u/{0}/{1}/project.json'.format(user_name, repo_name)
     project_json = App.cdn_s3_handler().get_json(project_json_key)
     project_json['user'] = user_name
     project_json['repo'] = repo_name
     project_json['repo_url'] = 'https://{0}/{1}/{2}'.format(App.gogs_url, user_name, repo_name)
     commit = {
         'id': commit_id,
         'created_at': build_log['created_at'],
         'status': build_log['status'],
         'success': build_log['success'],
         'started_at': None,
         'ended_at': None
     }
     if 'started_at' in build_log:
         commit['started_at'] = build_log['started_at']
     if 'ended_at' in build_log:
         commit['ended_at'] = build_log['ended_at']
     if 'commits' not in project_json:
         project_json['commits'] = []
     commits = []
     for c in project_json['commits']:
         if c['id'] != commit_id:
             commits.append(c)
     commits.append(commit)
     project_json['commits'] = commits
     project_file = os.path.join(output_dir, 'project.json')
     write_file(project_file, project_json)
     App.cdn_s3_handler().upload_file(project_file, project_json_key, cache_time=0)
     return project_json

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def upload_converted_files(s3_commit_key, unzip_dir):
     for root, dirs, files in os.walk(unzip_dir):
         for f in sorted(files):
             path = os.path.join(root, f)
             key = s3_commit_key + path.replace(unzip_dir, '')
             App.logger.debug('Uploading {0} to {1}'.format(f, key))
             App.cdn_s3_handler().upload_file(path, key, cache_time=0)

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
    def setUp(self):
        """Runs before each test."""
        App(prefix='{0}-'.format(self._testMethodName), db_connection_string='sqlite:///:memory:')
        App.cdn_s3_handler().create_bucket()
        App.pre_convert_s3_handler().create_bucket()
        App.cdn_s3_handler().upload_file = self.mock_cdn_upload_file
        App.cdn_s3_handler().get_json = self.mock_cdn_get_json
        App.pre_convert_s3_handler().upload_file = self.mock_s3_upload_file

        try:
            os.makedirs(ClientWebhookTest.base_temp_dir)
        except:
            pass

        self.temp_dir = tempfile.mkdtemp(dir=self.base_temp_dir, prefix='webhookTest_')
        self.job_converter_count = 0
        self.job_linter_count = 0
        self.uploaded_files = []

        self.job_data = {
            'job_id': '123456890',
            'status': 'started',
            'success': False,
            'resource_type': 'obs',
            'input_format': 'md',
            'output_format': 'html',
            'convert_module': 'module1',
            'created_at': datetime.utcnow(),
            'errors': []
        }
        self.register_modules()

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def setUp(self):
     """Runs before each test."""
     App(prefix='{0}-'.format(self._testMethodName), db_connection_string='sqlite:///:memory:')
     App.cdn_s3_handler().create_bucket()
     self.temp_dir = tempfile.mkdtemp(prefix="test_project_printer")
     self.printer = ProjectPrinter()
     self.mock_s3_obs_project()

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def update_project_json(self, commit_id, job, repo_name, repo_owner):
     """
     :param string commit_id:
     :param TxJob job:
     :param string repo_name:
     :param string repo_owner:
     :return:
     """
     project_json_key = 'u/{0}/{1}/project.json'.format(repo_owner, repo_name)
     project_json = App.cdn_s3_handler().get_json(project_json_key)
     project_json['user'] = repo_owner
     project_json['repo'] = repo_name
     project_json['repo_url'] = 'https://git.door43.org/{0}/{1}'.format(repo_owner, repo_name)
     commit = {
         'id': commit_id,
         'created_at': job.created_at,
         'status': job.status,
         'success': job.success,
         'started_at': None,
         'ended_at': None
     }
     if 'commits' not in project_json:
         project_json['commits'] = []
     commits = []
     for c in project_json['commits']:
         if c['id'] != commit_id:
             commits.append(c)
     commits.append(commit)
     project_json['commits'] = commits
     project_file = os.path.join(self.base_temp_dir, 'project.json')
     write_file(project_file, project_json)
     App.cdn_s3_handler().upload_file(project_file, project_json_key)

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
    def setUp(self):
        """Runs before each test."""
        App(prefix='{0}-'.format(self._testMethodName), db_connection_string='sqlite:///:memory:')
        App.cdn_s3_handler().create_bucket()
        App.cdn_s3_handler().upload_file = self.mock_cdn_upload_file
        App.cdn_s3_handler().get_json = self.mock_cdn_get_json
        App.cdn_s3_handler().key_exists = self.mock_cdn_key_exists

        try:
            os.makedirs(self.base_temp_dir)
        except:
            pass

        self.temp_dir = tempfile.mkdtemp(dir=self.base_temp_dir, prefix='callbackTest_')
        self.transferred_files = []
        self.raiseDownloadException = False
        self.source_folder = None

        self.results_key = 'u/results'
        self.lint_callback_data = {
            'identifier': 'dummy_id',
            's3_results_key': self.results_key,
            'success': True,
            'info': [],
            'warnings': [],
            'errors': []
        }
        self.expected_error_count = 0
        self.expected_warning_count = 0
        self.expected_log_count = 0
        self.expected_status = "success"
        self.expected_success = True
        self.expected_all_parts_completed = True
        self.expected_multipart = False

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def set_deployed_flags(self, project_key, part_count, skip=-1):
     tempf = tempfile.mktemp(prefix="temp", suffix="deployed")
     file_utils.write_file(tempf, ' ')
     for i in range(0, part_count):
         if i != skip:
             key = '{0}/{1}/deployed'.format(project_key, i)
             App.cdn_s3_handler().upload_file(tempf, key, cache_time=0)
     os.remove(tempf)

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
    def template_converted_files(self, build_log, download_key, output_dir, repo_name, resource_type, s3_commit_key,
                                 source_dir, start, template_file):
        App.cdn_s3_handler().download_dir(download_key + '/', source_dir)
        source_dir = os.path.join(source_dir, download_key.replace('/', os.path.sep))
        elapsed_seconds = int(time.time() - start)
        App.logger.debug("deploy download completed in " + str(elapsed_seconds) + " seconds")
        html_files = sorted(glob(os.path.join(source_dir, '*.html')))
        if len(html_files) < 1:
            content = ''
            if len(build_log['errors']) > 0:
                content += """
                        <div style="text-align:center;margin-bottom:20px">
                            <i class="fa fa-times-circle-o" style="font-size: 250px;font-weight: 300;color: red"></i>
                            <br/>
                            <h2>Critical!</h2>
                            <h3>Here is what went wrong with this build:</h3>
                        </div>
                    """
                content += '<div><ul><li>' + '</li><li>'.join(build_log['errors']) + '</li></ul></div>'
            else:
                content += '<h1 class="conversion-requested">{0}</h1>'.format(build_log['message'])
                content += '<p><i>No content is available to show for {0} yet.</i></p>'.format(repo_name)
            html = """
                    <html lang="en">
                        <head>
                            <title>{0}</title>
                        </head>
                        <body>
                            <div id="content">{1}</div>
                        </body>
                    </html>""".format(repo_name, content)
            repo_index_file = os.path.join(source_dir, 'index.html')
            write_file(repo_index_file, html)

        # merge the source files with the template
        templater = init_template(resource_type, source_dir, output_dir, template_file)
        try:
            self.run_templater(templater)
            success = True
        except Exception as e:
            App.logger.error("Error applying template {0} to resource type {1}:".format(template_file, resource_type))
            App.logger.error(e.message)
            App.logger.error('{0}: {1}'.format(str(e), traceback.format_exc()))
            self.close()
            success = False

        if success:
            # update index of templated files
            index_json_fname = 'index.json'
            index_json = self.get_templater_index(s3_commit_key, index_json_fname)
            App.logger.debug("initial 'index.json': " + json.dumps(index_json)[:256])
            self.update_index_key(index_json, templater, 'titles')
            self.update_index_key(index_json, templater, 'chapters')
            self.update_index_key(index_json, templater, 'book_codes')
            App.logger.debug("final 'index.json': " + json.dumps(index_json)[:256])
            self.write_data_to_file(output_dir, s3_commit_key, index_json_fname, index_json)
        return source_dir, success

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def mock_s3_obs_project(self):
     zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip')
     out_dir = os.path.join(self.temp_dir, 'en-obs-complete')
     unzip(zip_file, out_dir)
     project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678')
     self.project_files = [f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f))]
     self.project_key = 'door43/en-obs/12345678'
     for filename in self.project_files:
         App.cdn_s3_handler().upload_file(os.path.join(project_dir, filename), 'u/{0}/{1}'.format(self.project_key, filename))
     App.cdn_s3_handler().upload_file(os.path.join(out_dir, 'door43', 'en-obs', 'project.json'), 'u/door43/en-obs/project.json')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def test_print_obs(self):
     self.printer.print_project(self.project_key)
     self.assertTrue(App.cdn_s3_handler().key_exists('u/{0}/print_all.html'.format(self.project_key)))
     html = App.cdn_s3_handler().get_file_contents('u/{0}/print_all.html'.format(self.project_key))
     soup = BeautifulSoup(html, 'html.parser')
     self.assertEqual(len(soup.div), 69) 
     self.assertEqual(soup.html['lang'], 'en')
     self.assertEqual(soup.html['dir'], 'ltr')
     # Run again, shouldn't have to generate
     self.printer.print_project(self.project_key)
     self.assertTrue(App.cdn_s3_handler().key_exists('u/{0}/print_all.html'.format(self.project_key)))

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def upload_build_log_to_s3(self, build_log, s3_commit_key, part=''):
     """
     :param dict build_log:
     :param string s3_commit_key:
     :param string part:
     :return:
     """
     build_log_file = os.path.join(self.base_temp_dir, 'build_log.json')
     write_file(build_log_file, build_log)
     upload_key = '{0}/{1}build_log.json'.format(s3_commit_key, part)
     App.logger.debug('Saving build log to ' + upload_key)
     App.cdn_s3_handler().upload_file(build_log_file, upload_key, cache_time=0)

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def get_undeployed_parts(self, prefix):
     unfinished = []
     for o in App.cdn_s3_handler().get_objects(prefix=prefix, suffix='/build_log.json'):
         parts = o.key.split(prefix)
         if len(parts) == 2:
             parts = parts[1].split('/')
             if len(parts) > 1:
                 part_num = parts[0]
                 deployed_key = prefix + part_num + '/deployed'
                 if not App.cdn_s3_handler().key_exists(deployed_key):
                     App.logger.debug("Part {0} unfinished".format(part_num))
                     unfinished.append(part_num)
     return unfinished

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def mock_s3_tw_project(self):
     zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tw_converted.zip')
     out_dir = os.path.join(self.temp_dir, 'en_tw_converted')
     unzip(zip_file, out_dir)
     self.src_dir = src_dir = os.path.join(out_dir, 'en_tw_converted')
     self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))]
     self.project_key = 'u/door43/en_tw/12345678'
     for filename in self.project_files:
         App.cdn_s3_handler().upload_file(os.path.join(src_dir, filename), '{0}/{1}'.format(self.project_key,
                                                                                                filename))
     App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
                                      'u/door43/en_tw/project.json')
     App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
                                         'templates/project-page.html')

# 需要导入模块: from libraries.app.app import App [as 别名]
# 或者: from libraries.app.app.App import cdn_s3_handler [as 别名]
 def is_convert_finished(s3_results_key):
     key = "{0}/{1}".format(s3_results_key, 'finished')
     try:
         convert_finished = App.cdn_s3_handler().key_exists(key)
     except Exception as e:
         convert_finished = False
     return convert_finished