Python extract.extract_from_dir函数代码示例

def extract_messages(dirs):
    catalog = Catalog(
        project='Open Library',
        copyright_holder='Internet Archive'
    )
    METHODS = [
        ("**.py", "python"),
        ("**.html", "openlibrary.i18n:extract_templetor")
    ]
    COMMENT_TAGS = ["NOTE:"]

    for d in dirs:
        if '.html' in d:
            extracted = [(d,) + extract for extract in extract_from_file("openlibrary.i18n:extract_templetor", d)]
        else:
            extracted = extract_from_dir(d, METHODS, comment_tags=COMMENT_TAGS, strip_comment_tags=True)
        for filename, lineno, message, comments, context in extracted:
            catalog.add(message, None, [(filename, lineno)], auto_comments=comments)

    path = os.path.join(root, 'messages.pot')
    f = open(path, 'w')
    write_po(f, catalog)
    f.close()

    print('wrote template to', path)

 def extract(self, force=False):
     """
     Extract translation strings from sources directory with extract rules then 
     create the template catalog with finded translation strings
     
     Only proceed if the template catalog does not exists yet or if 
     ``force`` argument is ``True`` (this will overwrite previous existing 
     POT file)
     
     TODO: actually from the CLI usage this only update POT file when he does not 
     exist, else it keeps untouched, even if there changes or adds in translations
     """
     if force or not self.check_template_path():
         self.logger.info('Proceeding to extraction to update the template catalog (POT)')
         self._catalog_template = Catalog(project=self.settings.SITE_NAME, header_comment=self.header_comment)
         # Follow all paths to search for pattern to extract
         for extract_path in self.settings.I18N_EXTRACT_SOURCES:
             self.logger.debug('Searching for pattern to extract in : {0}'.format(extract_path))
             extracted = extract_from_dir(dirname=extract_path, method_map=self.settings.I18N_EXTRACT_MAP, options_map=self.settings.I18N_EXTRACT_OPTIONS)
             # Proceed to extract from given path
             for filename, lineno, message, comments, context in extracted:
                 filepath = os.path.normpath(os.path.join(os.path.basename(self.settings.SOURCES_DIR), filename))
                 self._catalog_template.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
         
         outfile = open(self.get_template_path(), 'wb')
         write_po(outfile, self._catalog_template)
         outfile.close()
         
     return self._catalog_template

def extract_to_catalog(catalog, data_iterator, keywords, comment_tags,
                       strip_comment_tags, log=log):

    for dirname, method_map, options_map in data_iterator:
        if not os.path.isdir(dirname):
            raise ConfigureError('%r is not a directory' % dirname)

        def callback(filename, method, options):
            if method == 'ignore':
                return
            filepath = _gen_filepath(dirname, filename)
            optstr = _gen_optstr(options)
            log.info('extracting messages from %s%s', filepath, optstr)

        extracted = extract_from_dir(dirname, method_map, options_map,
                                     keywords=keywords,
                                     comment_tags=comment_tags,
                                     callback=callback,
                                     strip_comment_tags=strip_comment_tags)

        # Add extracted strings to catalog
        for filename, lineno, message, comments, context in extracted:
            filepath = _gen_filepath(dirname, filename)
            catalog.add(message, None, [(filepath, lineno)],
                        auto_comments=comments, context=context)

    def run(self):
        mappings = self._get_mappings()
        with open(self.output_file, 'wb') as outfile:
            catalog = Catalog(project=self.project,
                              version=self.version,
                              msgid_bugs_address=self.msgid_bugs_address,
                              copyright_holder=self.copyright_holder,
                              charset=self.charset)

            for path, method_map, options_map in mappings:
                def callback(filename, method, options):
                    if method == 'ignore':
                        return

                    # If we explicitly provide a full filepath, just use that.
                    # Otherwise, path will be the directory path and filename
                    # is the relative path from that dir to the file.
                    # So we can join those to get the full filepath.
                    if os.path.isfile(path):
                        filepath = path
                    else:
                        filepath = os.path.normpath(os.path.join(path, filename))

                    optstr = ''
                    if options:
                        optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
                                                      k, v in options.items()])
                    self.log.info('extracting messages from %s%s', filepath, optstr)

                if os.path.isfile(path):
                    current_dir = os.getcwd()
                    extracted = check_and_call_extract_file(
                        path, method_map, options_map,
                        callback, self.keywords, self.add_comments,
                        self.strip_comments, current_dir
                    )
                else:
                    extracted = extract_from_dir(
                        path, method_map, options_map,
                        keywords=self.keywords,
                        comment_tags=self.add_comments,
                        callback=callback,
                        strip_comment_tags=self.strip_comments
                    )
                for fname, lineno, msg, comments, context, flags in extracted:
                    if os.path.isfile(path):
                        filepath = fname  # already normalized
                    else:
                        filepath = os.path.normpath(os.path.join(path, fname))

                    catalog.add(msg, None, [(filepath, lineno)], flags=flags,
                                auto_comments=comments, context=context)

            self.log.info('writing PO template file to %s', self.output_file)
            write_po(outfile, catalog, width=self.width,
                     no_location=self.no_location,
                     omit_header=self.omit_header,
                     sort_output=self.sort_output,
                     sort_by_file=self.sort_by_file,
                     include_lineno=self.include_lineno)

    def extract(self, src_path='.', charset='utf-8', locale=None, **kwargs):
        """Extracts translatable messages from sources. This function is based
        on the extract function of the `pybabel` command, which is not part of
        the public API of `babel`. Only the public API of `babel` is used here.

        :param src_path: base path of the source tree, default is the current
                         path.
        :param charset: see the `babel.messages.catalog.Catalog` docs. Default
                        is `utf-8`.
        :param locale: see the `babel.messages.catalog.Catalog` docs. Default
                        is `None`.

        Other optional keyword parameters are passed to
        `babel.messages.extract.extract_from_dir` see `babel` public API docs.
        """
        #: This is the babel.messages.catalog.Catalog to contain the
        #: extracted messages
        self.catalog = Catalog(charset=charset, locale=locale)

        if not pth.isdir(src_path):
            raise IOError('{} is not a directory'.format(src_path))

        #: Extracts the data from source in a low level format. This is
        #: the only way present in babel's public API.
        extracted = extract_from_dir(src_path, **kwargs)

        #: Constructs the catalog from the raw extracted data.
        #: Based on the source code of pybabel:
        #: babel.messages.frontend.extract_messages.run
        for filename, lineno, message, comments, context in extracted:
            self.catalog.add(message, None, [(filename, lineno)],
                             auto_comments=comments, context=context)

def run_gettext(dirname, for_js):
    catalog = Catalog()
    for filename, lineno, message, comments, context in extract_from_dir(
        dirname,
        method_map=[('**.js', 'javascript')] if for_js else [('**.py', 'python')]
    ):
        catalog.add(message, None, [(filename, lineno)],
                    auto_comments=comments, context=context)

    sio = cStringIO.StringIO()
    write_po(sio, catalog)

    return sio.getvalue()

def main():
    print "Extracting messages"
    root = path.abspath(path.join(path.dirname(__file__), '..'))
    catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
                      copyright_holder=COPYRIGHT,
                      charset="utf-8")

    def callback(filename, method, options):
        if method != "ignore":
            print strip_path(filename, root)

    extracted_py = extract_from_dir(root, PY_METHODS, {}, PY_KEYWORDS,
                                    COMMENT_TAGS, callback=callback,
                                    strip_comment_tags=True)

    extracted_js = extract_from_dir(root, [("static/js/**.js", "javascript")],
                                    {}, JS_KEYWORDS,
                                    COMMENT_TAGS, callback=callback,
                                    strip_comment_tags=True)

    for filename, lineno, message, comments in chain(extracted_py,
                                                     extracted_js):
        catalog.add(message, None, [(strip_path(filename, root), lineno)],
                    auto_comments=comments)

    output_path = path.join(root, "i18n")
    if not path.isdir(output_path):
        makedirs(output_path)

    f = file(path.join(output_path, "messages.pot"), "w")
    try:
        write_po(f, catalog, width=79)
    finally:
        f.close()

    print "All done."

    def set_nereid(self):
        """
        There are messages within the tryton code used in flash messages,
        returned responses etc. This is spread over the codebase and this
        function extracts the translation strings from code of installed
        modules.
        """
        pool = Pool()
        Translation = pool.get('ir.translation')
        to_create = []

        for module, directory in self._get_installed_module_directories():
            # skip messages from test files
            if 'tests' in directory:
                continue
            for (filename, lineno, messages, comments, context) in \
                    extract_from_dir(directory,):

                if isinstance(messages, basestring):
                    # messages could be a tuple if the function is ngettext
                    # where the messages for singular and plural are given as
                    # a tuple.
                    #
                    # So convert basestrings to tuples
                    messages = (messages, )

                for message in messages:
                    translations = Translation.search([
                        ('lang', '=', 'en_US'),
                        ('type', '=', 'nereid'),
                        ('name', '=', filename),
                        ('src', '=', message),
                        ('module', '=', module),
                    ], limit=1)
                    if translations:
                        continue
                    to_create.append({
                        'name': filename,
                        'res_id': lineno,
                        'lang': 'en_US',
                        'src': message,
                        'type': 'nereid',
                        'module': module,
                        'comments': comments and '\n'.join(comments) or None,
                    })
        if to_create:
            Translation.create(to_create)

    def build_pot(self, force=False):
        """
        Extract translation strings and create Portable Object Template (POT)
        from enabled source directories using defined extract rules.

        Note:
            May only work on internal '_pot' to return without touching
            'self._pot'.

        Keyword Arguments:
            force (boolean): Default behavior is to proceed only if POT file
                does not allready exists except if this argument is ``True``.

        Returns:
            babel.messages.catalog.Catalog: Catalog template object.
        """
        if force or not self.check_template_path():
            self.logger.info(("Proceeding to extraction to update the "
                              "template catalog (POT)"))
            self._pot = Catalog(project=self.settings.SITE_NAME,
                                header_comment=self.header_comment)
            # Follow all paths to search for pattern to extract
            for extract_path in self.settings.I18N_EXTRACT_SOURCES:
                msg = "Searching for pattern to extract in : {0}"
                self.logger.debug(msg.format(extract_path))
                extracted = extract_from_dir(
                    dirname=extract_path,
                    method_map=self.settings.I18N_EXTRACT_MAP,
                    options_map=self.settings.I18N_EXTRACT_OPTIONS
                )
                # Proceed to extract from given path
                for filename, lineno, message, comments, context in extracted:
                    filepath = os.path.normpath(
                        os.path.join(
                            os.path.basename(self.settings.SOURCES_DIR),
                            filename
                        )
                    )
                    self._pot.add(message, None, [(filepath, lineno)],
                                  auto_comments=comments, context=context)

            with io.open(self.get_template_path(), 'wb') as fp:
                write_po(fp, self._pot)

        return self._pot

def extract_python_strings(dirname, outfile, domain=None):
    """Extract translatable strings from all Python files in `dir`.

    Writes a PO template to `outfile`. Recognises `_` and `l_`. Needs babel!
    """
    from babel.messages.catalog import Catalog
    from babel.messages.extract import extract_from_dir
    from babel.messages.pofile import write_po
    base_dir = os.path.abspath(os.path.dirname(outfile))
    msgs = extract_from_dir(dirname, keywords={'_': None, 'l_': None},
                            comment_tags=['l10n:'])
    cat = Catalog(domain=domain, charset='utf-8')
    for fname, lineno, message, comments, context in msgs:
        filepath = os.path.join(dirname, fname)
        cat.add(message, None, [(os.path.relpath(filepath, base_dir), lineno)],
                auto_comments=comments, context=context)
    with open(outfile, 'wb') as f:
        write_po(f, cat)

    def handle(self, *args, **options):
        domains = options.get('domain')

        if domains == "all":
            domains = settings.DOMAIN_METHODS.keys()
        else:
            domains = [domains]

        root = settings.ROOT

        def callback(filename, method, options):
            if method != 'ignore':
                print "  %s" % filename

        for domain in domains:

            print "Extracting all strings in domain %s..." % (domain)

            methods = settings.DOMAIN_METHODS[domain]
            extracted = extract_from_dir(root,
                                         method_map=methods,
                                         keywords=DEFAULT_KEYWORDS,
                                         comment_tags=COMMENT_TAGS,
                                         callback=callback,
                                         options_map=OPTIONS_MAP,
                                         )
            catalog = create_pofile_from_babel(extracted)
            catalog.savefile(os.path.join(root, 'locale', 'z-%s.pot' % domain))

        if len(domains) > 1:
            print "Concatenating all domains..."
            pot_files = []
            for i in domains:
                pot_files.append(os.path.join(root, 'locale', 'z-%s.pot' % i))
            z_keys = open(os.path.join(root, 'locale', 'z-keys.pot'), 'w+t')
            z_keys.truncate()
            command = ["msgcat"] + pot_files
            p1 = Popen(command, stdout=z_keys)
            p1.communicate()
            z_keys.close()
            for i in domains:
                os.remove(os.path.join(root, 'locale', 'z-%s.pot' % i))

        print 'done'

    def set_wtforms(self):
        """
        There are some messages in WTForms which are provided by the framework,
        namely default validator messages and errors occuring during the
        processing (data coercion) stage. For example, in the case of the
        IntegerField, if someone entered a value which was not valid as
        an integer, then a message like “Not a valid integer value” would be
        displayed.
        """
        pool = Pool()
        Translation = pool.get('ir.translation')
        to_create = []
        for (filename, lineno, messages, comments, context) in \
                extract_from_dir(os.path.dirname(wtforms.__file__)):

            if isinstance(messages, basestring):
                # messages could be a tuple if the function is ngettext
                # where the messages for singular and plural are given as
                # a tuple.
                #
                # So convert basestrings to tuples
                messages = (messages, )

            for message in messages:
                translations = Translation.search([
                    ('lang', '=', 'en_US'),
                    ('type', '=', 'wtforms'),
                    ('name', '=', filename),
                    ('src', '=', message),
                    ('module', '=', 'nereid'),
                ], limit=1)
                if translations:
                    continue
                to_create.append({
                    'name': filename,
                    'res_id': lineno,
                    'lang': 'en_US',
                    'src': message,
                    'type': 'wtforms',
                    'module': 'nereid',
                    'comments': comments and '\n'.join(comments) or None,
                })
        if to_create:
            Translation.create(to_create)

        def extract_messages(   # noqa
                dirname, project=app.name, version=app.cfg.get('VERSION', ''),
                charset='utf-8', domain=self.cfg.domain, locale=self.cfg.default_locale):
            """Extract messages from source code.

            :param charset: charset to use in the output
            :param domain:  set domain name for locales
            :param project: set project name in output
            :param version: set project version in output
            """
            Locale.parse(locale)

            if not os.path.isdir(dirname):
                raise SystemExit('%r is not a directory' % dirname)

            catalog = Catalog(locale=locale, project=project, version=version, charset=charset)
            for filename, lineno, message, comments, context in extract_from_dir(
                    dirname, method_map=self.cfg.sources_map, options_map=self.cfg.options_map):
                filepath = os.path.normpath(os.path.join(dirname, filename))
                catalog.add(message, None, [(filepath, lineno)],
                            auto_comments=comments, context=context)

            locales_dir = self.cfg.locales_dirs[0]
            output = os.path.join(locales_dir, locale, 'LC_MESSAGES', '%s.po' % domain)

            if os.path.exists(output):
                with open(output, 'rb') as f:
                    template = read_po(f, locale=locale, charset=charset)
                    template.update(catalog)
                    catalog = template

            if not os.path.exists(os.path.dirname(output)):
                os.makedirs(os.path.dirname(output))

            logger.info('writing PO template file to %s', output)
            outfile = open(output, 'wb')

            try:
                write_po(outfile, catalog, include_previous=True)
            finally:
                outfile.close()

    def run(self):
        mappings = self._get_mappings()
        outfile = open(self.output_file, 'wb')
        try:
            catalog = Catalog(project=self.distribution.get_name(),
                              version=self.distribution.get_version(),
                              msgid_bugs_address=self.msgid_bugs_address,
                              copyright_holder=self.copyright_holder,
                              charset=self.charset)

            for dirname, (method_map, options_map) in mappings.items():
                def callback(filename, method, options):
                    if method == 'ignore':
                        return
                    filepath = os.path.normpath(os.path.join(dirname, filename))
                    optstr = ''
                    if options:
                        optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
                                                      k, v in options.items()])
                    log.info('extracting messages from %s%s', filepath, optstr)

                extracted = extract_from_dir(dirname, method_map, options_map,
                                             keywords=self._keywords,
                                             comment_tags=self._add_comments,
                                             callback=callback,
                                             strip_comment_tags=
                                                self.strip_comments)
                for filename, lineno, message, comments, context in extracted:
                    filepath = os.path.normpath(os.path.join(dirname, filename))
                    catalog.add(message, None, [(filepath, lineno)],
                                auto_comments=comments, context=context)

            log.info('writing PO template file to %s' % self.output_file)
            write_po(outfile, catalog, width=self.width,
                     no_location=self.no_location,
                     omit_header=self.omit_header,
                     sort_output=self.sort_output,
                     sort_by_file=self.sort_by_file)
        finally:
            outfile.close()

 def extract(self, force=False):
     """
     Extract translation strings from sources directory with extract rules then 
     create the template catalog with finded translation strings
     
     Only proceed if the template catalog does not exists yet or if 
     ``force`` argument is ``True`` (this will overwrite previous existing 
     POT file)
     """
     if force or not self.check_template_path():
         self.logger.warning('Template catalog (POT) does not exists, extracting it')
         self._catalog_template = Catalog(project=self.settings.SITE_NAME, header_comment=self.header_comment)
         extracted = extract_from_dir(dirname=self.settings.SOURCES_DIR, method_map=self.settings.I18N_EXTRACT_MAP, options_map=self.settings.I18N_EXTRACT_OPTIONS)
         
         for filename, lineno, message, comments, context in extracted:
             filepath = os.path.normpath(os.path.join(os.path.basename(self.settings.SOURCES_DIR), filename))
             self._catalog_template.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
         
         outfile = open(self.get_template_path(), 'wb')
         write_po(outfile, self._catalog_template)
         outfile.close()
         
     return self._catalog_template