Python vocabulary.Vocabulary方法代码示例

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def __init__(self,
                 token_sequences,
                 filename,
                 params,
                 is_input,
                 anonymizer=None):
        self.raw_vocab = Vocabulary(
            token_sequences,
            filename,
            functional_types=INPUT_FN_TYPES if is_input else OUTPUT_FN_TYPES,
            min_occur=MIN_INPUT_OCCUR if is_input else MIN_OUTPUT_OCCUR,
            ignore_fn=lambda x: snippets.is_snippet(x) or (
                anonymizer and anonymizer.is_anon_tok(x)))
        self.tokens = set(self.raw_vocab.token_to_id.keys())
        self.inorder_tokens = self.raw_vocab.id_to_token

        assert len(self.inorder_tokens) == len(self.raw_vocab) 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def __init__(self, transform, mode, batch_size, vocab_threshold, vocab_file, start_word, 
        end_word, unk_word, annotations_file, vocab_from_file, img_folder):
        self.transform = transform
        self.mode = mode
        self.batch_size = batch_size
        self.vocab = Vocabulary(vocab_threshold, vocab_file, start_word,
            end_word, unk_word, annotations_file, vocab_from_file)
        self.img_folder = img_folder
        if self.mode == "train" or self.mode == "val":
            self.coco = COCO(annotations_file)
            self.ids = list(self.coco.anns.keys())
            print("Obtaining caption lengths...")
            all_tokens = [nltk.tokenize.word_tokenize(
                          str(self.coco.anns[self.ids[index]]["caption"]).lower())
                            for index in tqdm(np.arange(len(self.ids)))]
            self.caption_lengths = [len(token) for token in all_tokens]
        # If in test mode
        else:
            test_info = json.loads(open(annotations_file).read())
            self.paths = [item["file_name"] for item in test_info["images"]] 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def _build_vocabulary(self, config):
        vocab_path = os.path.join(self.save_dir, 'vocab.pkl')
        if os.path.exists(vocab_path):
            with open(vocab_path, 'rb') as f:
                v = pickle.load(f)
            return v

        src_vcb = vocabulary.Vocabulary(
            self._load_words(config['src_vcb']), tags=None)
        trg_vcb = vocabulary.Vocabulary(
            self._load_words(config['trg_vcb']), tags=None)
        return src_vcb, trg_vcb 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def by_name(cls, name, vocab=None, autoflush=True):
        '''Return the tag with the given name, or None.

        By default only free tags (tags which do not belong to any vocabulary)
        are returned.

        If the optional argument ``vocab`` is given then only tags from that
        vocabulary are returned, or ``None`` if there is no tag with that name
        in that vocabulary.

        :param name: the name of the tag to return
        :type name: string
        :param vocab: the vocabulary to look in (optional, default: None)
        :type vocab: ckan.model.vocabulary.Vocabulary

        :returns: the tag object with the given id or name, or None if there is
            no tag with that id or name
        :rtype: ckan.model.tag.Tag

        '''
        if vocab:
            query = meta.Session.query(Tag).filter(Tag.name==name).filter(
                Tag.vocabulary_id==vocab.id)
        else:
            query = meta.Session.query(Tag).filter(Tag.name==name).filter(
                Tag.vocabulary_id==None)
        query = query.autoflush(autoflush)
        tag = query.first()
        return tag 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def get(cls, tag_id_or_name, vocab_id_or_name=None):
        '''Return the tag with the given id or name, or None.

        By default only free tags (tags which do not belong to any vocabulary)
        are returned.

        If the optional argument ``vocab_id_or_name`` is given then only tags
        that belong to that vocabulary will be returned, and ``None`` will be
        returned if there is no vocabulary with that vocabulary id or name or
        if there is no tag with that tag id or name in that vocabulary.

        :param tag_id_or_name: the id or name of the tag to return
        :type tag_id_or_name: string
        :param vocab_id_or_name: the id or name of the vocabulary to look for
            the tag in
        :type vocab_id_or_name: string

        :returns: the tag object with the given id or name, or None if there is
            no tag with that id or name
        :rtype: ckan.model.tag.Tag

        '''
        # First try to get the tag by ID.
        tag = Tag.by_id(tag_id_or_name)
        if tag:
            return tag
        else:
            # If that didn't work, try to get the tag by name and vocabulary.
            if vocab_id_or_name:
                vocab = vocabulary.Vocabulary.get(vocab_id_or_name)
                if vocab is None:
                    # The user specified an invalid vocab.
                    raise ckan.logic.NotFound("could not find vocabulary '%s'"
                            % vocab_id_or_name)
            else:
                vocab = None
            tag = Tag.by_name(tag_id_or_name, vocab=vocab)
            return tag
        # Todo: Make sure tag names can't be changed to look like tag IDs? 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def search_by_name(cls, search_term, vocab_id_or_name=None):
        '''Return all tags whose names contain a given string.

        By default only free tags (tags which do not belong to any vocabulary)
        are returned. If the optional argument ``vocab_id_or_name`` is given
        then only tags from that vocabulary are returned.

        :param search_term: the string to search for in the tag names
        :type search_term: string
        :param vocab_id_or_name: the id or name of the vocabulary to look in
            (optional, default: None)
        :type vocab_id_or_name: string

        :returns: a list of tags that match the search term
        :rtype: list of ckan.model.tag.Tag objects

        '''
        if vocab_id_or_name:
            vocab = vocabulary.Vocabulary.get(vocab_id_or_name)
            if vocab is None:
                # The user specified an invalid vocab.
                return None
            query = meta.Session.query(Tag).filter(Tag.vocabulary_id==vocab.id)
        else:
            query = meta.Session.query(Tag)
        search_term = search_term.strip().lower()
        query = query.filter(Tag.name.contains(search_term))
        query = query.distinct().join(Tag.package_tags)
        return query 

# 需要导入模块: import vocabulary [as 别名]
# 或者: from vocabulary import Vocabulary [as 别名]
def all(cls, vocab_id_or_name=None):
        '''Return all tags that are currently applied to any dataset.

        By default only free tags (tags which do not belong to any vocabulary)
        are returned. If the optional argument ``vocab_id_or_name`` is given
        then only tags from that vocabulary are returned.

        :param vocab_id_or_name: the id or name of the vocabulary to look in
            (optional, default: None)
        :type vocab_id_or_name: string

        :returns: a list of all tags that are currently applied to any dataset
        :rtype: list of ckan.model.tag.Tag objects

        '''
        if vocab_id_or_name:
            vocab = vocabulary.Vocabulary.get(vocab_id_or_name)
            if vocab is None:
                # The user specified an invalid vocab.
                raise ckan.logic.NotFound("could not find vocabulary '%s'"
                        % vocab_id_or_name)
            query = meta.Session.query(Tag).filter(Tag.vocabulary_id==vocab.id)
        else:
            query = meta.Session.query(Tag).filter(Tag.vocabulary_id == None)
            query = query.distinct().join(PackageTag)
            query = query.filter_by(state='active')
        return query