Python tensorflow_hub.Module方法代码示例

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def create_module_graph(module_spec):
  """Creates a graph and loads Hub Module into it.

  Args:
    module_spec: the hub.ModuleSpec for the image module being used.

  Returns:
    graph: the tf.Graph that was created.
    bottleneck_tensor: the bottleneck values output by the module.
    resized_input_tensor: the input images, resized as expected by the module.
    wants_quantization: a boolean, whether the module has been instrumented
      with fake quantization ops.
  """
  height, width = hub.get_expected_image_size(module_spec)
  with tf.Graph().as_default() as graph:
    resized_input_tensor = tf.placeholder(tf.float32, [None, height, width, 3])
    m = hub.Module(module_spec)
    bottleneck_tensor = m(resized_input_tensor)
    wants_quantization = any(node.op in FAKE_QUANT_OPS
                             for node in graph.as_graph_def().node)
  return graph, bottleneck_tensor, resized_input_tensor, wants_quantization 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def export_module_spec_with_checkpoint(module_spec,
                                       checkpoint_path,
                                       export_path,
                                       scope_prefix=""):
  """Exports given checkpoint as tfhub module with given spec."""

  # The main requirement is that it is possible to know how to map from
  # module variable name to checkpoint variable name.
  # This is trivial if the original code used variable scopes,
  # but can be messy if the variables to export are interwined
  # with variables not export.
  with tf.Graph().as_default():
    m = hub.Module(module_spec)
    assign_map = {
        scope_prefix + name: value for name, value in m.variable_map.items()
    }
    tf.train.init_from_checkpoint(checkpoint_path, assign_map)
    init_op = tf.initializers.global_variables()
    with tf.Session() as session:
      session.run(init_op)
      m.export(export_path, session) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def from_hub_module(cls, hub_module, use_spm=True):
    """Get the vocab file and casing info from the Hub module."""
    with tf.Graph().as_default():
      albert_module = hub.Module(hub_module)
      tokenization_info = albert_module(signature="tokenization_info",
                                        as_dict=True)
      with tf.Session() as sess:
        vocab_file, do_lower_case = sess.run(
            [tokenization_info["vocab_file"],
             tokenization_info["do_lower_case"]])
    if use_spm:
      spm_model_file = vocab_file
      vocab_file = None
    return FullTokenizer(
        vocab_file=vocab_file, do_lower_case=do_lower_case,
        spm_model_file=spm_model_file) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def _create_model_from_hub(hub_module, is_training, input_ids, input_mask,
                           segment_ids):
  """Creates an ALBERT model from TF-Hub."""
  tags = set()
  if is_training:
    tags.add("train")
  albert_module = hub.Module(hub_module, tags=tags, trainable=True)
  albert_inputs = dict(
      input_ids=input_ids,
      input_mask=input_mask,
      segment_ids=segment_ids)
  albert_outputs = albert_module(
      inputs=albert_inputs,
      signature="tokens",
      as_dict=True)
  return (albert_outputs["pooled_output"], albert_outputs["sequence_output"]) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def __init__(self, uri, batch_size=100):
        """Create a new UseEncoderClient object

        Args:
            uri: The uri to the tensorflow_hub USE module
            batch_size: maximum number of sentences to encode at once
        """
        self._batch_size = batch_size
        self._session = tf.Session(graph=tf.Graph())
        with self._session.graph.as_default():
            glog.info("Loading %s model from tensorflow hub", uri)
            embed_fn = tf_hub.Module(uri)
            self._fed_texts = tf.placeholder(shape=[None], dtype=tf.string)
            self._embeddings = embed_fn(self._fed_texts)
            encoding_info = embed_fn.get_output_info_dict().get('default')
            if encoding_info:
                self._encoding_dim = encoding_info.get_shape()[-1].value
            init_ops = (
                tf.global_variables_initializer(), tf.tables_initializer())
        glog.info("Initializing graph.")
        self._session.run(init_ops) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def load_model(self, model: str, model_path: str, max_seq_length: int):
        g = tf.Graph()
        with g.as_default():
            hub_module = hub.Module(model_path)
            self.tokens = tf.placeholder(dtype=tf.string, shape=[None, max_seq_length])
            self.sequence_len = tf.placeholder(dtype=tf.int32, shape=[None])

            elmo_inputs = dict(
                tokens=self.tokens,
                sequence_len=self.sequence_len
            )
            self.elmo_outputs = hub_module(elmo_inputs, signature="tokens", as_dict=True)
            init_op = tf.group([tf.global_variables_initializer()])
        g.finalize()
        self.sess = tf.Session(graph=g)
        self.sess.run(init_op)

        self.model_name = model
        self.max_seq_length = max_seq_length 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def create_module_graph(module_spec):
    """Creates a graph and loads Hub Module into it.

    Args:
      module_spec: the hub.ModuleSpec for the image module being used.

    Returns:
      graph: the tf.Graph that was created.
      bottleneck_tensor: the bottleneck values output by the module.
      resized_input_tensor: the input images, resized as expected by the module.
      wants_quantization: a boolean, whether the module has been instrumented
        with fake quantization ops.
    """
    height, width = hub.get_expected_image_size(module_spec)
    with tf.Graph().as_default() as graph:
        resized_input_tensor = tf.placeholder(tf.float32, [None, height, width, 3])
        m = hub.Module(module_spec)
        bottleneck_tensor = m(resized_input_tensor)
        wants_quantization = any(node.op in FAKE_QUANT_OPS
                                 for node in graph.as_graph_def().node)
    return graph, bottleneck_tensor, resized_input_tensor, wants_quantization 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def __init__(self):

    logging.info('Initialising embedding utility...')
    embed_module = hub.Module(MODULE_URL)
    placeholder = tf.placeholder(dtype=tf.string)
    embed = embed_module(placeholder)
    session = tf.Session()
    session.run([tf.global_variables_initializer(), tf.tables_initializer()])
    logging.info('tf.Hub module is loaded.')

    def _embeddings_fn(sentences):
      computed_embeddings = session.run(
        embed, feed_dict={placeholder: sentences})
      return computed_embeddings

    self.embedding_fn = _embeddings_fn
    logging.info('Embedding utility initialised.') 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def __init__(self, hub_path=gin.REQUIRED, name="HubEmbedding", **kwargs):
    """Constructs a HubEmbedding.

    Args:
      hub_path: Path to the TFHub module.
      name: String with the name of the model.
      **kwargs: Other keyword arguments passed to tf.keras.Model.
    """
    super(HubEmbedding, self).__init__(name=name, **kwargs)

    def _embedder(x):
      embedder_module = hub.Module(hub_path)
      return embedder_module(dict(images=x), signature="representation")

    self.embedding_layer = relational_layers.MultiDimBatchApply(
        tf.keras.layers.Lambda(_embedder)) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def __init__(self,
                 model_path=None,
                 batch_size=32):
        self.resources_dir = os.path.join(os.path.dirname(__file__), 'resources')

        if model_path is None:
            model_path = os.path.join('pretrained_USE', '1fb57c3ffe1a38479233ee9853ddd7a8ac8a8c47')
        if os.path.exists(os.path.join(self.resources_dir, model_path)):
            self.url = os.path.join(self.resources_dir, model_path)
        else:
            os.environ['TFHUB_CACHE_DIR'] = os.path.join(self.resources_dir, 'pretrained_USE')
            self.url = "https://tfhub.dev/google/universal-sentence-encoder/2"

        # load the use model from saved location or tensorflow hub
        self.embed = hub.Module(self.url, trainable=True)
        self.sess = tf.Session()
        self.epochs = 10
        self.lr = 0.01
        self.batch_size = batch_size
        self.sess.run([tf.global_variables_initializer(),
                       tf.tables_initializer()]) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def __init__(self, uri):
        """Create a new `USEDualEncoder` object."""
        self._session = tf.Session(graph=tf.Graph())
        with self._session.graph.as_default():
            glog.info("Loading %s model from tensorflow hub", uri)
            embed_fn = tensorflow_hub.Module(uri)
            self._fed_texts = tf.placeholder(shape=[None], dtype=tf.string)
            self._context_embeddings = embed_fn(
                dict(input=self._fed_texts),
                signature="question_encoder",
                as_dict=True,
            )['outputs']
            empty_strings = tf.fill(
                tf.shape(self._fed_texts), ""
            )
            self._response_embeddings = embed_fn(
                dict(input=self._fed_texts, context=empty_strings),
                signature="response_encoder",
                as_dict=True,
            )['outputs']
            init_ops = (
                tf.global_variables_initializer(), tf.tables_initializer())
        glog.info("Initializing graph.")
        self._session.run(init_ops) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def _run_elmo():
    ### embedding_file = "https://tfhub.dev/google/elmo/2"
    elmo = hub.Module(embedding_file, trainable=True)

    words = [word.strip().split()[0] for word in open(vocab_file).readlines()]
    gpu_options = tf.GPUOptions(allow_growth=True)
    session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options)

    fw = open(out_file, 'w')
    fw.write('{} {}\n'.format(len(words), emb_dim))
    batch_size = 4096
    with tf.Session(config=session_config) as sess:
        i = 0
        while i < len(words):
            print('batch {}/{}...'.format(i/batch_size, len(words)/batch_size))
            _words = [words[i:(i+batch_size)]]
            embeddings = elmo(
                inputs={
                    "tokens": _words,
                    "sequence_len": [1]*len(_words)
                },
                signature="tokens",
                as_dict=True)["elmo"]
            sess.run(tf.global_variables_initializer())
            _emb = sess.run([embeddings])[0][0]
            for word, emb in zip(_words[0], np.array(_emb)):
                fw.write('{} {}\n'.format(word, ' '.join([str(_) for _ in emb])))
            i += batch_size
    fw.close() 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def create_tokenizer_from_hub_module():
  """Get the vocab file and casing info from the Hub module."""
  with tf.Graph().as_default():
    bert_module = hub.Module(BERT_MODEL_HUB)
    tokenization_info = bert_module(signature="tokenization_info", as_dict=True)
    with tf.Session() as sess:
      vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
                                            tokenization_info["do_lower_case"]])
      
  return FullTokenizer(
      vocab_file=vocab_file, do_lower_case=do_lower_case) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def create_tokenizer_from_hub_module(bert_hub_module_handle):
  """Get the vocab file and casing info from the Hub module."""
  with tf.Graph().as_default():
    bert_module = hub.Module(bert_hub_module_handle)
    tokenization_info = bert_module(signature="tokenization_info", as_dict=True)
    with tf.Session() as sess:
      vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
                                            tokenization_info["do_lower_case"]])
  return tokenization.FullTokenizer(
      vocab_file=vocab_file, do_lower_case=do_lower_case) 

# 需要导入模块: import tensorflow_hub [as 别名]
# 或者: from tensorflow_hub import Module [as 别名]
def from_hub_module(cls, hub_module, spm_model_file):
        """Get the vocab file and casing info from the Hub module."""
        import tensorflow_hub as hub
        with tf.Graph().as_default():
            albert_module = hub.Module(hub_module)
            tokenization_info = albert_module(signature="tokenization_info",
                                              as_dict=True)
            with tf.Session() as sess:
                vocab_file, do_lower_case = sess.run(
                    [tokenization_info["vocab_file"],
                     tokenization_info["do_lower_case"]])
        return FullTokenizer(
            vocab_file=vocab_file, do_lower_case=do_lower_case,
            spm_model_file=spm_model_file)