Scala Settings类代码示例

//设置package包名称以及导入依赖的类
package au.csiro.data61.magda.search.elasticsearch

import akka.actor.Scheduler
import akka.event.LoggingAdapter
import au.csiro.data61.magda.AppConfig
import au.csiro.data61.magda.util.ErrorHandling.retry
import com.sksamuel.elastic4s.{ TcpClient, ElasticsearchClientUri }
import org.elasticsearch.common.settings.Settings

import scala.concurrent.{ ExecutionContext, Future }
import scala.concurrent.duration._

trait ClientProvider {
  def getClient(implicit scheduler: Scheduler, logger: LoggingAdapter, ec: ExecutionContext): Future[TcpClient]
}

class DefaultClientProvider extends ClientProvider {
  private var clientFuture: Option[Future[TcpClient]] = None

  override def getClient(implicit scheduler: Scheduler, logger: LoggingAdapter, ec: ExecutionContext): Future[TcpClient] = {
    val outerFuture = clientFuture match {
      case Some(future) => future
      case None =>
        val future = retry(() => Future {
          val uri = ElasticsearchClientUri(AppConfig.conf().getString("elasticSearch.serverUrl"))
          val settings = Settings.builder().put("cluster.name", "myesdb").build()
          TcpClient.transport(settings, uri)
        }, 10 seconds, 10, onRetry(logger))
          .map { client =>
            logger.info("Successfully connected to elasticsearch client")
            client
          }

        clientFuture = Some(future)

        future
    }

    outerFuture
  }

  private def onRetry(logger: LoggingAdapter)(retriesLeft: Int, error: Throwable) = logger.error("Failed to make initial contact with ES server, {} retries left", retriesLeft, error)
} 

//设置package包名称以及导入依赖的类
package com.yannick_cw.elastic_indexer4s.elasticsearch.elasic_config

import com.sksamuel.elastic4s.TcpClient
import org.elasticsearch.common.settings.Settings
import org.joda.time.DateTime

import scala.concurrent.duration.{FiniteDuration, _}

case class ElasticWriteConfig(
  hosts: List[String],
  port: Int,
  cluster: String,
  indexPrefix: String,
  docType: String,
  mappingSetting: MappingSetting = TypedMappingSetting(),
  writeBatchSize: Int = 50,
  writeConcurrentRequest: Int = 10,
  writeMaxAttempts: Int = 5,
  logWriteSpeedEvery: FiniteDuration = 1 minute,
  waitForElasticTimeout: FiniteDuration = 5 seconds
) {
  val indexName = indexPrefix + "_" + new DateTime().toString("yyyy-MM-dd't'HH:mm:ss")
  private def settings = Settings.builder().put("cluster.name", cluster).build()
  lazy val client: TcpClient = TcpClient.transport(settings, "elasticsearch://" + hosts
    .map(host => s"$host:$port").mkString(","))
}

object ElasticWriteConfig {
  def apply(
    esTargetHosts: List[String],
    esTargetPort: Int,
    esTargetCluster: String,
    esTargetIndexPrefix: String,
    esTargetType: String
  ): ElasticWriteConfig =
    new ElasticWriteConfig(esTargetHosts, esTargetPort, esTargetCluster, esTargetIndexPrefix, esTargetType)
} 

//设置package包名称以及导入依赖的类
package com.techmonad.es

import java.io.File

import com.techmonad.logger.Logging
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest
import org.elasticsearch.client.Client
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.node.NodeBuilder


trait ESTestHelper extends Logging {

  val indexName = "catalogue-test-index"

  private val esDataDir = "es-data"

 lazy private val settings = Settings.settingsBuilder()
    .put("cluster.name", "elasticsearch-test")
    .put("discovery.zen.ping.multicast.enabled", false)
    .put("http.enabled", false)
    .put("client", true)
    .put("path.home", esDataDir)
    .put("local", true)
    .put("client.transport.sniff", false)


  lazy val node = NodeBuilder.nodeBuilder().settings(settings).node()

  lazy val client: Client = node.client()


  def init() = {
    val scalaBook ="""{"id":2, "type":"book", "author":"Martin Odersky", "title": "Programming in Scala"}"""
    val akkaBook ="""{"id":3, "type":"book", "author":"Derek Wyatt" , "title": "Akka Concurrency"}"""
    val builder = client.prepareBulk()
    builder.add(client.prepareIndex().setIndex(indexName).setId("2").setType("book").setSource(scalaBook))
    builder.add(client.prepareIndex().setIndex(indexName).setId("3").setType("book").setSource(akkaBook))
    val result = builder.execute().actionGet().hasFailures
    info("inserted record into ES, result -> " + result)
    refresh()
  }

  def refresh() = {
    client.admin().indices().refresh(new RefreshRequest(indexName)).get
  }

  def close() = {
    info("Closing ES node....")
    node.close()
    info("Deleting es data directory....")
    FileUtils.deleteDirectory(new File(esDataDir));
  }
} 

//设置package包名称以及导入依赖的类
package util

import org.elasticsearch.common.settings.Settings
import org.elasticsearch.node.{Node, NodeBuilder}

trait TestHelper {

  val localClient: Node = {
    val esSettings = Settings.settingsBuilder()
      .put("cluster.name", "elasticsearch")
      .put("discovery.zen.ping.multicast.enabled", false)
      .put("http.enabled", false)
      .put("client", true)
      .put("path.home","data")
      .put("local", true)
      .put("client.transport.sniff", false)
    val node =NodeBuilder.nodeBuilder().settings(esSettings).node()
    node
  }

} 

//设置package包名称以及导入依赖的类
package models

import javax.inject.Inject

import com.sksamuel.elastic4s.ElasticClient
import com.sksamuel.elastic4s.ElasticDsl._
import org.elasticsearch.common.settings.Settings
import play.api.Logger

import scala.collection.mutable
import scala.util.{Failure, Success, Try}

@javax.inject.Singleton
class FundsService @Inject() (
  configuration: play.api.Configuration
) {

  val settings = Settings.settingsBuilder()
    .put("http.enabled", false)
    .put("path.home", configuration.underlying.getString("elastic.data.path"))
  val client = ElasticClient.local(settings.build)

  Try(insertBlocking(Funds(mutable.Buffer.empty))) match {
    case Success(_) => ()

    case Failure(cause) =>
      Logger.error(s"Could not create index - " + cause)
  }

  def find(filter: String, offset: Int, pageSize: Int) = {
    client.execute { search in "gov" / "funds" query filter from offset size pageSize }
  }

  def findBlocking(filter: String, offset: Int, pageSize: Int) = {
    client.execute { search in "gov" / "funds" query filter from offset size pageSize }.await
  }

  def insert(funds: Funds) = {
    client.execute { index into "gov" / "funds" fields funds.data }
  }

  def insertBlocking(funds: Funds) = {
    client.execute { index into "gov" / "funds" fields funds.data }.await
  }

  def deleteAll() = {
    client.execute { deleteIndex("gov") }
  }
}

case class Funds(data: mutable.Buffer[(String, String)]) 

//设置package包名称以及导入依赖的类
package twitter

import com.sksamuel.elastic4s._
import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.mappings.FieldType.{DateType, GeoPointType, StringType}
import org.elasticsearch.common.geo.GeoPoint
import org.elasticsearch.common.settings.Settings


object ESLoader {

  val settings = Settings.builder
    .put("http.enabled", "false")
    .put("cluster.name", "elasticsearch_airton").build()

  val uri = ElasticsearchClientUri("elasticsearch://localhost:9300")
  val client = ElasticClient.transport(settings, uri)

  def saveToES(tweet: AnalyzedTweet) = client.execute {index into "twitter2/tweets" fields toMap(tweet) }.await

  def toMap(tweet: AnalyzedTweet) = {
    val l = tweet.location match {
      case Some(location) => new GeoPoint(location("lat"), location("lon")).geohash()
      case None           => null
    }
    Map(
      "text"      -> tweet.text,
      "created"   -> tweet.created,
      "location"  -> l,
      "language"  -> tweet.language,
      "user"      -> tweet.user,
      "sentiment" -> tweet.sentiment)
  }

  def createIndex = client.execute {
    create index "twitter2" mappings(
      "tweets" as (
        "text"      typed StringType,
        "language"  typed StringType,
        "user"      typed StringType,
        "sentiment" typed StringType index "not_analyzed",
        "location"  typed GeoPointType,
        "created"   typed DateType
        ))
  }.await

  def main(args: Array[String]) = {
    println("kdkvadvad")
    createIndex
    println("kdkvadvad")
  }

} 

//设置package包名称以及导入依赖的类
package houseprices.elasticsearch.config

import org.elasticsearch.client.Client
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.transport.InetSocketTransportAddress
import org.elasticsearch.node.NodeBuilder
import org.elasticsearch.common.settings.Settings
import java.net.InetAddress
import org.elasticsearch.node.Node

trait EsClientBuilder {
  this: EsConfig =>

  val settings = Settings.settingsBuilder()
    .put("http.enabled", this.httpEnabled)
    .put("path.home", this.pathHome)
    .put("path.data", this.pathData)
    .put("node.local", this.isLocal)
    .put("node.client", this.isClient)
    .put("node.name", "node." + this.clusterName)

  def createNode: Node = {
    NodeBuilder.nodeBuilder()
      .clusterName(clusterName)
      .settings(settings)
      .build()
  }

  def build: Client = {
    createNode.start().client()
  }
}

object EsClientBuilder {
  def buildClient(env: String = "dev"): Client = {
    env match {
      case "qa" => (new EsClientBuilder with QaConfig) build
      case _ => (new EsClientBuilder with DevConfig) build
    }
  }

  def transportClient = {
    val settings = Settings.settingsBuilder().put("cluster.name", "dev.pricepaid")
    TransportClient.builder().settings(settings).build().addTransportAddress(new InetSocketTransportAddress(InetAddress.getLocalHost, 9300))
  }

} 

//设置package包名称以及导入依赖的类
package houseprices.elasticsearch

import org.elasticsearch.client.Client
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest
import org.slf4j.LoggerFactory
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.io.stream.StreamOutput
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput
import org.elasticsearch.common.io.stream.BytesStreamOutput

class CreateIndex(val client: Client, val indexName: String, val typeName: String, mappingJsonSource: Option[String]) {

  private val log = LoggerFactory.getLogger(getClass)
  private val admin = client.admin()

  def recreate = {
    log.info("Deleting index " + indexName)
    val f = admin.indices.prepareDelete(indexName).execute
      .addListener(new ActionListener[DeleteIndexResponse] {
        def onFailure(error: Throwable) = createIndex
        def onResponse(response: DeleteIndexResponse) = createIndex
      })
  }

  def createIfNotExists = {
    val response = admin.indices.prepareExists(indexName).execute.actionGet
    if (!response.isExists) createIndex
    else
      log.info("{} index already exists...skipping", indexName)
  }

  private def createIndex = {
    log.info("Creating index " + indexName)
    admin.indices.prepareCreate(indexName)
      .setSettings(Settings.builder
        .put("index.number_of_shards", 1)
        .put("index.number_of_replicas", 0))
      .execute.actionGet

    mappingJsonSource.map { json =>
      log.info("Updating mapping...")
      val mappingRequest = new PutMappingRequest(indexName).`type`(typeName).source(json)
      admin.indices.putMapping(mappingRequest).actionGet();
    }
    admin.cluster.prepareHealth(indexName).setWaitForActiveShards(1).execute.actionGet
  }
} 

//设置package包名称以及导入依赖的类
package com.puroguramingu.util

import java.io.File
import java.util.UUID

import com.sksamuel.elastic4s.ElasticClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.node.NodeBuilder
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite}

trait ElasticSearch extends BeforeAndAfterEach with BeforeAndAfterAll {
  this: Suite =>

  private val tempFile = File.createTempFile("elasticsearchtests", "tmp")
  private val homeDir = new File(tempFile.getParent + "/" + UUID.randomUUID().toString)

  homeDir.mkdir()
  homeDir.deleteOnExit()
  tempFile.deleteOnExit()

  protected val settings = Settings.settingsBuilder()
    .put("node.http.enabled", true)
    .put("http.enabled", true)
    .put("index.store.type", "niofs")
    .put("path.home", homeDir.getAbsolutePath)
    .put("index.number_of_shards", 1)
    .put("index.number_of_replicas", 0)
    .put("es.logger.level", "DEBUG")
    .put("http.port", 9500)
    .put("transport.tcp.port", 9400)
    .build()

  val node = NodeBuilder.nodeBuilder().settings(settings).local(false).build().start()

  val client = ElasticClient.fromNode(node)

} 

//设置package包名称以及导入依赖的类
package com.pharmpress.dmdbrowser

import com.google.inject.AbstractModule
import com.pharmpress.elasticsearch.{Elastic4sServiceImpl, IElasticSearch}
import com.sksamuel.elastic4s.{ElasticClient, ElasticsearchClientUri}
import org.elasticsearch.common.settings.{ImmutableSettings, Settings}
import play.api.{Configuration, Environment, Logger}


class ApplicationModule(
                         environment: Environment,
                         configuration: Configuration
                       ) extends AbstractModule {

  def configure: Unit = {

    val conf = configuration.underlying
    val rpsConfigUri = conf.getString("es.rps.client.uri")
    val rpsConfigCluster = conf.getString("es.rps.cluster.name")

    bind(classOf[IElasticSearch]) toInstance new Elastic4sServiceImpl(() => {
      Logger.info(s"Connecting to Elasticsearch at $rpsConfigUri with cluster $rpsConfigCluster")
      ElasticClient.remote(
        ImmutableSettings.settingsBuilder()
          .classLoader(classOf[Settings].getClassLoader)
          .put("cluster.name", rpsConfigCluster)
          .build(),
        ElasticsearchClientUri(rpsConfigUri)
      )
    })
  }
} 

//设置package包名称以及导入依赖的类
package com.github.jparkie.spark.elasticsearch.transport

import com.github.jparkie.spark.elasticsearch.conf.SparkEsTransportClientConf
import org.apache.spark.Logging
import org.elasticsearch.client.Client
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.InetSocketTransportAddress

import scala.collection.mutable

private[elasticsearch] trait SparkEsTransportClientManager extends Serializable with Logging {
  @transient
  private[transport] val internalTransportClients = mutable.HashMap.empty[SparkEsTransportClientConf, TransportClient]

  private[transport] def buildTransportSettings(clientConf: SparkEsTransportClientConf): Settings = {
    val esSettingsBuilder = Settings.builder()

    clientConf.transportSettings foreach { currentSetting =>
      esSettingsBuilder.put(currentSetting._1, currentSetting._2)
    }

    esSettingsBuilder.build()
  }

  private[transport] def buildTransportClient(clientConf: SparkEsTransportClientConf, esSettings: Settings): TransportClient = {
    import SparkEsTransportClientConf._

    val esClient = TransportClient.builder()
      .settings(esSettings)
      .build()

    getTransportAddresses(clientConf.transportAddresses, clientConf.transportPort) foreach { inetSocketAddress =>
      esClient.addTransportAddresses(new InetSocketTransportAddress(inetSocketAddress))
    }

    sys.addShutdownHook {
      logInfo("Closed Elasticsearch Transport Client.")

      esClient.close()
    }

    logInfo(s"Connected to the following Elasticsearch nodes: ${esClient.connectedNodes()}.")

    esClient
  }

  
  def closeTransportClient(clientConf: SparkEsTransportClientConf): Unit = synchronized {
    internalTransportClients.remove(clientConf) match {
      case Some(transportClient) =>
        transportClient.close()
      case None =>
        logError(s"No TransportClient for $clientConf.")
    }
  }
}

object SparkEsTransportClientManager extends SparkEsTransportClientManager 

//设置package包名称以及导入依赖的类
package io.github.chenfh5.lucene_analysis.pinyin

import org.elasticsearch.analysis.PinyinConfig
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.index.analysis.PinyinAnalyzer
import org.slf4j.LoggerFactory

import io.github.chenfh5.lucene_analysis.CustomAnalyzer


object PinyinClient extends CustomAnalyzer {
  private val LOG = LoggerFactory.getLogger(getClass.getName)

  
  private lazy val pinyinSetting = {
    Settings.builder()
        .put("keep_first_letter", true)
        .put("keep_full_pinyin", false) //necessary for my business
        .put("keep_joined_full_pinyin", true)
        .put("none_chinese_pinyin_tokenize", false)
        .put("limit_first_letter_length", 30)
        .put("keep_original", true)
  }

  private lazy val pinyinAnalyzer = {
    val setting = pinyinSetting.build()
    val pinyinAnalyzer = new PinyinAnalyzer(new PinyinConfig(setting))
    LOG.info("this is the pinyinAnalyzer={}, initialized successfully", pinyinAnalyzer)

    pinyinAnalyzer
  }

  def getPinyinTokens(inputText: String) = {
    getTokens(inputText, pinyinAnalyzer)
  }

} 

//设置package包名称以及导入依赖的类
package io.github.chenfh5.lucene_analysis.ik

import java.io.File

import org.apache.commons.lang3.StringUtils
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.env.Environment
import org.slf4j.LoggerFactory
import org.wltea.analyzer.cfg.Configuration
import org.wltea.analyzer.lucene.IKAnalyzer

import io.github.chenfh5.lucene_analysis.CustomAnalyzer


object IkClient extends CustomAnalyzer {
  private val LOG = LoggerFactory.getLogger(getClass.getName)

  private val ikSetting = {
    val path = new File(getClass.getClassLoader.getResource("es-config").toURI)
    Settings.builder()
        .put("path.home", "")
        .put("path.conf", path)
        .put("use_smart", true)
  }

  private lazy val ikAnalyzerSmart = {
    val setting = ikSetting.build()
    val ikAnalyzerSmart = new IKAnalyzer(new Configuration(new Environment(setting), setting))
    LOG.info("this is the ikAnalyzerSmart={}, initialized successfully", ikAnalyzerSmart)

    ikAnalyzerSmart
  }

  private lazy val ikAnalyzerFull = {
    val setting = ikSetting
        .put("use_smart", false)
        .build()
    val ikAnalyzerFull = new IKAnalyzer(new Configuration(new Environment(setting), setting))
    LOG.info("this is the ikAnalyzerFull={}, initialized successfully", ikAnalyzerFull)

    ikAnalyzerFull
  }

  def getIkTokens(inputText: String, useSmart: Boolean = true) = {
    require(StringUtils.isNotBlank(inputText), "An input text must be specified")
    val ikAnalyzer = if (useSmart) ikAnalyzerSmart else ikAnalyzerFull

    getTokens(inputText, ikAnalyzer)
  }

} 

//设置package包名称以及导入依赖的类
package io.github.chenfh5.java_api

import java.net.InetAddress

import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.InetSocketTransportAddress
import org.slf4j.LoggerFactory

import io.github.chenfh5.common.OwnConfigReader


object EsClient {
  private val LOG = LoggerFactory.getLogger(getClass.getName)

  private val clusterName = OwnConfigReader.getOwnProperty.clusterName
  private val ips = OwnConfigReader.getOwnProperty.ips
  private val port = OwnConfigReader.getOwnProperty.javaClientPort.toInt

  private val transportClient = {
    val settings = Settings
        .builder()
        .put("client.transport.sniff", true)
        .put("cluster.name", clusterName)
        .build()

    val transportClient = addIps(TransportClient.builder().settings(settings).build())
    LOG.info("this is the transportClient={}, initialized successfully", transportClient)
    transportClient
  }

  private def addIps(transportClient: TransportClient) = {
    val ipArray = ips.split(",")
    for (oneIP <- ipArray) {
      transportClient
          .addTransportAddresses(new InetSocketTransportAddress(InetAddress.getByName(oneIP.trim), port))
    }
    transportClient
  }

  def getEsClient = this.transportClient

  def getBuckClient = {
    this.transportClient.prepareBulk()
  }

} 

//设置package包名称以及导入依赖的类
package com.stratio.ioft.persistence

import com.sksamuel.elastic4s.ElasticClient
import com.sksamuel.elastic4s.ElasticDsl._
import com.stratio.ioft.settings.IOFTConfig
import org.apache.spark.streaming.dstream.DStream
import org.elasticsearch.common.settings.Settings

import scala.collection.JavaConversions._

object ESPersistence extends IOFTConfig {

  val settings = Settings.settingsBuilder()

  esConfig.entrySet().map {
    entry =>
      settings.put(entry.getKey, esConfig.getAnyRef(entry.getKey))
  }

  val client = ElasticClient.local(settings.build)

  def persist(dStream: DStream[(String, String)]) ={
    dStream.foreachRDD{ rdd =>
      client.execute{ index into "ioft" / "drone" fields rdd.collect()}
    }
  }

}