Scala ByteString类代码示例

//设置package包名称以及导入依赖的类
package com.stacktrace.yo.scrapeline.imdb.pipelines

import java.nio.file.Paths

import akka.NotUsed
import akka.stream.scaladsl.{FileIO, Flow, Keep, Sink, Source}
import akka.stream.{ActorMaterializer, IOResult}
import akka.util.ByteString
import com.stacktrace.yo.scrapeline.core.ScrapeClient.jsoup
import com.stacktrace.yo.scrapeline.core._
import com.stacktrace.yo.scrapeline.imdb.Domain.MovieNameAndDetailUrl
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.model.Document
import net.ruippeixotog.scalascraper.scraper.ContentExtractors.elementList

import scala.concurrent.Future

class MovieListPipeline(implicit val m: ActorMaterializer) {

  def getPipelineSource: Source[jsoup.DocumentType, NotUsed] = Source.single(ScrapeClient.scrape("http://www.the-numbers.com/movie/budgets/all"))

  def getParseFlow: Flow[Document, MovieNameAndDetailUrl, NotUsed] = {
    Flow[Document]
      .mapConcat(doc => {
        val table = doc >> elementList("table tr")
        val movieLinkTuples = table.flatMap(tr => {
          val name = tr >> elementList("tr b a")
          name.map(
            link => {
              MovieNameAndDetailUrl(link.text, "http://www.the-numbers.com/" + link.attr("href"))
            }
          )
        })
        movieLinkTuples
      })
  }

  def getPipeOut: Sink[MovieNameAndDetailUrl, Future[IOResult]] = Flow[MovieNameAndDetailUrl]
    .map(s => ByteString(s.name + "\n"))
    .toMat(FileIO.toPath(Paths.get("movie.txt")))(Keep.right)

  def buildAndRun: Future[IOResult] = {
    getPipelineSource
      .via(getParseFlow)
      .runWith(getPipeOut)
  }

} 

//设置package包名称以及导入依赖的类
package com.ulasakdeniz.hakker.ws

import akka.http.scaladsl.model.{HttpEntity, HttpResponse}
import akka.http.scaladsl.unmarshalling.Unmarshaller
import akka.stream.scaladsl.Sink
import akka.util.ByteString

import scala.concurrent.Future
import scala.util.{Failure, Success, Try}

package object http {

  implicit final class HttpResponseUtil(responseF: Future[HttpResponse]) extends HttpClientApi {

    val entityData: Future[ByteString] =
      for {
        response <- responseF
        byteString <- {
          val source = response.entity.dataBytes
          source.runWith(Sink.head[ByteString])
        }
      } yield byteString

    def mapStrict[T](f: HttpResponse => T): Future[T] =
      for {
        response   <- responseF
        byteString <- entityData
        strictEntity = HttpEntity.Strict(response.entity.contentType, byteString)
      } yield f(response.withEntity(strictEntity))

    def entityAs[T](implicit unmarshaller: Unmarshaller[String, T]): Future[Try[T]] = {
      val result = for {
        byteString <- entityData
        t          <- unmarshaller(byteString.utf8String)
      } yield Success(t)
      result.recover {
        case t: Throwable => Failure(t)
      }
    }
  }
} 

//设置package包名称以及导入依赖的类
package com.karasiq.torrentstream

import akka.stream.stage.{Context, PushPullStage, SyncDirective, TerminationDirective}
import akka.util.ByteString
import com.karasiq.bittorrent.dispatcher.DownloadedPiece

final class TorrentStreamingStage(pieceLength: Int, private var ranges: Seq[TorrentFileOffset]) extends PushPullStage[DownloadedPiece, ByteString] {
  private var currentRange: TorrentFileOffset = ranges.head

  private var currentOffset: Long = currentRange.start

  private var buffer: Seq[DownloadedPiece] = Vector.empty

  override def onUpstreamFinish(ctx: Context[ByteString]): TerminationDirective = {
    if (buffer.nonEmpty) ctx.absorbTermination()
    else ctx.finish()
  }

  private def deliverBuffer(ctx: Context[ByteString]): SyncDirective = buffer match {
    case Seq(DownloadedPiece(index, data), rest @ _*) if (index.toLong * pieceLength) <= currentOffset ?
      val pieceOffset = (currentOffset - (index * pieceLength)).toInt
      val chunkLength = Array(data.length.toLong - pieceOffset, currentRange.end - currentOffset).min.toInt
      require(chunkLength > 0)
      buffer = rest
      currentOffset += chunkLength
      val chunk = data.slice(pieceOffset, pieceOffset + chunkLength)
      if (currentOffset >= currentRange.end) {
        if (ranges.tail.nonEmpty) {
          currentRange = ranges.tail.head
          ranges = ranges.tail
          currentOffset = currentRange.start
          ctx.push(chunk)
        } else {
          ctx.pushAndFinish(chunk)
        }
      } else {
        ctx.push(chunk)
      }

    case _ ?
      if (ctx.isFinishing) {
        ctx.finish()
      } else {
        ctx.pull()
      }
  }

  override def onPush(elem: DownloadedPiece, ctx: Context[ByteString]): SyncDirective = {
    buffer = (buffer :+ elem).sortBy(_.pieceIndex)
    deliverBuffer(ctx)
  }

  override def onPull(ctx: Context[ByteString]): SyncDirective = {
    deliverBuffer(ctx)
  }
} 

//设置package包名称以及导入依赖的类
package com.karasiq.bittorrent.format

import akka.util.ByteString

import scala.annotation.tailrec
import scala.collection.mutable.ArrayBuffer

case class TorrentPiece(index: Int, size: Int, sha1: ByteString, file: TorrentFile)
case class TorrentPieceBlock(piece: TorrentPiece, offset: Int, size: Int)

object TorrentPiece {
  def pieces(files: TorrentFiles): IndexedSeq[TorrentPiece] = {
    // Total torrent size
    val totalSize = files.files.map(_.size).sum

    @tailrec
    def pieceSequenceRec(buffer: ArrayBuffer[TorrentPiece], offset: Long, fileOffset: Long, pieceIndex: Int, fileSeq: Seq[TorrentFile]): IndexedSeq[TorrentPiece] = fileSeq match {
      case Seq(currentFile, fs @ _*) if fs.nonEmpty && fileOffset >= currentFile.size ?
        pieceSequenceRec(buffer, offset, 0L, pieceIndex, fs)

      case fs @ Seq(currentFile, _*) if offset < totalSize ?
        val length = Array(files.pieceLength.toLong, totalSize - offset).min
        require(length <= Int.MaxValue)
        val sha1 = files.pieces.slice(pieceIndex * 20, (pieceIndex * 20) + 20)
        val piece = TorrentPiece(buffer.length, length.toInt, sha1, currentFile)
        pieceSequenceRec(buffer :+ piece, offset + length, fileOffset + length, pieceIndex + 1, fs)

      case other ?
        buffer.result()
    }
    pieceSequenceRec(new ArrayBuffer[TorrentPiece](files.pieces.length / 20), 0L, 0L, 0, files.files)
  }

  def blocks(piece: TorrentPiece, sizeLimit: Int): IndexedSeq[TorrentPieceBlock] = {
    @tailrec
    def pieceBlockRec(buffer: ArrayBuffer[TorrentPieceBlock], offset: Int): IndexedSeq[TorrentPieceBlock] = {
      if (offset >= piece.size) {
        buffer.result()
      } else {
        val block = TorrentPieceBlock(piece, offset, Array(sizeLimit, piece.size - offset).min)
        pieceBlockRec(buffer :+ block, offset + block.size)
      }
    }
    pieceBlockRec(new ArrayBuffer[TorrentPieceBlock](piece.size / sizeLimit + 1), 0)
  }
} 

//设置package包名称以及导入依赖的类
package com.karasiq.bittorrent.protocol.extensions

import java.net.{InetAddress, InetSocketAddress}
import java.nio.ByteBuffer

import akka.util.ByteString
import com.karasiq.bittorrent.format.{BEncode, BEncodedDictionary, BEncodedString}
import com.karasiq.bittorrent.protocol.{BitTorrentTcpProtocol, TcpMessageProtocol}

trait PeerExchange extends PeerExchangeMessages with PeerExchangeTcp

trait PeerExchangeMessages {
  case class PeerExchangeList(addresses: Seq[InetSocketAddress])
}

trait PeerExchangeTcp { self: PeerExchangeMessages ?
  implicit object PeerExchangeListTcpProtocol extends TcpMessageProtocol[PeerExchangeList] {
    private val ipv4Length: Int = 4
    private val ipv6Length: Int = 16
    private val portLength: Int = 2

    override def toBytes(value: PeerExchangeList): ByteString = {
      val (ipv4, ipv6) = value.addresses.partition(_.getAddress.getAddress.length == ipv4Length)
      def packAddress(address: InetSocketAddress): ByteString = {
        val port = ByteBuffer.allocate(portLength)
        port.putShort(address.getPort.toShort)
        port.flip()
        ByteString(address.getAddress.getAddress) ++ ByteString(port)
      }
      BEncodedDictionary(Vector(
        "dropped" ? BEncodedString(ByteString.empty),
        "added" ? BEncodedString(ipv4.map(packAddress).fold(ByteString.empty)(_ ++ _)),
        "added.f" ? BEncodedString(ByteString(Array.fill(ipv4.length)(1.toByte))),
        "added6" ? BEncodedString(ipv6.map(packAddress).fold(ByteString.empty)(_ ++ _)),
        "added6.f" ? BEncodedString(ByteString(Array.fill(ipv6.length)(1.toByte)))
      )).toBytes
    }

    override def fromBytes(bs: ByteString): Option[PeerExchangeList] = {
      import com.karasiq.bittorrent.format.BEncodeImplicits._
      BEncode.parse(bs.toArray[Byte]).collectFirst {
        case BEncodedDictionary(values) ?
          val map = values.toMap
          val ipv4 = map.byteString("added").fold(Iterator[ByteString]())(_.grouped(ipv4Length + portLength))
          val ipv6 = map.byteString("added6").fold(Iterator[ByteString]())(_.grouped(ipv6Length + portLength))
          val addresses = (ipv4 ++ ipv6).map { bytes ?
            val address = InetAddress.getByAddress(bytes.dropRight(portLength).toArray)
            val port = BitTorrentTcpProtocol.int32FromBytes(bytes.takeRight(portLength))
            new InetSocketAddress(address, port)
          }
          PeerExchangeList(addresses.toVector)
      }
    }
  }
} 

//设置package包名称以及导入依赖的类
package akka.remote.transport.netty

import akka.actor.Address
import akka.remote.transport.AssociationHandle
import akka.remote.transport.AssociationHandle.{ HandleEventListener, InboundPayload }
import akka.remote.transport.Transport.AssociationEventListener
import akka.util.ByteString
import java.net.{ SocketAddress, InetAddress, InetSocketAddress }
import org.jboss.netty.buffer.{ ChannelBuffer, ChannelBuffers }
import org.jboss.netty.channel._
import scala.concurrent.{ Future, Promise }


private[remote] class UdpAssociationHandle(val localAddress: Address,
                                           val remoteAddress: Address,
                                           private val channel: Channel,
                                           private val transport: NettyTransport) extends AssociationHandle {

  override val readHandlerPromise: Promise[HandleEventListener] = Promise()

  override def write(payload: ByteString): Boolean = {
    if (!channel.isConnected)
      channel.connect(new InetSocketAddress(InetAddress.getByName(remoteAddress.host.get), remoteAddress.port.get))

    if (channel.isWritable && channel.isOpen) {
      channel.write(ChannelBuffers.wrappedBuffer(payload.asByteBuffer))
      true
    } else false
  }

  override def disassociate(): Unit = try channel.close()
  finally transport.udpConnectionTable.remove(transport.addressToSocketAddress(remoteAddress))

} 

//设置package包名称以及导入依赖的类
package com.sorrentocorp.akka.stream

import akka.NotUsed
import akka.stream._, scaladsl._, stage._
import akka.util.ByteString

import scala.util._, control.NonFatal

class Neo4jRespFraming extends GraphStage[FlowShape[ByteString, Neo4jRespToken]] {
  override protected def initialAttributes: Attributes = Attributes.name("Neo4jRespFraming.scanner")

  val in = Inlet[ByteString]("Neo4jRespFraming.in")
  val out = Outlet[Neo4jRespToken]("Neo4jRespFraming.out")
  override val shape = FlowShape(in, out)

  override def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) with InHandler with OutHandler {
    private val lexer = new Neo4jRespLexer

    setHandlers(in, out, this)

    override def onPush(): Unit = {
      val curr = grab(in)
      lexer.offer(curr)
      tryPopBuffer()
    }

    override def onPull(): Unit = {
      tryPopBuffer()
    }

    override def onUpstreamFinish(): Unit = {
      lexer.poll match {
        case Some(token) => emit(out, token)
        case None => completeStage()
      }
    }

    def tryPopBuffer(): Unit = {
      try lexer.poll match {
        case Some(token) => emit(out, token)
        case None => if (isClosed(in)) completeStage() else pull(in)
      } catch {
        case NonFatal(ex) => failStage(ex)
      }
    }
  }
} 

//设置package包名称以及导入依赖的类
package tutorial


import java.nio.file.Paths

import akka.actor.ActorSystem
import akka.stream._
import akka.stream.scaladsl._
import akka.util.ByteString
import akka.{Done, NotUsed}

import scala.concurrent._

object WaitForTwoFlowsToComplete {

  def main(args: Array[String]) = {

    implicit val system = ActorSystem("WaitForTwoFlowsToComplete")
    implicit val ec = system.dispatcher
    implicit val materializer = ActorMaterializer()

    val source: Source[Int, NotUsed] = Source(1 to 100)

    val f1Fut: Future[Done] = source.runForeach(i => println(i))

    //declaration of what happens when we scan (= transform) the source
    val factorials: Source[BigInt, NotUsed] = source.scan(BigInt(1))((acc, next) => acc * next)
    val f2fut: Future[IOResult] =
      factorials
        .map(num => ByteString(s"$num\n"))
        .runWith(FileIO.toPath(Paths.get("factorials.txt")))

    val aggFut = for {
      f1Result <- f1Fut
      f2Result <- f2fut
    } yield (f1Result, f2Result)

    aggFut.onComplete {  results =>
      println("Resulting futures from flows completed with results: " + results + " - about to terminate")
      system.terminate()
    }}
} 

//设置package包名称以及导入依赖的类
package tutorial

import java.nio.file.Paths

import akka.actor.ActorSystem
import akka.stream._
import akka.stream.scaladsl._
import akka.util.ByteString
import akka.{Done, NotUsed}

import scala.concurrent.duration._

import scala.concurrent._

object WaitForThreeFlowsToComplete {

  def main(args: Array[String]) = {

    implicit val system = ActorSystem("WaitForThreeFlowsToComplete")
    implicit val ec = system.dispatcher
    implicit val materializer = ActorMaterializer()

    def reusableLineSink(filename: String): Sink[String, Future[IOResult]] =
      Flow[String]
        .map(s => ByteString(s + "\n"))
        //Keep.right means: we want to retain what the FileIO.toPath sink has to offer
        .toMat(FileIO.toPath(Paths.get(filename)))(Keep.right)


    val source: Source[Int, NotUsed] = Source(1 to 100)

    val f1Fut: Future[Done] = source.runForeach(i => println(i))

    //declaration of what happens when we scan (= transform) the source
    val factorials = source.scan(BigInt(1))((acc, next) => acc * next)
    val f2fut: Future[IOResult] = factorials.map(_.toString).runWith(reusableLineSink("factorial2.txt"))

    val f3fut = factorials
      .zipWith(Source(0 to 10))((num, idx) => s"$idx! = $num")
      .throttle(1, 1.second, 1, ThrottleMode.shaping)
      .runWith(reusableLineSink("factorial3.txt"))

    val aggFut = for {
      f1Result <- f1Fut
      f2Result <- f2fut
      f3Result <- f3fut
    } yield (f1Result, f2Result, f3Result)

    aggFut.onComplete{  results =>
      println("Resulting Futures from Flows completed with results: " + results + " - about to terminate")
      system.terminate()
    }}
} 

//设置package包名称以及导入依赖的类
package edu.uw.at.iroberts.wirefugue.kafka.serdes

import java.time.Instant

import akka.util.ByteString
import edu.uw.at.iroberts.wirefugue.kafka.producer._
import edu.uw.at.iroberts.wirefugue.pcap.{Packet, PcapFileRaw}
import org.apache.kafka.common.serialization._


class PacketSerializer extends Serializer[Packet] with StatelessSerializer[Packet] {
  override def serialize(topic: String, p: Packet): Array[Byte] = {
    packetToProtobufPacket(p).toByteArray
  }
}

class PacketDeserializer extends Deserializer[Packet] with StatelessDeserializer[Packet] {
  override def deserialize(topic: String, data: Array[Byte]): Packet = {
    import edu.uw.at.iroberts.wirefugue.protobufs.packet.{Packet => ProtoPacket}
    val pp = ProtoPacket.parseFrom(data)
    Packet(
      timestamp = pp.sensorTsEpochNanos match {
        case Some(nanos) => Instant.ofEpochSecond(0, nanos)
        case _ => throw new RuntimeException("Packet has no timestamp")
      },
      network = PcapFileRaw.LinkType.ETHERNET, // TODO: don't assume link type
      originalLength = pp.data match {
        case Some(bytes) => bytes.size
        case _ => throw new RuntimeException("Packet has no data")
      },
      data = pp.data match {
        case Some(bytes) => ByteString(bytes.toByteArray)
        case _ => throw new RuntimeException("Packet has no data")
      }
    )
  }
}

class PacketSerde extends Serde[Packet] with StatelessSerde[Packet] {
  val protoPacketSerde = new protobuf.PacketSerde

  override def serializer = new PacketSerializer

  override def deserializer = new PacketDeserializer

} 

//设置package包名称以及导入依赖的类
package com.scalaio.http.client.actor

import akka.actor.{Actor, ActorLogging, ActorRef, Props}
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.HttpMethods._
import akka.http.scaladsl.model._
import akka.stream.{ActorMaterializer, ActorMaterializerSettings}
import akka.util.{ByteString, Timeout}
import play.api.libs.json.Json

import scala.concurrent.Future
import scala.concurrent.duration._

class HttpClientAsActor(notifier: ActorRef) extends Actor with ActorLogging {

  import akka.pattern.pipe
  import context.dispatcher
  implicit val timeout = Timeout(5 seconds)
  implicit val materializer = ActorMaterializer(ActorMaterializerSettings(context.system))

  val http = Http(context.system)

  override def preStart() = {
    http
      .singleRequest(HttpRequest(method = GET, uri = "https://jsonplaceholder.typicode.com/posts/1"))
      .pipeTo(self)
  }

  def receive = {
    case HttpResponse(StatusCodes.OK, headers, entity, _) =>
      val response: Future[ByteString] = entity.dataBytes.runFold(ByteString(""))(_ ++ _)
      log.info(s"got response $headers $entity")
      response pipeTo self
      context become handlingMessage

    case [email protected](code, _, _, _) =>
      log.warning("Request failed, response code: " + code)
      resp.discardEntityBytes()
  }

  def handlingMessage: Receive = {
    case content: ByteString =>
      log.info("Success was OK: " + content)
      val contentAsString = (Json.parse(content.utf8String) \ "title").as[String]
      notifier ! contentAsString
      context become receive
  }

}

object HttpClientAsActor {

  def props(notifier: ActorRef) = Props(classOf[HttpClientAsActor], notifier)

} 

//设置package包名称以及导入依赖的类
package com.scalaio.tcp.client

import akka.actor.{ Actor, ActorRef, Props }
import akka.io.Tcp
import akka.util.ByteString
import java.net.InetSocketAddress

object ClientActor {
  def props(remote: InetSocketAddress, tcp: ActorRef, replies: ActorRef) =
    Props(classOf[ClientActor], remote, tcp, replies)
}

class ClientActor(remote: InetSocketAddress, tcp:ActorRef, listener: ActorRef) extends Actor {
  import Tcp._
  import context.system

  tcp ! Connect(remote)

  def receive = {
    case CommandFailed(_: Connect) =>
      listener ! "connect failed"
      context stop self

    case c @ Connected(remote, local) =>
      listener ! c
      val connection = sender()
      connection ! Register(self)
      context become {
        case data: ByteString =>
          connection ! Write(data)
        case CommandFailed(w: Write) =>
          // O/S buffer was full
          listener ! "write failed"
        case Received(data) =>
          listener ! data
        case "close" =>
          connection ! Close
        case _: ConnectionClosed =>
          listener ! "connection closed"
          context stop self
      }
  }
} 

//设置package包名称以及导入依赖的类
package com.init6.coders.binary.packets

import akka.util.ByteString
import com.init6.coders.binary.BinaryPacket

import scala.util.Try


object SidCdKey2 extends BinaryPacket {

  override val PACKET_ID = Packets.SID_CDKEY2

  def apply(): ByteString = {
    build(
      ByteString.newBuilder
        .putInt(1)
        .putByte(0)
        .result()
    )
  }

  case class SidCdKey2()

  def unapply(data: ByteString): Option[SidCdKey2] = {
    Try {
      SidCdKey2()
    }.toOption
  }
} 

//设置package包名称以及导入依赖的类
package com.init6.coders.binary.packets

import akka.util.ByteString
import com.init6.coders.binary.{BinaryPacket, DeBuffer}

import scala.util.Try


object SidCreateAccount extends BinaryPacket {

  override val PACKET_ID = Packets.SID_CREATEACCOUNT

  val RESULT_FAILED = 0x00
  val RESULT_ACCOUNT_CREATED = 0x01

  def apply(result: Int): ByteString = {
    build(
      ByteString.newBuilder
        .putInt(result)
        .result()
    )
  }

  def unapply(data: ByteString): Option[SidCreateAccount] = {
    Try {
      val debuffer = DeBuffer(data)
      val passwordHash = debuffer.byteArray(20)
      val username = debuffer.string()
      SidCreateAccount(passwordHash, username)
    }.toOption
  }
}

case class SidCreateAccount(passwordHash: Array[Byte], username: String) 

//设置package包名称以及导入依赖的类
package com.init6.coders.binary.packets

import akka.util.ByteString
import com.init6.coders.binary.{BinaryPacket, DeBuffer}

import scala.util.Try


object SidAuthInfo extends BinaryPacket {

  override val PACKET_ID = Packets.SID_AUTH_INFO

  def apply(serverToken: Int, udpToken: Int = 0xDEADBEEF): ByteString = {
    build(
      ByteString.newBuilder
        .putInt(0)
        .putInt(serverToken)
        .putInt(udpToken)
        .putInt(0x4341AC00)
        .putInt(0x01C50B25)
        .putBytes("IX86ver3.mpq")
        .putBytes("A=125933019 B=665814511 C=736475113 4 A=A+S B=B^C C=C^A A=A^B")
        .result()
    )
  }

  case class SidAuthInfo(productId: String, versionByte: Byte)

  def unapply(data: ByteString): Option[SidAuthInfo] = {
    Try {
      val debuffer = DeBuffer(data)
      debuffer.skip(8)
      val productId = debuffer.byteArray(4)
      val verbyte = debuffer.byte(8)
      SidAuthInfo(new String(productId), verbyte)
    }.toOption
  }
}