Scala Normalizer类代码示例

//设置package包名称以及导入依赖的类
package io.soheila.cms.utils.slugs

import java.text.Normalizer
import java.time.LocalDateTime
import io.soheila.cms.utils.slugs.StringImplicits._

object SlugGenerator {
  def generate(): String = {
    generate(LocalDateTime.now().toString)
  }

  
  def generate(input: String): String = {
    var slug = generateInitialSlug(input)

    // Go lower case and transform whitespace
    slug = slug.trim.toLowerCase
    slug = RegexPatterns.WHITESPACE.matcher(slug).replaceAll("-")

    // Special chars
    slug = Normalizer.normalize(slug, Normalizer.Form.NFD)
    Mappings.SlUGSPECIALCHARACTER.foreach {
      case (key, value) => slug = slug.replaceAll(key, value)
    }

    // All other chars...
    slug = RegexPatterns.NONLATIN.matcher(slug).replaceAll("")

    // Remove extra dashes
    val isDash: (Char => Boolean) = _ == '-'
    slug.replaceAll("(-){2,}", "-").dropWhile(isDash).dropWhileInverse(isDash)
  }

  def generateUnique(name: String, similarSlugs: (String => List[String])): String = {
    val baseSlug = generate(name)

    var slug = baseSlug
    val existingSlugs = similarSlugs(baseSlug)

    var num = 0
    while (existingSlugs.contains(slug)) {
      num += 1
      slug = baseSlug + "-" + num
    }

    slug
  }

  private def generateInitialSlug(str: String): String = {
    if (str.isEmpty) LocalDateTime.now().toString
    else str
  }

} 

//设置package包名称以及导入依赖的类
package utils

import java.text.Normalizer
import java.util.regex.Pattern
import java.security.SecureRandom
import scala.util.Try

object StringUtils {

  def generateUuid(): String = {
    java.util.UUID.randomUUID().toString
  }

  def deAccent(str: String): String = {
    if (str == null || str.isEmpty) {
      ""
    } else {
      val nfdNormalizedString: String = Normalizer.normalize(str, Normalizer.Form.NFD);
      val pattern: Pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
      pattern.matcher(nfdNormalizedString).replaceAll("");
    }
  }

  
  //Random Generator
  private[this] val random = new SecureRandom()

  // Generate a random string of length n from the given alphabet
  private[this] def randomString(alphabet: String)(n: Int): String = {
    Stream.continually(random.nextInt(alphabet.size)).map(alphabet).take(n).mkString
  }

  // Generate a random alphabnumeric string of length n
  def randomAlphanumericString(n: Int): String = {
    randomString("abcdefghijklmnopqrstuvwxyz0123456789")(n)
  }
} 

//设置package包名称以及导入依赖的类
package scrape.model

import java.text.Normalizer

import utils.Zip

import scala.collection.GenIterable
import scala.xml._

case class TimeTablePage(title: Title, trains: GenIterable[Train])

object TimeTablePage {
  def fromXML(xml: NodeSeq) = {
    val columns = for {
      body <- xml \ "body"
      wrapper <- divIdFilter("wrapper")(body)
      container <- divIdFilter("container02")(wrapper)
      tables <- container \ "table" \ "tbody" \ "tr"
      tdw90 <- tdWidthFilter("90%")(tables)
      column <- tdw90 \ "table" \ "tbody" \ "tr" \ "td" \ "table" \ "tbody" \ "tr"
    } yield column
    require(columns.size >= 6, "Parse error")

    val title = Title.fromXML(columns(0))

    val numbers = ((columns(1) \\ "span").tail \ "span").map(_.text)
    val part1 = columns(2) \ "td"
    val types = spanClsFilter("s")(part1).map(_.text.replaceAll("\\[|\\]", ""))
    val names = spanClsFilter("m")(part1).map(_.text)
    val specials = ((columns(3) \\ "span").tail \ "span").map { it =>
      norm(it.text).trim.nonEmpty
    }
    val urls = (columns(4) \ "td" \ "span" \ "a").map { it => (it \ "@href").text }

    val part2 = columns(6) \ "td"
    val stations = (part2.head \ "span" \ "span").text.lines.map(_.trim).toList.tail.init
    val stoptypes = part2.tail.head.text.filter { !_.isWhitespace }.map { _.toString }
    val stoptimes = part2.tail.tail.map { it =>
      (it \ "span").map(_.text)
    }
    val createStops = CreateStops(stations, stoptypes, title.line)
    val stops = stoptimes.map(createStops.create)

    val trains = Zip.zip6(names, numbers, types, stops, urls, specials).map {
      case (name, num, typ, stop, url, sp) => {
        Train(name, num, typ, stop, normalizeURL(url), sp)
      }
    }
    TimeTablePage(title, trains)
  }

  private def norm(str: String) = Normalizer.normalize(str, Normalizer.Form.NFKC)
  private def filter(arg: String)(element: String)(text: String)(xs: NodeSeq) =
    xs \ arg filter (_ \ element contains Text(text))
  private def divIdFilter(id: String)(xs: NodeSeq) = filter("div")("@id")(id)(xs)
  private def tdWidthFilter(width: String)(xs: NodeSeq) =
    filter("td")("@width")(width)(xs)
  private def spanClsFilter(cls: String)(xs: NodeSeq) = filter("span")("@class")(cls)(xs)
  private def normalizeURL(url: String) = url.replace("../../", "/newdata/")
} 

//设置package包名称以及导入依赖的类
package rere.sasl.scram

import java.text.Normalizer

import rere.sasl.util.BinaryString

package object crypto {
  val CLIENT_KEY = "Client Key"
  val SERVER_KEY = "Server Key"

  // Naive implementation, but it's better than nothing
  def normalize(str: String): String = {
    Normalizer.normalize(str, Normalizer.Form.NFKC)
  }

  def xor(a: BinaryString, b: BinaryString): Either[String, BinaryString] = {
    if (a.length != b.length) {
      Left("Mismatch of keys length.")
    } else {
      val result = new Array[Byte](a.length)
      var i = result.length - 1
      while (i >= 0) {
        result(i) = (a(i) ^ b(i)).toByte
        i -= 1
      }
      Right(result)
    }
  }
} 

//设置package包名称以及导入依赖的类
package util

//https://gist.github.com/sam/5213151

object TextUtil {

  def slugify(input: String): String = {
    import java.text.Normalizer
    Normalizer.normalize(input, Normalizer.Form.NFD)
      .replaceAll("[^\\w\\s-]", "") // Remove all non-word, non-space or non-dash characters
      .replace('-', ' ')            // Replace dashes with spaces
      .trim                         // Trim leading/trailing whitespace (including what used to be leading/trailing dashes)
      .replaceAll("\\s+", "-")      // Replace whitespace (including newlines and repetitions) with single dashes
      .toLowerCase                  // Lowercase the final results
  }
} 

//设置package包名称以及导入依赖的类
package de.fuberlin.wiwiss.silk.plugins.transformer.linguistic

import de.fuberlin.wiwiss.silk.linkagerule.input.SimpleTransformer
import java.text.Normalizer
import java.util.regex.Pattern
import de.fuberlin.wiwiss.silk.runtime.plugin.Plugin


  private val DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+");

  private val NONDIACRITICS = Map(
    //replace non-diacritics as their equivalent chars
    ('\u0141' -> "l"), // BiaLystock
    ('\u0142' -> "l"), // Bialystock
    ('ß' -> "ss"),
    ('æ' -> "ae"),
    ('ø' -> "o"),
    ('©' -> "c"),
    ('\u00D0' -> "d"), // all Ð ð from http://de.wikipedia.org/wiki/%C3%90
    ('\u00F0' -> "d"),
    ('\u0110' -> "d"),
    ('\u0111' -> "d"),
    ('\u0189' -> "d"),
    ('\u0256' -> "d"),
    ('\u00DE' -> "th"), // thorn Þ
    ('\u00FE' -> "th")) // thorn þ


  def evaluate(value: String) = {
    simplifyString(value)
  }

  private def simplifyString(str: String): String = {
    if (str == null) {
      null
    } else {
      stripNonDiacritics(stripDiacritics(str))
    }
  }

  private def stripNonDiacritics(orig: String): String = {
    val ret = new StringBuffer()
    for (i <- 0 until orig.length()) {
      val source = orig.charAt(i)
      val replace = NONDIACRITICS.get(source).getOrElse(source)
      ret.append(replace)
    }
    ret.toString
  }

  private def stripDiacritics(str: String): String = {
    val normalized = Normalizer.normalize(str, Normalizer.Form.NFD)
    DIACRITICS_AND_FRIENDS.matcher(normalized).replaceAll("")
  }


} 

//设置package包名称以及导入依赖的类
package lila.common

import java.text.Normalizer
import java.util.regex.Matcher.quoteReplacement

object String {

  private val slugR = """[^\w-]""".r

  def slugify(input: String) = {
    val nowhitespace = input.trim.replace(" ", "-")
    val normalized = Normalizer.normalize(nowhitespace, Normalizer.Form.NFD)
    val slug = slugR.replaceAllIn(normalized, "")
    slug.toLowerCase
  }

  final class Delocalizer(netDomain: String) {

    private val regex = ("""\w{2}\.""" + quoteReplacement(netDomain)).r

    def apply(url: String) = regex.replaceAllIn(url, netDomain)
  }

  def shorten(text: String, length: Int, sep: String = "…") = {
    val t = text.replace("\n", " ")
    if (t.size > (length + sep.size)) (t take length) ++ sep
    else t
  }

  object base64 {
    import java.util.Base64
    import java.nio.charset.StandardCharsets
    def encode(txt: String) =
      Base64.getEncoder.encodeToString(txt getBytes StandardCharsets.UTF_8)
    def decode(txt: String): Option[String] = try {
      Some(new String(Base64.getDecoder decode txt))
    }
    catch {
      case _: java.lang.IllegalArgumentException => none
    }
  }
} 

//设置package包名称以及导入依赖的类
package parku30.mapslt

import java.text.Normalizer
import java.text.Normalizer.Form

object StringUtil {

  implicit class StringImprovements(val s: String) {
    implicit def sanitize:String = {
      def toCamel(s: String): String = {
        val split = s.split(" ")
        val tail = split.tail.map { x => x.head.toUpper + x.tail }
        split.head + tail.mkString
      }

      var sanitized = s.trim
      sanitized = sanitized.replace("regioninis parkas", "RP")
      sanitized = sanitized.replace("nacionalinis parkas", "NP")
      sanitized = sanitized.replace("gamtinis rezervatas", "RZV")
      sanitized = sanitized.replace(" r. sav.", "")
      sanitized = Normalizer.normalize(sanitized, Form.NFD).replaceAll("\\p{InCombiningDiacriticalMarks}+", "")
      sanitized = toCamel(sanitized)
      sanitized = sanitized.replaceAll("\\W", "")

      sanitized
    }
  }

} 

//设置package包名称以及导入依赖的类
package at.hazm.ml.nlp

import java.text.Normalizer

object Text {

  
  def normalize(text:String):String = {
    // ????????????????????? & ???????
    val reduceSpaces = text.foldLeft(new StringBuilder(text.length)) {
      case (buffer, ch) if Character.isWhitespace(ch) || Character.isISOControl(ch) =>
        if(buffer.nonEmpty && buffer.charAt(buffer.length - 1) != ' ') {
          buffer.append(' ')
        } else buffer
      case (buffer, ch) => buffer.append(ch)
    }
    // ???????
    if(reduceSpaces.nonEmpty && reduceSpaces.last == ' ') {
      reduceSpaces.setLength(reduceSpaces.length - 1)
    }
    // ??????? & ?????
    Normalizer.normalize(reduceSpaces, Normalizer.Form.NFKC).toUpperCase
  }
}