本文整理汇总了Golang中code/google/com/p/mahonia.NewDecoder函数的典型用法代码示例。如果您正苦于以下问题：Golang NewDecoder函数的具体用法？Golang NewDecoder怎么用？Golang NewDecoder使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了NewDecoder函数的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: ConvertToString
//转码
func ConvertToString(src string, srcCode string, tagCode string) string {
srcCoder := mahonia.NewDecoder(srcCode)
srcResult := srcCoder.ConvertString(src)
tagCoder := mahonia.NewDecoder(tagCode)
_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
result := string(cdata)
return result
}示例2: ProcessFile
// Function ProcessFile process the given file and incorporate the information
// into the NGramGenerator g for future N-Gram model generation.
func (g *NGramGenerator) ProcessFile(filename string) error {
var decoder mahonia.Decoder
if g.charset != "" {
decoder = mahonia.NewDecoder(g.charset)
}
lineProcessor := func(line string) (bool, error) {
line = strings.Trim(line, " \t\n\f\b\r")
if decoder != nil {
line = decoder.ConvertString(line)
}
tokens := strings.Split(line, " ")
var prevToken string
for i, t := range tokens {
//Monogram frequency
g.uniGram[t]++
g.uniGramCount++
//Bigram frequency
var key BiGramKey
if i == 0 {
key = BiGramKey{SentenceStartTag, t}
} else {
key = BiGramKey{prevToken, t}
}
fmt.Printf("%v\n", key)
g.biGram[key]++
g.biGramCount++
prevToken = t
}
return true, nil
}
return util.ForEachLineInFile(filename, lineProcessor)
}示例3: NewSegCNCorpus
func NewSegCNCorpus(source_charset string) *SegCNCorpus {
if source_charset != "" {
return &SegCNCorpus{nil, 0, mahonia.NewDecoder(source_charset)}
}
return &SegCNCorpus{nil, 0, nil}
}示例4: XmlCharsetReader
func XmlCharsetReader(charset string, input io.Reader) (io.Reader, error) {
decoder := mahonia.NewDecoder(charset)
if decoder == nil {
return nil, fmt.Errorf("Unknown charset %s", charset)
}
return decoder.NewReader(input), nil
}示例5: Grep
func (self *Grepper) Grep() {
for arg := range self.In {
fh, err := os.Open(arg.Path)
f := bufio.NewReader(fh)
if err != nil {
panic(err)
}
buf := make([]byte, 1024)
decoder := mahonia.NewDecoder(arg.Encode)
m := make([]*print.Match, 0)
var lineNum = 1
for {
buf, _, err = f.ReadLine()
if err != nil {
break
}
s := string(buf)
if decoder != nil && arg.Encode != file.UTF8 && arg.Encode != file.ASCII {
s = decoder.ConvertString(s)
}
if strings.Contains(s, arg.Pattern) {
m = append(m, &print.Match{lineNum, s})
}
lineNum++
}
self.Out <- &print.Params{arg.Pattern, arg.Path, m}
fh.Close()
}
close(self.Out)
}示例6: main
func main() {
content, err := ioutil.ReadFile("studygolang.xml")
if err != nil {
log.Fatal(err)
}
var result Result
err = xml.Unmarshal(content, &result)
if err != nil {
log.Fatal(err)
}
log.Println(result)
log.Println(result.Persons[0].Name)
//input := "<?xml version=\"1.0\" encoding=\"GBK\"?><request><head><h_exch_code>800101</h_exch_code><h_bank_no>1111</h_bank_no><h_user_id>1038738897</h_user_id><h_branch_id>B00008211</h_branch_id><h_fact_date>20110321</h_fact_date><h_fact_time>16:28:30</h_fact_time><h_exch_date>20130929</h_exch_date><h_serial_no>123456</h_serial_no><h_rsp_code>hj123545</h_rsp_code><h_rsp_msg>ar</h_rsp_msg></head><body><record><user_pwd>54f9b3396fe28c208d525db21588965c</user_pwd></record></body></request>"
//inputReader := strings.NewReader(input)
content, err = ioutil.ReadFile("request.xml")
decoder := mahonia.NewDecoder("gb18030")
r := bufio.NewReader(decoder.NewReader(content))
if err != nil {
log.Fatal(err)
}
log.Println(content)
var request Request_800101
err = xml.Unmarshal(content, &request)
if err != nil {
log.Fatal(err)
}
log.Println(request)
}示例7: UTF8Reader
func (ur *utf8reader) UTF8Reader(charset string, body io.Reader) (r io.Reader, err error) {
alias := strings.ToLower(charset)
newname, ok := charsetaliases[alias]
if ok {
charset = newname
}
r = mahonia.NewDecoder(charset).NewReader(body)
return
}示例8: convert
func convert(s string) string {
var dec mahonia.Decoder
dec = mahonia.NewDecoder("gbk")
if ret, ok := dec.ConvertStringOK(s); ok {
fmt.Println("GBK to UTF-8: ", ret)
return ret
}
return s
}示例9: getNormalizedFieldName
func (dt *DbfTable) getNormalizedFieldName(name string) (s string) {
e := mahonia.NewEncoder(dt.fileEncoding)
b := []byte(e.ConvertString(name))
if len(b) > 10 {
b = b[0:10]
}
d := mahonia.NewDecoder(dt.fileEncoding)
s = d.ConvertString(string(b))
return
}示例10: Find
func (this *QQwry) Find(ip string) {
if this.filepath == "" {
return
}
file, err := os.OpenFile(this.filepath, os.O_RDONLY, 0400)
defer file.Close()
if err != nil {
return
}
this.file = file
this.Ip = ip
offset := this.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
// log.Println("loc offset:", offset)
if offset <= 0 {
return
}
var country []byte
var area []byte
mode := this.readMode(offset + 4)
// log.Println("mode", mode)
if mode == REDIRECT_MODE_1 {
countryOffset := this.readUInt24()
mode = this.readMode(countryOffset)
// log.Println("1 - mode", mode)
if mode == REDIRECT_MODE_2 {
c := this.readUInt24()
country = this.readString(c)
countryOffset += 4
} else {
country = this.readString(countryOffset)
countryOffset += uint32(len(country) + 1)
}
area = this.readArea(countryOffset)
} else if mode == REDIRECT_MODE_2 {
countryOffset := this.readUInt24()
country = this.readString(countryOffset)
area = this.readArea(offset + 8)
} else {
country = this.readString(offset + 4)
area = this.readArea(offset + uint32(5+len(country)))
}
enc := mahonia.NewDecoder("gbk")
this.Country = enc.ConvertString(string(country))
this.City = enc.ConvertString(string(area))
}示例11: toUtf8
func toUtf8(content []byte) (error, string) {
detector := chardet.NewTextDetector()
result, err := detector.DetectBest(content)
if err != nil {
return err, ""
}
if result.Charset == "utf8" {
return nil, string(content)
}
decoder := mahonia.NewDecoder(result.Charset)
return nil, decoder.ConvertString(string(content))
}示例12: SelfPage
func SelfPage(cururl string) {
x, _ := goquery.NewDocument(cururl)
title := x.Find("title").Text()
if url, b := x.Find("#content a img").Attr("src"); b == true {
if alt, b := x.Find("#content a img").Attr("alt"); b == true {
title = alt
}
enc := mahonia.NewDecoder("gbk")
if _, e := AddSpiderData(url, enc.ConvertString(title)); e != nil {
fmt.Println(e)
}
}
}示例13: GetTotalPhysicalMemory
func GetTotalPhysicalMemory() (int64, error) {
d, err := exec.Command("cmd", "/c", "wmic memphysical get MaxCapacity").Output()
if err != nil {
return -1, err
}
dec := mahonia.NewDecoder("GBK")
s := dec.ConvertString(string(d))
r := strings.NewReader(s)
var tmp string
var n int64
fmt.Fscanln(r, &tmp)
fmt.Fscanln(r, &n)
return n, nil
}示例14: Test_quick
func Test_quick(t *testing.T) {
dbfHandle := DBFOpen("../map/bou2_4p.dbf", "rb")
defer DBFClose(dbfHandle)
decoder := mahonia.NewDecoder("gbk")
for i := 0; i < DBFGetRecordCount(dbfHandle); i++ {
for j := 0; j < DBFGetFieldCount(dbfHandle); j++ {
name, type_, _, _ := DBFGetFieldInfo(dbfHandle, j)
switch type_ {
case String:
p(name, decoder.ConvertString(string(DBFReadStringAttribute(dbfHandle, i, j))))
case Integer, Logical:
p(name, DBFReadIntegerAttribute(dbfHandle, i, j))
case Double:
p(name, DBFReadDoubleAttribute(dbfHandle, i, j))
}
}
}
}示例15: Login
func (ac *Account) Login() error {
var pld PreLoginData
err := pld.Load(ac.UserName)
if err != nil {
return err
}
client := &http.Client{}
data := `entry=weibo&gateway=1&from=&savestate=7&useticket=1&pagerefer=http%3A%2F%2Fweibo.com%2Fa%2Fdownload&vsnf=1&su=` + ac.UserName + `&service=miniblog&servertime=` + strconv.FormatInt(pld.Servertime, 10) + `&nonce=` + pld.Nonce + `&pwencode=rsa2&rsakv=` + pld.Rsakv + `&sp=` + pld.GetPwd(ac.Password) + `&encoding=UTF-8&prelt=415&url=http%3A%2F%2Fweibo.com%2Fajaxlogin.php%3Fframelogin%3D1%26callback%3Dparent.sinaSSOController.feedBackUrlCallBack&returntype=META`
fmt.Printf("form data: %v\n", data)
req, err := http.NewRequest("POST", SSO_LOGIN_URL+"?"+data, nil) // bytes.NewBuffer([]byte(data)))
if err != nil {
return err
}
SetHeader(req)
resp, err := client.Do(req)
if err != nil {
return err
}
fmt.Printf("resp: %v\n", resp)
defer resp.Body.Close()
fmt.Println("sso login cookie: ", resp.Header["Set-Cookie"])
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
enc := mahonia.NewDecoder("GBK")
bodyString := enc.ConvertString(string(body))
sIdx := strings.Index(bodyString, `replace("`)
if sIdx > 0 {
ac.loginUrl = bodyString[sIdx+9:]
sIdx = strings.Index(ac.loginUrl, `")`)
if sIdx > 0 {
ac.loginUrl = ac.loginUrl[:sIdx]
fmt.Println("loginUrl:", ac.loginUrl)
if strings.Contains(ac.loginUrl, "ticket") {
return ac.saveCookie()
}
}
}
return errors.New(fmt.Sprintf("fail to login, loginUrl: %s", ac.loginUrl))
}