Golang mime.ParseMediaType函数代码示例

func Fuzz(data []byte) int {
	sdata := string(data)
	mt, params, err := mime.ParseMediaType(sdata)
	if err != nil {
		return 0
	}
	sdata1 := mime.FormatMediaType(mt, params)
	mt1, params1, err := mime.ParseMediaType(sdata1)
	if err != nil {
		if err.Error() == "mime: no media type" {
			// https://github.com/golang/go/issues/11289
			return 0
		}
		if err.Error() == "mime: invalid media parameter" {
			// https://github.com/golang/go/issues/11290
			return 0
		}
		fmt.Printf("%q(%q, %+v) -> %q\n", sdata, mt, params, sdata1)
		panic(err)
	}
	if !fuzz.DeepEqual(mt, mt1) {
		fmt.Printf("%q -> %q\n", mt, mt1)
		panic("mediatype changed")
	}
	if !fuzz.DeepEqual(params, params1) {
		fmt.Printf("%+v -> %+v\n", params, params1)
		panic("params changed")
	}
	return 1
}

func init() {
	m = New()
	m.AddFunc("dummy/copy", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		io.Copy(w, r)
		return nil
	})
	m.AddFunc("dummy/nil", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		return nil
	})
	m.AddFunc("dummy/err", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		return errDummy
	})
	m.AddFunc("dummy/charset", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		_, param, _ := mime.ParseMediaType(mediatype)
		w.Write([]byte(param["charset"]))
		return nil
	})
	m.AddFunc("dummy/params", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		_, param, _ := mime.ParseMediaType(mediatype)
		return m.Minify(param["type"]+"/"+param["sub"], w, r)
	})
	m.AddFunc("type/sub", func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		w.Write([]byte("type/sub"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^type/.+$"), func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		w.Write([]byte("type/*"))
		return nil
	})
	m.AddFuncRegexp(regexp.MustCompile("^.+/.+$"), func(m Minifier, mediatype string, w io.Writer, r io.Reader) error {
		w.Write([]byte("*/*"))
		return nil
	})
}

// fixContentType checks resp's Content-Type header. If it is missing or too
// general, it sniffs the body content to find the content type, and updates
// the Content-Type header.
func fixContentType(resp *http.Response) {
	ct, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err != nil || !strings.Contains(ct, "/") {
		ct = ""
	}

	switch ct {
	case "unknown/unknown", "application/unknown", "*/*", "", "application/octet-stream":
		// These types tend to be used for content whose type is unknown,
		// so we should try to second-guess them.
		preview := make([]byte, 512)
		n, _ := resp.Body.Read(preview)
		preview = preview[:n]

		if n > 0 {
			ct, _, _ = mime.ParseMediaType(http.DetectContentType(preview))
			if ct != "application/octet-stream" {
				resp.Header.Set("Content-Type", ct)
			}

			// Make the preview data available for re-reading.
			var rc struct {
				io.Reader
				io.Closer
			}
			rc.Reader = io.MultiReader(bytes.NewBuffer(preview), resp.Body)
			rc.Closer = resp.Body
			resp.Body = rc
		}
	}
}

// parseMIMEParts will recursively walk a MIME entity and return a []mime.Part containing
// each (flattened) mime.Part found.
// It is important to note that there are no limits to the number of recursions, so be
// careful when parsing unknown MIME structures!
func parseMIMEParts(hs textproto.MIMEHeader, b io.Reader) ([]*part, error) {
	var ps []*part
	// If no content type is given, set it to the default
	if _, ok := hs["Content-Type"]; !ok {
		hs.Set("Content-Type", defaultContentType)
	}
	ct, params, err := mime.ParseMediaType(hs.Get("Content-Type"))
	if err != nil {
		return ps, err
	}
	// If it's a multipart email, recursively parse the parts
	if strings.HasPrefix(ct, "multipart/") {
		if _, ok := params["boundary"]; !ok {
			return ps, ErrMissingBoundary
		}
		mr := multipart.NewReader(b, params["boundary"])
		for {
			var buf bytes.Buffer
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				return ps, err
			}
			if _, ok := p.Header["Content-Type"]; !ok {
				p.Header.Set("Content-Type", defaultContentType)
			}
			subct, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
			if strings.HasPrefix(subct, "multipart/") {
				sps, err := parseMIMEParts(p.Header, p)
				if err != nil {
					return ps, err
				}
				ps = append(ps, sps...)
			} else {
				// Otherwise, just append the part to the list
				// Copy the part data into the buffer
				if _, err := io.Copy(&buf, p); err != nil {
					return ps, err
				}
				ps = append(ps, &part{body: buf.Bytes(), header: p.Header})
			}
		}
	} else {
		// If it is not a multipart email, parse the body content as a single "part"
		var buf bytes.Buffer
		if _, err := io.Copy(&buf, b); err != nil {
			return ps, err
		}
		ps = append(ps, &part{body: buf.Bytes(), header: hs})
	}
	return ps, nil
}

// GetBodyStr returns plain string crawled.
func (self *Context) initText() {
	// 采用surf内核下载时，尝试自动转码
	if self.Request.DownloaderID == request.SURF_ID {
		var contentType, pageEncode string
		// 优先从响应头读取编码类型
		contentType = self.Response.Header.Get("Content-Type")
		if _, params, err := mime.ParseMediaType(contentType); err == nil {
			if cs, ok := params["charset"]; ok {
				pageEncode = strings.ToLower(strings.TrimSpace(cs))
			}
		}
		// 响应头未指定编码类型时，从请求头读取
		if len(pageEncode) == 0 {
			contentType = self.Request.Header.Get("Content-Type")
			if _, params, err := mime.ParseMediaType(contentType); err == nil {
				if cs, ok := params["charset"]; ok {
					pageEncode = strings.ToLower(strings.TrimSpace(cs))
				}
			}
		}

		switch pageEncode {
		// 不做转码处理
		case "", "utf8", "utf-8", "unicode-1-1-utf-8":
		default:
			// 指定了编码类型，但不是utf8时，自动转码为utf8
			// get converter to utf-8
			// Charset auto determine. Use golang.org/x/net/html/charset. Get response body and change it to utf-8
			destReader, err := charset.NewReaderLabel(pageEncode, self.Response.Body)
			if err == nil {
				self.text, err = ioutil.ReadAll(destReader)
				if err == nil {
					self.Response.Body.Close()
					return
				} else {
					logs.Log.Warning(" *     [convert][%v]: %v (ignore transcoding)\n", self.GetUrl(), err)
				}
			} else {
				logs.Log.Warning(" *     [convert][%v]: %v (ignore transcoding)\n", self.GetUrl(), err)
			}
		}
	}

	// 不做转码处理
	var err error
	self.text, err = ioutil.ReadAll(self.Response.Body)
	self.Response.Body.Close()
	if err != nil {
		panic(err.Error())
		return
	}

}

// binMIME handles the special case where the only content of the message is an
// attachment.  It is called by ParseMIME when needed.
func binMIME(mailMsg *mail.Message) (*MIMEBody, error) {
	// Determine mediatype
	ctype := mailMsg.Header.Get("Content-Type")
	mediatype, mparams, err := mime.ParseMediaType(ctype)
	if err != nil {
		mediatype = "attachment"
	}

	// Build the MIME part representing most of this message
	p := NewMIMEPart(nil, mediatype)
	content, err := decodeSection(mailMsg.Header.Get("Content-Transfer-Encoding"), mailMsg.Body)
	if err != nil {
		return nil, err
	}
	p.SetContent(content)
	p.SetHeader(make(textproto.MIMEHeader, 4))

	// Determine and set headers for: content disposition, filename and
	// character set
	disposition, dparams, err := mime.ParseMediaType(mailMsg.Header.Get("Content-Disposition"))
	if err == nil {
		// Disposition is optional
		p.SetDisposition(disposition)
		p.SetFileName(DecodeHeader(dparams["filename"]))
	}
	if p.FileName() == "" && mparams["name"] != "" {
		p.SetFileName(DecodeHeader(mparams["name"]))
	}
	if p.FileName() == "" && mparams["file"] != "" {
		p.SetFileName(DecodeHeader(mparams["file"]))
	}
	if p.Charset() == "" {
		p.SetCharset(mparams["charset"])
	}

	p.Header().Set("Content-Type", mailMsg.Header.Get("Content-Type"))
	p.Header().Set("Content-Disposition", mailMsg.Header.Get("Content-Disposition"))

	// Add our part to the appropriate section of MIMEBody
	m := &MIMEBody{
		header:         mailMsg.Header,
		Root:           NewMIMEPart(nil, mediatype),
		IsTextFromHTML: false,
	}

	if disposition == "inline" {
		m.Inlines = append(m.Inlines, p)
	} else {
		m.Attachments = append(m.Attachments, p)
	}

	return m, err
}

func (msg Jmessage) DecBody() (mailbody []byte, err error) {
	contentType := msg.Header.Get("Content-Type")
	if contentType == "" {
		return readPlainText(map[string][]string(msg.Header), msg.Body)
	}
	mediaType, params, err := mime.ParseMediaType(contentType)
	debug.Printf("MediaType: %s, %v\n", mediaType, params)
	if err != nil {
		debug.Printf("Error: %v", err)
		return
	}
	mailbody = make([]byte, 0)
	if strings.HasPrefix(mediaType, MEDIATYPE_MULTI) {
		// multipart/...
		mr := multipart.NewReader(msg.Body, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				return mailbody, err
			}
			if err != nil {
				debug.Printf("Error: %v", err)
			}
			mt, _, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
			if err != nil {
				debug.Printf("Error: %v", err)
				return nil, err
			}
			debug.Printf("MediaType-inner: %s\n", mt)
			if strings.HasPrefix(mt, MEDIATYPE_TEXT) {
				// text/plain
				return readPlainText(p.Header, p)
			}
			if strings.HasPrefix(mt, MEDIATYPE_MULTI_ALT) {
				// multipart/alternative
				return readAlternative(p)
			}
			// slurp, err := ioutil.ReadAll(p)
			// if err != nil {
			//   debug.Printf("Error: %v", err)
			// }
			// for key, values := range p.Header {
			//   debug.Printf("%s:%v", key, values)
			// }
			// fmt.Printf("Slurping...: %q\n", slurp)
		}
	} else {
		// text/plain, text/html
		return readPlainText(map[string][]string(msg.Header), msg.Body)
	}
	return
}

// IsAttachment returns true, if the given header defines an attachment.
// First it checks, if the Content-Disposition header defines an attachement.
// If this test is false, the Content-Type header is checked.
//
// Valid Attachment-Headers:
//
//    Content-Disposition: attachment; filename="frog.jpg"
//    Content-Type: attachment; filename="frog.jpg"
//
func IsAttachment(header mail.Header) bool {
	mediatype, _, _ := mime.ParseMediaType(header.Get("Content-Disposition"))
	if strings.ToLower(mediatype) == "attachment" {
		return true
	}

	mediatype, _, _ = mime.ParseMediaType(header.Get("Content-Type"))
	if strings.ToLower(mediatype) == "attachment" {
		return true
	}

	return false
}

// parseBody will accept a a raw body, break it into all its parts and then convert the
// message to UTF-8 from whatever charset it may have.
func parseBody(header mail.Header, body []byte) (html []byte, text []byte, isMultipart bool, err error) {
	var mediaType string
	var params map[string]string
	mediaType, params, err = mime.ParseMediaType(header.Get("Content-Type"))
	if err != nil {
		return
	}

	if strings.HasPrefix(mediaType, "multipart/") {
		isMultipart = true
		mr := multipart.NewReader(bytes.NewReader(body), params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			}
			if err != nil {
				break
			}

			slurp, err := ioutil.ReadAll(p)
			if err != nil {
				break
			}

			partMediaType, partParams, err := mime.ParseMediaType(p.Header.Get("Content-Type"))
			if err != nil {
				break
			}

			var htmlT, textT []byte
			htmlT, textT, err = parsePart(partMediaType, partParams["charset"], p.Header.Get("Content-Transfer-Encoding"), slurp)
			if len(htmlT) > 0 {
				html = htmlT
			} else {
				text = textT
			}
		}
	} else {

		splitBody := bytes.SplitN(body, headerSplitter, 2)
		if len(splitBody) < 2 {
			err = errors.New("unexpected email format. (single part and no \\r\\n\\r\\n separating headers/body")
			return
		}

		body = splitBody[1]
		html, text, err = parsePart(mediaType, params["charset"], header.Get("Content-Transfer-Encoding"), body)
	}
	return
}

func (m *meta) parseContentType(ct string) error {
	if ct == "" {
		m.MediaType = "application/octet-stream"
		return nil
	}

	mediatype, params, err := mime.ParseMediaType(ct)
	if err != nil {
		return err
	}

	if mediatype == "multipart/form-data" {
		boundary, ok := params["boundary"]
		if !ok {
			return errors.New("meta: boundary not defined")
		}

		m.MediaType = mediatype
		m.Boundary = boundary
	} else {
		m.MediaType = "application/octet-stream"
	}

	return nil
}

// GetContentType returns content-type of the message
func (m *Email) GetContentType() (contentType string, params map[string]string, err error) {
	hdrCt, err := m.GetHeader("Content-Type")
	if err != nil {
		return
	}
	return mime.ParseMediaType(hdrCt)
}

// also from goiardi calc and encodebody data
func calcBodyHash(r *http.Request) (string, error) {
	var bodyStr string
	var err error
	if r.Body == nil {
		bodyStr = ""
	} else {
		save := r.Body
		save, r.Body, err = drainBody(r.Body)
		if err != nil {
			return "", err
		}
		mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
		if err != nil {
			return "", err
		}
		if strings.HasPrefix(mediaType, "multipart/form-data") {
			bodyStr, err = readFileFromRequest(r, params["boundary"])
			if err != nil {
				return "", err
			}
		} else {
			buf := new(bytes.Buffer)
			buf.ReadFrom(r.Body)
			bodyStr = buf.String()
		}
		r.Body = save
	}
	chkHash := hashStr(bodyStr)
	return chkHash, err
}

func matchesContentType(contentType, expectedType string) bool {
	mimetype, _, err := mime.ParseMediaType(contentType)
	if err != nil {
		utils.Errorf("Error parsing media type: %s error: %s", contentType, err.Error())
	}
	return err == nil && mimetype == expectedType
}

func (t *Transport) updateToken(tok *Token, v url.Values) error {
	v.Set("client_id", t.ClientId)
	v.Set("client_secret", t.ClientSecret)
	r, err := (&http.Client{Transport: t.transport()}).PostForm(t.TokenURL, v)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if r.StatusCode != 200 {
		return OAuthError{"updateToken", r.Status}
	}
	var b struct {
		Access    string        `json:"access_token"`
		Refresh   string        `json:"refresh_token"`
		ExpiresIn time.Duration `json:"expires_in"`
		Id        string        `json:"id_token"`
	}

	content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
	switch content {
	case "application/x-www-form-urlencoded", "text/plain":
		body, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return err
		}
		vals, err := url.ParseQuery(string(body))
		if err != nil {
			return err
		}

		b.Access = vals.Get("access_token")
		b.Refresh = vals.Get("refresh_token")
		b.ExpiresIn, _ = time.ParseDuration(vals.Get("expires_in") + "s")
		b.Id = vals.Get("id_token")
	default:
		if err = json.NewDecoder(r.Body).Decode(&b); err != nil {
			return err
		}
		// The JSON parser treats the unitless ExpiresIn like 'ns' instead of 's' as above,
		// so compensate here.
		b.ExpiresIn *= time.Second
	}
	tok.AccessToken = b.Access
	// Don't overwrite `RefreshToken` with an empty value
	if len(b.Refresh) > 0 {
		tok.RefreshToken = b.Refresh
	}
	if b.ExpiresIn == 0 {
		tok.Expiry = time.Time{}
	} else {
		tok.Expiry = time.Now().Add(b.ExpiresIn)
	}
	if b.Id != "" {
		if tok.Extra == nil {
			tok.Extra = make(map[string]string)
		}
		tok.Extra["id_token"] = b.Id
	}
	return nil
}

func getEncoding(resp *gohttp.Response, body []byte) int {
	charsetTable := map[string]int{
		"gb18030":   CP_GBK,
		"gb2312":    CP_GBK,
		"hz":        CP_GBK,
		"big5":      CP_BIG5,
		"shift_jis": CP_SHIFT_JIS,
		"euc-jp":    CP_SHIFT_JIS,
		"utf-8":     CP_UTF8,
	}

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) == 0 {
		return CP_UTF8
	}

	_, params, err := mime.ParseMediaType(ctype)
	if err != nil {
		return CP_UTF8
	}

	charset, ok := params["charset"]
	if ok == false {
		return CP_UTF8
	}

	encoding, ok := charsetTable[string(String(charset).ToLower())]
	if ok == false {
		encoding = CP_UTF8
	}

	return encoding
}