Python urllib.urlparse函数代码示例

 def _isswe(self, url):
     if re.search(".se$", urlparse(url).netloc):
         return "sasong"
     elif re.search(".dk$", urlparse(url).netloc):
         return "saeson"
     else:
         return "sesong"

    def get(self, options):
        match = re.search(r'data-videoid="([^"]+)"', self.get_urldata())
        if not match:
            parse = urlparse(self.url)
            match = re.search(r'video/(\d+)/', parse.fragment)
            if not match:
                log.error("Can't find video id")
                sys.exit(2)
        videoid = match.group(1)
        data = get_http_data("http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website" % videoid)
        jsondata = json.loads(data)

        if options.output_auto:
            directory = os.path.dirname(options.output)
            title = "%s" % jsondata["title"]
            title = filenamify(title)
            if len(directory):
                options.output = "%s/%s" % (directory, title)
            else:
                options.output = title
        if "hds" in jsondata["streamUrls"]:
            parse = urlparse(jsondata["streamUrls"]["hds"])
            manifest = "%s://%s%s?%s&hdcore=3.3.0" % (parse.scheme, parse.netloc, parse.path, parse.query)
            streams = hdsparse(copy.copy(options), manifest)
            if streams:
                for n in list(streams.keys()):
                    yield streams[n]
        if "hls" in jsondata["streamUrls"]:
            streams = hlsparse(jsondata["streamUrls"]["hls"])
            for n in list(streams.keys()):
                yield HLS(copy.copy(options), streams[n], n)
        if "mp4" in jsondata["streamUrls"]:
            yield HTTP(copy.copy(options), jsondata["streamUrls"]["mp4"])

    def get(self, options):
        if re.findall("svt.se", self.url):
            match = re.search(r"data-json-href=\"(.*)\"", self.get_urldata())
            if match:
                filename = match.group(1).replace("&", "&").replace("&format=json", "")
                url = "http://www.svt.se%s" % filename
            else:
                log.error("Can't find video file")
                sys.exit(2)
        else:
            url = self.url

        pos = url.find("?")
        if pos < 0:
            dataurl = "%s?&output=json&format=json" % url
        else:
            dataurl = "%s&output=json&format=json" % url
        data = json.loads(get_http_data(dataurl))
        if "live" in data["video"]:
            options.live = data["video"]["live"]
        else:
            options.live = False

        if data["video"]["subtitleReferences"]:
            try:
                subtitle = data["video"]["subtitleReferences"][0]["url"]
            except KeyError:
                pass
            if len(subtitle) > 0:
                yield subtitle_wsrt(subtitle)

        for i in data["video"]["videoReferences"]:
            parse = urlparse(i["url"])

            if parse.path.find("m3u8") > 0:
                streams = hlsparse(i["url"])
                for n in list(streams.keys()):
                    yield HLS(copy.copy(options), streams[n], n)
            elif parse.path.find("f4m") > 0:
                match = re.search(r"\/se\/secure\/", i["url"])
                if not match:
                    parse = urlparse(i["url"])
                    manifest = "%s://%s%s?%s&hdcore=3.3.0" % (parse.scheme, parse.netloc, parse.path, parse.query)
                    streams = hdsparse(copy.copy(options), manifest)
                    for n in list(streams.keys()):
                        yield streams[n]
            elif parse.scheme == "rtmp":
                embedurl = "%s?type=embed" % url
                data = get_http_data(embedurl)
                match = re.search(r"value=\"(/(public)?(statiskt)?/swf(/video)?/svtplayer-[0-9\.a-f]+swf)\"", data)
                swf = "http://www.svtplay.se%s" % match.group(1)
                options.other = "-W %s" % swf
                yield RTMP(copy.copy(options), i["url"], i["bitrate"])
            else:
                yield HTTP(copy.copy(options), i["url"], "0")

    def get(self):
        if self.exclude():
            yield ServiceError("Excluding video")
            return

        match = re.search('iframe src="(/embed/[^"]+)"', self.get_urldata())
        if not match:
            yield ServiceError("Cant find video")
            return
        parse = urlparse(self.url)

        url = "{0}://{1}{2}".format(parse.scheme, parse.netloc, match.group(1))

        data = self.http.get(url)

        match = re.search('src="([^"]+vtt)"', data.text)
        if match:
            yield subtitle(copy.copy(self.options), "wrst", match.group(1))

        match = re.search('source src="([^"]+)" type="application/x-mpegURL"', data.text)
        if not match:
            yield ServiceError("Cant find video file")
            return

        streams = hlsparse(self.options, self.http.request("get", match.group(1)), match.group(1))
        for n in list(streams.keys()):
            yield streams[n]

 def get_mediaid(self):
     match = re.search(r"mediaId = '([^']+)';", self.get_urldata())
     if not match:
         match = re.search(r'media-id="([^"]+)"', self.get_urldata())
     if not match:
         match = re.search(r'screen9-mid="([^"]+)"', self.get_urldata())
     if not match:
         match = re.search(r'data-id="([^"]+)"', self.get_urldata())
     if not match:
         match = re.search(r'data-videoid="([^"]+)"', self.get_urldata())
     if not match:
         match = re.search('s.src="(https://csp-ssl.picsearch.com[^"]+|http://csp.picsearch.com/rest[^"]+)', self.get_urldata())
         if match:
             data = self.http.request("get", match.group(1))
             match = re.search(r'mediaid": "([^"]+)"', data.text)
         if not match:
             match = re.search('iframe src="(//csp.screen9.com[^"]+)"', self.get_urldata())
             if match:
                 url = "http:%s" % match.group(1)
                 data = self.http.request("get", url)
                 match = re.search(r"mediaid: '([^']+)'", data.text)
     if not match:
         urlp = urlparse(self.url)
         match = urlp.fragment
     return match

    def get(self, options):
        error, data = self.get_urldata()
        if error:
            log.error("Can't download webpage")
            return

        if self.exclude(options):
            return

        parse = urlparse(self.url)
        vidoid = parse.path[parse.path.rfind("/")+1:]
        match = re.search(r'JSONdata = ({.*});', data)
        if not match:
            log.error("Cant find json data")
            return
        janson = json.loads(match.group(1))
        playlist = janson["playlist"]
        for i in playlist:
            if i["brightcoveId"] == vidoid:
                if i["HLSURL"]:
                    streams = hlsparse(i["HLSURL"])
                    for n in list(streams.keys()):
                        yield HLS(copy.copy(options), streams[n], n)
                for n in i["renditions"]:
                    if n["container"] == "MP4":
                        yield HTTP(copy.copy(options), n["URL"], int(n["rate"])/1000)

    def get(self, options):
        parse = urlparse(self.url)
        try:
            other = parse.fragment
        except KeyError:
            log.error("Something wrong with that url")
            sys.exit(2)
        match = re.search("^/(.*).html", other)
        if not match:
            log.error("Cant find video file")
            sys.exit(2)
        url = "http://www.hbo.com/data/content/%s.xml" % match.group(1)
        data = get_http_data(url)
        xml = ET.XML(data)
        videoid = xml.find("content")[1].find("videoId").text
        url = "http://render.cdn.hbo.com/data/content/global/videos/data/%s.xml" % videoid
        data = get_http_data(url)
        xml = ET.XML(data)
        ss = xml.find("videos")
        if is_py2_old:
            sa = list(ss.getiterator("size"))
        else:
            sa = list(ss.iter("size"))

        for i in sa:
            videourl = i.find("tv14").find("path").text
            match = re.search("/([a-z0-9]+:[a-z0-9]+)/", videourl)
            options.other = "-y %s" % videourl[videourl.index(match.group(1)):]
            yield RTMP(copy.copy(options), videourl[:videourl.index(match.group(1))], i.attrib["width"])

def hdsparse(options, data, manifest):
    streams = {}
    bootstrap = {}
    xml = ET.XML(data)

    if is_py2_old:
        bootstrapIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}media")
    else:
        bootstrapIter = xml.iter("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.iter("{http://ns.adobe.com/f4m/1.0}media")

    if xml.find("{http://ns.adobe.com/f4m/1.0}drmAdditionalHeader") is not None:
        log.error("HDS DRM protected content.")
        return
    for i in bootstrapIter:
        if "id" in i.attrib:
            bootstrap[i.attrib["id"]] = i.text
        else:
            bootstrap["0"] = i.text
    parse = urlparse(manifest)
    querystring = parse.query
    for i in mediaIter:
        if len(bootstrap) == 1:
            bootstrapid = bootstrap["0"]
        else:
            bootstrapid = bootstrap[i.attrib["bootstrapInfoId"]]
        streams[int(i.attrib["bitrate"])] = HDS(options, i.attrib["url"], i.attrib["bitrate"], manifest=manifest, bootstrap=bootstrapid,
                                                metadata=i.find("{http://ns.adobe.com/f4m/1.0}metadata").text, querystring=querystring)
    return streams

    def find_all_episodes(self, options):
        episodes = []
        match = re.search('"ContentPageProgramStore":({.*}),"ApplicationStore', self.get_urldata())
        if match:
            janson = json.loads(match.group(1))
            season = re.search("sasong-(\d+)", urlparse(self.url).path)
            if season:
                season = season.group(1)
            seasons = []
            for i in janson["format"]["seasons"]:
                if season:
                    if int(season) == i["seasonNumber"]:
                        seasons.append(i["seasonNumber"])
                else:
                    seasons.append(i["seasonNumber"])

            for i in seasons:
                if "program" in janson["format"]["videos"][str(i)]:
                    for n in janson["format"]["videos"][str(i)]["program"]:
                        episodes = self._videos_to_list(n["sharingUrl"],n["id"],episodes)
                if self.options.include_clips:
                    if "clip" in janson["format"]["videos"][str(i)]:
                        for n in janson["format"]["videos"][str(i)]["clip"]:
                            episodes = self._videos_to_list(n["sharingUrl"],n["id"],episodes)

        if options.all_last > 0:
            return sorted(episodes[-options.all_last:])
        return sorted(episodes)

    def get(self, options):
        data = self.get_urldata()
        match = re.search('data-subtitlesurl = "(/.*)"', data)
        if match:
            parse = urlparse(self.url)
            subtitle = "%s://%s%s" % (parse.scheme, parse.netloc, match.group(1))
            yield subtitle_tt(subtitle)
        match = re.search(r'data-media="(.*manifest.f4m)"', data)
        if match:
            manifest_url = match.group(1)
        else:
            match = re.search(r'data-video-id="(\d+)"', data)
            if match is None:
                log.error("Can't find video id.")
                sys.exit(2)
            vid = match.group(1)
            match = re.search(r"PS_VIDEO_API_URL : '([^']*)',", data)
            if match is None:
                log.error("Can't find server address with media info")
                sys.exit(2)
            dataurl = "%smediaelement/%s" % (match.group(1), vid)
            data = json.loads(get_http_data(dataurl))
            manifest_url = data["mediaUrl"]
            options.live = data["isLive"]

        hlsurl = manifest_url.replace("/z/", "/i/").replace("manifest.f4m", "master.m3u8")
        streams = hlsparse(hlsurl)
        for n in list(streams.keys()):
            yield HLS(copy.copy(options), streams[n], n)

        manifest_url = "%s?hdcore=2.8.0&g=hejsan" % manifest_url
        streams = hdsparse(copy.copy(options), manifest_url)
        for n in list(streams.keys()):
            yield streams[n]

    def find_all_episodes(self, options):
        parse = urlparse(self.url)
        show = parse.path[parse.path.find("/", 1)+1:]
        if not re.search("%", show):
            show = quote_plus(show)
        error, data = get_http_data("http://webapi.tv4play.se/play/video_assets?type=episode&is_live=false&platform=web&node_nids=%s&per_page=99999" % show)
        if error:
            log.error("Can't get api page")
            return
        jsondata = json.loads(data)
        episodes = []
        n = 1
        for i in jsondata["results"]:
            try:
                days = int(i["availability"]["availability_group_free"])
            except (ValueError, TypeError):
                days = 999
            if days > 0:
                video_id = i["id"]
                url = "http://www.tv4play.se/program/%s?video_id=%s" % (
                    show, video_id)
                episodes.append(url)
                if n == options.all_last:
                    break
                n += 1

        return episodes

    def get(self):
        parse = urlparse(self.url)
        try:
            other = parse.fragment
        except KeyError:
            log.error("Something wrong with that url")
            return

        if self.exclude():
            yield ServiceError("Excluding video")
            return

        match = re.search("^/(.*).html", other)
        if not match:
            log.error("Cant find video file")
            return
        url = "http://www.hbo.com/data/content/{0}.xml".format(match.group(1))
        data = self.http.request("get", url).content
        xml = ET.XML(data)
        videoid = xml.find("content")[1].find("videoId").text
        url = "http://render.cdn.hbo.com/data/content/global/videos/data/{0}.xml".format(videoid)
        data = self.http.request("get", url).content
        xml = ET.XML(data)
        ss = xml.find("videos")
        if is_py2_old:
            sa = list(ss.getiterator("size"))
        else:
            sa = list(ss.iter("size"))

        for i in sa:
            videourl = i.find("tv14").find("path").text
            match = re.search("/([a-z0-9]+:[a-z0-9]+)/", videourl)
            self.options.other = "-y {0}".format(videourl[videourl.index(match.group(1)):])
            yield RTMP(copy.copy(self.options), videourl[:videourl.index(match.group(1))], i.attrib["width"])

def hdsparse(options, res, manifest):
    streams = {}
    bootstrap = {}
    if res.status_code == 403:
        streams[0] = ServiceError("Can't read HDS playlist. permission denied")
        return streams
    xml = ET.XML(res.text)

    if is_py2_old:
        bootstrapIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}media")
    else:
        bootstrapIter = xml.iter("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.iter("{http://ns.adobe.com/f4m/1.0}media")

    if xml.find("{http://ns.adobe.com/f4m/1.0}drmAdditionalHeader") is not None:
        streams[0] = ServiceError("HDS DRM protected content.")
        return streams
    for i in bootstrapIter:
        if "id" in i.attrib:
            bootstrap[i.attrib["id"]] = i.text
        else:
            bootstrap["0"] = i.text
    parse = urlparse(manifest)
    querystring = parse.query
    manifest = "%s://%s%s" % (parse.scheme, parse.netloc, parse.path)
    for i in mediaIter:
        if len(bootstrap) == 1:
            bootstrapid = bootstrap["0"]
        else:
            bootstrapid = bootstrap[i.attrib["bootstrapInfoId"]]
        streams[int(i.attrib["bitrate"])] = HDS(copy.copy(options), i.attrib["url"], i.attrib["bitrate"], manifest=manifest, bootstrap=bootstrapid,
                                                metadata=i.find("{http://ns.adobe.com/f4m/1.0}metadata").text, querystring=querystring, cookies=res.cookies)
    return streams

def findvid(url, data):
    parse = urlparse(url)
    if "tv4play.se" in url:
        try:
            vid = parse_qs(parse.query)["video_id"][0]
        except KeyError:
            return None
    else:
        match = re.search(r"\"vid\":\"(\d+)\",", data)
        if match:
            vid = match.group(1)
        else:
            match = re.search(r"-(\d+)$", url)
            if match:
                vid = match.group(1)
            else:
                match = re.search(r"meta content='([^']+)' property='og:video'", data)
                if match:
                    match = re.search(r"vid=(\d+)&", match.group(1))
                    if match:
                        vid = match.group(1)
                    else:
                        log.error("Can't find video id for %s", url)
                        return
                else:
                    return None
    return vid

    def find_all_episodes(self, options):
        episodes = []
        matches = re.findall(r'<button class="show-more" data-url="([^"]+)" data-partial="([^"]+)"',
                             self.get_urldata())
        for encpath, enccomp in matches:
            newstyle = '_' in encpath
            if newstyle:
                encbasepath = encpath.split('_')[0]
                path = base64.b64decode(encbasepath + '===').decode('latin1') if is_py3 else base64.b64decode(encbasepath + '===')
            else:
                path = base64.b64decode(encpath + '===').decode('latin1') if is_py3 else base64.b64decode(encpath + '===')

            if '/view/' in path:
                continue

            params = 'offset=0&limit=1000'
            if newstyle:
                encparams = base64.b64encode(params.encode('latin1')).decode('latin1').rstrip('=') if is_py3 else \
                    base64.b64encode(params).rstrip('=')
                encpath = '{0}_{1}'.format(encbasepath, encparams)
            else:
                path = '{0}?{1}'.format(urlparse(path).path, params)
                encpath = base64.b64encode(path.encode('latin1')).decode('latin1').rstrip('=') if is_py3 else \
                    base64.b64encode(path).rstrip('=')

            url = urljoin('https://www.dr.dk/tv/partial/',
                          '{0}/{1}'.format(enccomp, encpath))
            data = self.http.request('get', url).content.decode('latin1') if is_py3 else \
                self.http.request('get', url).content

            matches = re.findall(r'"program-link" href="([^"]+)">', data)
            episodes = [urljoin('https://www.dr.dk/', url) for url in matches]
            break

        if not episodes:
            prefix = '/'.join(urlparse(self.url).path.rstrip('/').split('/')[:-1])
            matches = re.findall(r'"program-link" href="([^"]+)">', self.get_urldata())
            episodes = [urljoin('https://www.dr.dk/', url)
                        for url in matches
                        if url.startswith(prefix)]

        if options.all_last != -1:
            episodes = episodes[:options.all_last]
        else:
            episodes.reverse()

        return episodes