Python utils.get_http_data函数代码示例

    def get(self, options):
        data = self.get_urldata()
        match = re.search('data-aptomaId="([-0-9a-z]+)"', data)
        if not match:
            log.error("Can't find video info")
            sys.exit(2)
        videoId = match.group(1)
        match = re.search(r'data-isLive="(\w+)"', data)
        if not match:
            log.error("Can't find live info")
            sys.exit(2)
        if match.group(1) == "true":
            options.live = True
        if not options.live:
            dataurl = "http://aftonbladet-play-metadata.cdn.drvideo.aptoma.no/video/%s.json" % videoId
            data = get_http_data(dataurl)
            data = json.loads(data)
            videoId = data["videoId"]

        streamsurl = "http://aftonbladet-play-static-ext.cdn.drvideo.aptoma.no/actions/video/?id=%s&formats&callback=" % videoId
        streams = json.loads(get_http_data(streamsurl))
        hlsstreams = streams["formats"]["hls"]
        if "level3" in hlsstreams.keys():
            hls = hlsstreams["level3"]["csmil"][0]
        else:
            hls = hlsstreams["akamai"]["m3u8"][0]
        address = hls["address"]
        path = hls["path"]

        for i in hls["files"]:
            if "filename" in i.keys():
                playlist = "http://%s/%s/%s/master.m3u8" % (address, path, i["filename"])
            else:
                playlist = "http://%s/%s/%s/master.m3u8" % (address, path, hls["filename"])
            yield HLS(copy.copy(options), playlist, i["bitrate"])

    def get(self, options):
        match = re.search(r'mrss\s+:\s+"([^"]+)"', self.get_urldata())
        if not match:
            log.error("Can't find id for the video")
            sys.exit(2)
        swfurl = re.search(r'embedSWF\( "([^"]+)"', self.get_urldata())
        options.other = "-W %s" % swfurl.group(1)
        data = get_http_data(match.group(1))
        xml = ET.XML(data)
        mediagen = xml.find("channel").find("item").find("{http://search.yahoo.com/mrss/}group")
        title = xml.find("channel").find("item").find("title").text
        if options.output_auto:
            directory = os.path.dirname(options.output)
            if len(directory):
                options.output = "%s/%s" % (directory, title)
            else:
                options.output = title
        contenturl = mediagen.find("{http://search.yahoo.com/mrss/}content").attrib["url"]
        content = get_http_data(contenturl)
        xml = ET.XML(content)
        ss = xml.find("video").find("item")
        if is_py2_old:
            sa = list(ss.getiterator("rendition"))
        else:
            sa = list(ss.iter("rendition"))

        for i in sa:
            yield RTMP(options, i.find("src").text, i.attrib["bitrate"])

def download_hds(options, url, swf=None):
    data = get_http_data(url)
    streams = {}
    bootstrap = {}
    xml = ET.XML(data)
    prefix = xml.find("{http://ns.adobe.com/f4m/1.0}id").text

    if sys.version_info < (2, 7):
        bootstrapIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.getiterator("{http://ns.adobe.com/f4m/1.0}media")
    else:
        bootstrapIter = xml.iter("{http://ns.adobe.com/f4m/1.0}bootstrapInfo")
        mediaIter = xml.iter("{http://ns.adobe.com/f4m/1.0}media")

    for i in bootstrapIter:
        bootstrap[i.attrib["id"]] = i.text

    for i in mediaIter:
        streams[int(i.attrib["bitrate"])] = {"url": i.attrib["url"], "bootstrapInfoId": i.attrib["bootstrapInfoId"], "metadata": i.find("{http://ns.adobe.com/f4m/1.0}metadata").text}

    test = select_quality(options, streams)

    bootstrap = base64.b64decode(bootstrap[test["bootstrapInfoId"]])
    box = readboxtype(bootstrap, 0)
    if box[2] == "abst":
        antal = readbox(bootstrap, box[0])

    baseurl = url[0:url.rfind("/")]
    i = 1

    if options.output != "-":
        extension = re.search("(\.[a-z0-9]+)$", options.output)
        if not extension:
            options.output = "%s.flv" % options.output
        log.info("Outfile: %s", options.output)
        file_d = open(options.output, "wb")
    else:
        file_d = sys.stdout

    metasize = struct.pack(">L", len(base64.b64decode(test["metadata"])))[1:]
    file_d.write(binascii.a2b_hex(b"464c560105000000090000000012"))
    file_d.write(metasize)
    file_d.write(binascii.a2b_hex(b"00000000000000"))
    file_d.write(base64.b64decode(test["metadata"]))
    file_d.write(binascii.a2b_hex(b"00000000"))
    total = antal[1]["total"]
    eta = ETA(total)
    while i <= total:
        url = "%s/%sSeg1-Frag%s" % (baseurl, test["url"], i)
        if options.output != "-":
            eta.update(i)
            progressbar(total, i, ''.join(["ETA: ", str(eta)]))
        data = get_http_data(url)
        number = decode_f4f(i, data)
        file_d.write(data[number:])
        i += 1

    if options.output != "-":
        file_d.close()
        progress_stream.write('\n')

    def get(self, options):
        vid = None
        data = self.get_urldata()
        match = re.search(r'video url-([^"]+)', data)
        if not match:
            match = re.search(r'embed.jsp\?id=([^&]+)&', data)
            if not match:
                log.error("Cant find video id")
                sys.exit(2)
            vid = match.group(1)
        if not vid:
            path = unquote_plus(match.group(1))
            data = get_http_data("http://www.svd.se%s" % path)
            match = re.search(r'embed.jsp\?id=([^&]+)&', data)
            if not match:
                log.error("Cant find video id2")
                sys.exit(2)
            vid = match.group(1)

        url = "http://amz.lwcdn.com/api/cache/VideoCache.jsp?id=%s" % vid
        data = get_http_data(url)
        xml = ET.XML(data)
        videofile = xml.find("{http://www.lemonwhale.com/xml11}VideoFile")
        mediafiles = videofile.find("{http://www.lemonwhale.com/xml11}MediaFiles")
        high = mediafiles.find("{http://www.lemonwhale.com/xml11}VideoURLHigh")
        if high.text:
            yield HTTP(copy.copy(options), high.text, 720)
        file = mediafiles.find("{http://www.lemonwhale.com/xml11}VideoURL").text
        yield HTTP(copy.copy(options), file, 480)

    def get(self, options, url):
        parse = urlparse(url)
        try:
            other = parse[5]
        except KeyError:
            log.error("Something wrong with that url")
            sys.exit(2)
        match = re.search("^/(.*).html", other)
        if not match:
            log.error("Cant find video file")
            sys.exit(2)
        url = "http://www.hbo.com/data/content/%s.xml" % match.group(1)
        data = get_http_data(url)
        xml = ET.XML(data)
        videoid = xml.find("content")[1].find("videoId").text
        url = "http://render.cdn.hbo.com/data/content/global/videos/data/%s.xml" % videoid
        data = get_http_data(url)
        xml = ET.XML(data)
        ss = xml.find("videos")
        if sys.version_info < (2, 7):
            sa = list(ss.getiterator("size"))
        else:
            sa = list(ss.iter("size"))
        streams = {}
        for i in sa:
            stream = {}
            stream["path"] = i.find("tv14").find("path").text
            streams[int(i.attrib["width"])] = stream

        test = select_quality(options, streams)

        download_rtmp(options, test["path"])

    def get(self, options, url):
        parse = urlparse(url)
        data = get_http_data(url)
        match = re.search("abTvArticlePlayer-player-(.*)-[0-9]+-[0-9]+-clickOverlay", data)
        if not match:
            log.error("Can't find video file")
            sys.exit(2)
        try:
            start = parse_qs(parse[4])["start"][0]
        except KeyError:
            start = 0
        url = "http://www.aftonbladet.se/resource/webbtv/article/%s/player" % match.group(1)
        data = get_http_data(url)
        xml = ET.XML(data)
        url = xml.find("articleElement").find("mediaElement").find("baseUrl").text
        path = xml.find("articleElement").find("mediaElement").find("media").attrib["url"]
        live = xml.find("articleElement").find("mediaElement").find("isLive").text
        options.other = "-y %s" % path

        if start > 0:
            options.other = "%s -A %s" % (options.other, str(start))

        if live == "true":
            options.live = True

        if url == None:
            log.error("Can't find any video on that page")
            sys.exit(3)

        if url[0:4] == "rtmp":
            download_rtmp(options, url)
        else:
            filename = url + path
            download_http(options, filename)

    def get(self, options):
        parse = urlparse(self.url)
        try:
            other = parse.fragment
        except KeyError:
            log.error("Something wrong with that url")
            sys.exit(2)
        match = re.search("^/(.*).html", other)
        if not match:
            log.error("Cant find video file")
            sys.exit(2)
        url = "http://www.hbo.com/data/content/%s.xml" % match.group(1)
        data = get_http_data(url)
        xml = ET.XML(data)
        videoid = xml.find("content")[1].find("videoId").text
        url = "http://render.cdn.hbo.com/data/content/global/videos/data/%s.xml" % videoid
        data = get_http_data(url)
        xml = ET.XML(data)
        ss = xml.find("videos")
        if is_py2_old:
            sa = list(ss.getiterator("size"))
        else:
            sa = list(ss.iter("size"))

        for i in sa:
            videourl = i.find("tv14").find("path").text
            match = re.search("/([a-z0-9]+:[a-z0-9]+)/", videourl)
            options.other = "-y %s" % videourl[videourl.index(match.group(1)):]
            yield RTMP(copy.copy(options), videourl[:videourl.index(match.group(1))], i.attrib["width"])

    def get(self, options, url):
        data = get_http_data(url)
        match = re.search("xmlUrl: '(http://www.expressen.*)'", data)
        if not match:
            log.error("Can't find video file")
            sys.exit(2)
        url = "http://tv.expressen.se/%s/?standAlone=true&output=xml" % quote_plus(match.group(1))
        url = match.group(1)
        data = get_http_data(url)
        xml = ET.XML(data)
        ss = xml.find("vurls")
        if sys.version_info < (2, 7):
            sa = list(ss.getiterator("vurl"))
        else:
            sa = list(ss.iter("vurl"))
        streams = {}

        for i in sa:
            streams[int(i.attrib["bitrate"])] = i.text

        test = select_quality(options, streams)

        filename = test
        match = re.search("rtmp://([0-9a-z\.]+/[0-9]+/)(.*)", filename)

        filename = "rtmp://%s" % match.group(1)
        options.other = "-y %s" % match.group(2)

        download_rtmp(options, filename)

    def get(self, options):
        if re.findall(r"sydsvenskan.se", self.url):
            data = self.get_urldata()
            match = re.search(r"data-qbrick-mcid=\"([0-9A-F]+)\"", data)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            mcid = match.group(1)
            host = "http://vms.api.qbrick.com/rest/v3/getsingleplayer/%s" % mcid
        elif re.findall(r"di.se", self.url):
            data = self.get_urldata()
            match = re.search("src=\"(http://qstream.*)\"></iframe", data)
            if not match:
                log.error("Can't find video info")
                sys.exit(2)
            data = get_http_data(match.group(1))
            match = re.search(r"data-qbrick-ccid=\"([0-9A-Z]+)\"", data)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            host = "http://vms.api.qbrick.com/rest/v3/getplayer/%s" % match.group(1)
        elif re.findall(r"svd.se", self.url):
            match = re.search(r'video url-([^"]*)\"', self.get_urldata())
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            path = unquote_plus(match.group(1))
            data = get_http_data("http://www.svd.se%s" % path)
            match = re.search(r"mcid=([A-F0-9]+)\&width=", data)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            host = "http://vms.api.qbrick.com/rest/v3/getsingleplayer/%s" % match.group(1)
        else:
            log.error("Can't find site")
            sys.exit(2)

        data = get_http_data(host)
        xml = ET.XML(data)
        try:
            url = xml.find("media").find("item").find("playlist").find("stream").find("format").find("substream").text
        except AttributeError:
            log.error("Can't find video file")
            sys.exit(2)
        live = xml.find("media").find("item").find("playlist").find("stream").attrib["isLive"]
        if live == "true":
            options.live = True
        data = get_http_data(url)
        xml = ET.XML(data)
        server = xml.find("head").find("meta").attrib["base"]
        streams = xml.find("body").find("switch")
        if is_py2_old:
            sa = list(streams.getiterator("video"))
        else:
            sa = list(streams.iter("video"))

        for i in sa:
            options.other = "-y '%s'" % i.attrib["src"]
            yield RTMP(copy.copy(options), server, i.attrib["system-bitrate"])

    def get(self, options, url):
        if re.findall("dn.se", url):
            data = get_http_data(url)
            match = re.search("data-qbrick-mcid=\"([0-9A-F]+)\"", data)
            if not match:
                match = re.search("mediaId = \'([0-9A-F]+)\';", data)
                if not match:
                    log.error("Can't find video file")
                    sys.exit(2)
                mcid = "%sDE1BA107" % match.group(1)
            else:
                mcid = match.group(1)
            host = "http://vms.api.qbrick.com/rest/v3/getsingleplayer/%s" % mcid
        elif re.findall("di.se", url):
            data = get_http_data(url)
            match = re.search("ccid: \"(.*)\"\,", data)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            host = "http://vms.api.qbrick.com/rest/v3/getplayer/%s" % match.group(1)
        elif re.findall("svd.se", url):
            match = re.search("_([0-9]+)\.svd", url)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            data = get_http_data("http://www.svd.se/?service=ajax&type=webTvClip&articleId=%s" % match.group(1))
            match = re.search("mcid=([A-F0-9]+)\&width=", data)
            if not match:
                log.error("Can't find video file")
                sys.exit(2)
            host = "http://vms.api.qbrick.com/rest/v3/getsingleplayer/%s" % match.group(1)
        else:
            log.error("Can't find site")
            sys.exit(2)

        data = get_http_data(host)
        xml = ET.XML(data)
        try:
            url = xml.find("media").find("item").find("playlist").find("stream").find("format").find("substream").text
        except AttributeError:
            log.error("Can't find video file")
            sys.exit(2)

        data = get_http_data(url)
        xml = ET.XML(data)
        server = xml.find("head").find("meta").attrib["base"]
        streams = xml.find("body").find("switch")
        if sys.version_info < (2, 7):
            sa = list(streams.getiterator("video"))
        else:
            sa = list(streams.iter("video"))
        streams = {}
        for i in sa:
            streams[int(i.attrib["system-bitrate"])] = i.attrib["src"]

        path = select_quality(options, streams)

        options.other = "-y %s" % path
        download_rtmp(options, server)

    def get(self, options):
        match = re.search(r".*video/([0-9]+)", self.url)
        if not match:
            log.error("Can't find video file")
            sys.exit(2)

        video_id = match.group(1)
        if options.username and options.password:
            #bogus
            cc = Cookie(None, 'asdf', None, '80', '80', 'www.kanal5play.se', None, None, '/', None, False, False, 'TestCookie', None, None, None)
            self.cj.set_cookie(cc)
            #get session cookie
            data = get_http_data("http://www.kanal5play.se/", cookiejar=self.cj)
            authurl = "https://kanal5swe.appspot.com/api/user/login?callback=jQuery171029989&email=%s&password=%s&_=136250" % (options.username, options.password)
            data = get_http_data(authurl)
            match = re.search(r"({.*})\);", data)
            jsondata = json.loads(match.group(1))
            if jsondata["success"] == False:
                log.error(jsondata["message"])
                sys.exit(2)
            authToken = jsondata["userData"]["auth"]
            cc = Cookie(version=0, name='authToken',
                          value=authToken,
                          port=None, port_specified=False,
                          domain='www.kanal5play.se',
                          domain_specified=True,
                          domain_initial_dot=True, path='/',
                          path_specified=True, secure=False,
                          expires=None, discard=True, comment=None,
                          comment_url=None, rest={'HttpOnly': None})
            self.cj.set_cookie(cc)

        url = "http://www.kanal5play.se/api/getVideo?format=FLASH&videoId=%s" % video_id
        data = json.loads(get_http_data(url, cookiejar=self.cj))
        if not options.live:
            options.live = data["isLive"]
        if data["hasSubtitle"]:
            yield subtitle_json("http://www.kanal5play.se/api/subtitles/%s" % video_id)

        for i in data["streams"]:
            if i["drmProtected"]:
                log.error("We cant download drm files for this site.")
                sys.exit(2)
            steambaseurl = data["streamBaseUrl"]
            bitrate = i["bitrate"]
            if bitrate > 1000:
                bitrate = bitrate / 1000
            options2 = copy.copy(options)
            options2.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/K5StandardPlayer.swf", i["source"])
            options2.live = True
            yield RTMP(options2, steambaseurl, bitrate)

        url = "http://www.kanal5play.se/api/getVideo?format=IPAD&videoId=%s" % video_id
        data = json.loads(get_http_data(url, cookiejar=self.cj))
        for i in data["streams"]:
                streams = hlsparse(i["source"])
                for n in list(streams.keys()):
                    yield HLS(copy.copy(options), streams[n], n)

 def get(self, options, url):
     data = get_http_data(url)
     match = re.search(r'(http://load.cache.is/vodruv.*)"', data)
     js_url = match.group(1)
     js = get_http_data(js_url)
     tengipunktur = js.split('"')[1]
     match = re.search(r"http.*tengipunktur [+] '([:]1935.*)'", data)
     m3u8_url = "http://" + tengipunktur + match.group(1)
     base_url = m3u8_url.rsplit("/", 1)[0]
     download_hls(options, m3u8_url, base_url)

    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        m3u8 = get_http_data(self.url)
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        try:
            keydata = globaldata["KEY"]
            encrypted = True
        except KeyError:
            pass

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            key = get_http_data(match.group(1))
            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)
        if self.options.output != "-":
            extension = re.search(r"(\.[a-z0-9]+)$", self.options.output)
            if not extension:
                self.options.output = "%s.ts" % self.options.output
            log.info("Outfile: %s", self.options.output)
            if os.path.isfile(self.options.output) and not self.options.force:
                log.info("File already exists. use --force to overwrite")
                return
            file_d = open(self.options.output, "wb")
        else:
            file_d = sys.stdout

        n = 0
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if self.options.output != "-":
                eta.increment()
                progressbar(len(files), n, ''.join(['ETA: ', str(eta)]))
                n += 1

            data = get_http_data(item)
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            progress_stream.write('\n')

    def get(self, options):
        if re.findall("svt.se", self.url):
            match = re.search(r"data-json-href=\"(.*)\"", self.get_urldata())
            if match:
                filename = match.group(1).replace("&", "&").replace("&format=json", "")
                url = "http://www.svt.se%s" % filename
            else:
                log.error("Can't find video file")
                sys.exit(2)
        else:
            url = self.url

        pos = url.find("?")
        if pos < 0:
            dataurl = "%s?&output=json&format=json" % url
        else:
            dataurl = "%s&output=json&format=json" % url
        data = json.loads(get_http_data(dataurl))
        if "live" in data["video"]:
            options.live = data["video"]["live"]
        else:
            options.live = False

        if data["video"]["subtitleReferences"]:
            try:
                subtitle = data["video"]["subtitleReferences"][0]["url"]
            except KeyError:
                pass
            if len(subtitle) > 0:
                yield subtitle_wsrt(subtitle)

        for i in data["video"]["videoReferences"]:
            parse = urlparse(i["url"])

            if parse.path.find("m3u8") > 0:
                streams = hlsparse(i["url"])
                for n in list(streams.keys()):
                    yield HLS(copy.copy(options), streams[n], n)
            elif parse.path.find("f4m") > 0:
                match = re.search(r"\/se\/secure\/", i["url"])
                if not match:
                    parse = urlparse(i["url"])
                    manifest = "%s://%s%s?%s&hdcore=3.3.0" % (parse.scheme, parse.netloc, parse.path, parse.query)
                    streams = hdsparse(copy.copy(options), manifest)
                    for n in list(streams.keys()):
                        yield streams[n]
            elif parse.scheme == "rtmp":
                embedurl = "%s?type=embed" % url
                data = get_http_data(embedurl)
                match = re.search(r"value=\"(/(public)?(statiskt)?/swf(/video)?/svtplayer-[0-9\.a-f]+swf)\"", data)
                swf = "http://www.svtplay.se%s" % match.group(1)
                options.other = "-W %s" % swf
                yield RTMP(copy.copy(options), i["url"], i["bitrate"])
            else:
                yield HTTP(copy.copy(options), i["url"], "0")

    def download(self):
        if self.options.live and not self.options.force:
            raise LiveHLSException(self.url)

        error, m3u8 = get_http_data(self.url)
        if error:
            log.error("Cant get m3u8 file.")
            return
        globaldata, files = parsem3u(m3u8)
        encrypted = False
        key = None
        if "KEY" in globaldata:
            keydata = globaldata["KEY"]
            encrypted = True

        if encrypted:
            try:
                from Crypto.Cipher import AES
            except ImportError:
                log.error("You need to install pycrypto to download encrypted HLS streams")
                sys.exit(2)

            match = re.search(r'URI="(https?://.*?)"', keydata)
            error, key = get_http_data(match.group(1))
            if error:
                log.error("Can't get crypto key to decode files.")
                return
            rand = os.urandom(16)
            decryptor = AES.new(key, AES.MODE_CBC, rand)

        file_d = output(self.options, "ts")
        if hasattr(file_d, "read") is False:
            return

        n = 1
        eta = ETA(len(files))
        for i in files:
            item = _get_full_url(i[0], self.url)

            if self.options.output != "-":
                eta.increment()
                progressbar(len(files), n, "".join(["ETA: ", str(eta)]))
                n += 1

            error, data = get_http_data(item)
            if error:
                log.error("Missing segment in playlist")
                return
            if encrypted:
                data = decryptor.decrypt(data)
            file_d.write(data)

        if self.options.output != "-":
            file_d.close()
            progress_stream.write("\n")