C# HtmlAgilityPack.HtmlDocument.Load方法代码示例

        public override List<Show> Grab(string xmlParameters, ILogger logger)
        {
            var shows = new List<Show>();
            var doc = XDocument.Parse(xmlParameters);
            var sdElement = doc.Descendants("StartDate").FirstOrDefault();
            var startDateDiff = sdElement != null && sdElement.Value != null ? Convert.ToInt32(sdElement.Value) : -1;
            var edElement = doc.Descendants("EndDate").FirstOrDefault();
            var endDateDays = edElement != null && edElement.Value != null ? Convert.ToInt32(edElement.Value) : 3;

            foreach (Channel c in Enum.GetValues(typeof(Channel)))
            {
                var wr = WebRequest.Create(string.Format(url, (int)c, DateTime.Now.Date.AddDays(startDateDiff).ToString(DateFormat), DateTime.Now.Date.AddDays(endDateDays).ToString(DateFormat)));
                wr.Timeout = 30000;
                logger.WriteEntry(string.Format("Grabbing hot.net.il channel {0} ", c.ToString()), LogType.Info);
                var res = (HttpWebResponse)wr.GetResponse();

                var html = new HtmlAgilityPack.HtmlDocument();
                html.Load(res.GetResponseStream(), Encoding.UTF8);

                foreach (var tr in html.DocumentNode.Descendants("tr").Where(x => x.Attributes.Contains("class") && x.Attributes["class"].Value == "redtr_off"))
                {
                    var tds = tr.Descendants("td").ToList();
                    var show = new Show();
                    show.Title = tds[2].InnerText;
                    show.StartTime = DateTime.SpecifyKind(Convert.ToDateTime(tds[4].InnerText),DateTimeKind.Unspecified);
                    show.StartTime = TimeZoneInfo.ConvertTime(show.StartTime, TimeZoneInfo.Local, TimeZoneInfo.Utc);
                    show.EndTime = show.StartTime.Add(Convert.ToDateTime(tds[5].InnerText).TimeOfDay);
                    show.Channel = c.ToString();
                    shows.Add(show);
                }
            }
            return shows;
        }

        public static async Task<DataGroup> NewsGoVnGroup_Parse(string xmlString, DataModel.DataGroup group, int takeNum)
        {
            StringReader _stringReader = new StringReader(xmlString);
            XDocument _xdoc = XDocument.Load(_stringReader);
            var channelElement = _xdoc.Element("rss").Element("channel");
            if (channelElement != null)
            {
                group.Title = channelElement.Element("title").Value;
                group.Subtitle = channelElement.Element("title").Value;
                group.Description = channelElement.Element("description").Value;

                var items = channelElement.Elements("item");
                foreach (var item in items)
                {
                    if (group.Items.Count == takeNum && takeNum >= 0) break;

                    DataItem dataItem = new DataItem();
                    dataItem.Title = item.Element("title").Value;
                    dataItem.Description = StripHTML(item.Element("description").Value);
                    dataItem.Link = new Uri(item.Element("link").Value, UriKind.Absolute);
                    dataItem.PubDate = item.Element("pubDate").Value;

                    HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
                    htmlDoc.Load(new StringReader(item.Element("description").Value));

                    HtmlAgilityPack.HtmlNode imageLink = getFirstNode("img", htmlDoc.DocumentNode);
                    dataItem.ImageUri = new Uri(imageLink.GetAttributeValue("src", string.Empty).Replace("96.62.jpg", "240.155.jpg"), UriKind.Absolute);

                    dataItem.Group = group;
                    group.Items.Add(dataItem);
                }
            }

            return group;
        }

		private async Task ParseFromAddress(string url)
		{
			var response = await new System.Net.Http.HttpClient().GetStreamAsync(url);
			var page = new HtmlAgilityPack.HtmlDocument();
			page.Load(response);
			var itemNodes = page.DocumentNode.SelectNodes("//form[@id='list-form']/div[@class='list-items']/div[@class='percent-wrap']");
			foreach (var item in itemNodes)
			{
				var titleNode = item.SelectNodes(".//p[@class='title']/a").FirstOrDefault();
				var priceNode = item.SelectNodes(".//p[@class='price']").FirstOrDefault();
				string link = titleNode.Attributes["href"].Value;
				string title = titleNode.Attributes["title"].Value;
				double price;
				double.TryParse(priceNode.InnerText.Trim().Replace("US$", "").Replace(".",","), out price);
				var lumenMatch = new Regex(@"\d{3,4}[-\s]?(lm|lumen)", RegexOptions.IgnoreCase).Match(title);
				if (!lumenMatch.Groups[0].Success)
					continue;
				string lumenCountString = Regex.Replace(lumenMatch.Groups[0].Value, "[^0-9]", "");
				int lumenCount;
				if (!int.TryParse(lumenCountString, out lumenCount))
					continue;
				Items.Add(new RowItem
					{
						LumenCount =  lumenCount,
						Price = price,
						Link = "http://dx.com" + link
					});
			}
			this.Title = Items.Count.ToString();
		}

        public static void FetchRates(int bankId, string bankName)
        {
            var bankExchange = new BankExchange();
            bankExchange.BankId = bankId;

            if (bankName.ToLowerInvariant() == "bank asya")
            {
                var ds = new DataSet("fxPrices");
                ds.ReadXml("http://www.bankasya.com.tr/xml/kur_list.xml");

                bankExchange.USDBuying = Int32.Parse(ds.Tables[1].Rows[0]["Kur"].ToString().Replace(".", ""));
                bankExchange.USDSelling = Int32.Parse(ds.Tables[1].Rows[1]["Kur"].ToString().Replace(".", ""));
                bankExchange.EURBuying = Int32.Parse(ds.Tables[1].Rows[2]["Kur"].ToString().Replace(".", ""));
                bankExchange.EURSelling = Int32.Parse(ds.Tables[1].Rows[3]["Kur"].ToString().Replace(".", ""));

                bankExchange.Save();
            }
            else if(bankName.ToLowerInvariant() == "finansbank")
            {
                var doc = new HtmlAgilityPack.HtmlDocument();
                doc.Load("http://www.finansbank.com.tr/bankacilik/alternatif-dagitim-kanallari/internet-bankaciligi/doviz_kurlari.aspx?IntSbMO_FB_Mevduatoranlari_PU".DownloadPage());

            }
            else
            {
                throw new Exception("Banka adı bunlardan biri olabilir: Bank Asya");
            }
        }

        public CookieContainer DoLogin()
        {
            const string formUrl = "http://dbunet.dbu.dk";
            var req = (HttpWebRequest)WebRequest.Create(formUrl);
            req.Method = "GET";

            var resp = req.GetResponse() as HttpWebResponse;
            var loginPage = new HtmlDocument();
            loginPage.Load(resp.GetResponseStream());
            
            var viewstate = loginPage.DocumentNode.SelectSingleNode("//input[@name='__VIEWSTATE']").GetAttributeValue("value","---");

            string proxy = null;
            
            var formParams = string.Format("?__VIEWSTATE={0}&_ctl2:sys_txtUsername={1}&_ctl2:sys_txtPassword={2}&_ctl2:cbRememberMe=true&_ctl2:ibtnSignin=Continue&_ctl2:ibtnSignin.x=33&_ctl2:ibtnSignin.y=3&TopMenu_ClientState=", viewstate, UserName, Password);
            req = (HttpWebRequest)WebRequest.Create(formUrl);
            req.ContentType = "application/x-www-form-urlencoded";
            req.Method = "POST";
            req.KeepAlive = true;
            req.AllowAutoRedirect = false;
            req.Proxy = new WebProxy(proxy, true); // ignore for local addresses
            req.CookieContainer = new CookieContainer(); // enable cookies
            byte[] bytes = Encoding.ASCII.GetBytes(formParams);
            req.ContentLength = bytes.Length;
            using (var os = req.GetRequestStream())
            {
                os.Write(bytes, 0, bytes.Length);
            }
            resp = (HttpWebResponse)req.GetResponse();

            return req.CookieContainer;
        }

        public virtual void Execute(IJobExecutionContext context)
        {
            var dateTimeMin = DateTime.Parse(DateTime.Now.ToString("yyyy-MM-dd"));//当天零点
            var list = _ppismItemRepository.GetAll(x => (x.ItemSource == PPism.Model.Enum.DictPPItemSource.天猫 || x.ItemSource == PPism.Model.Enum.DictPPItemSource.淘宝) && (!x.LastListenTime.HasValue || x.LastListenTime.Value < dateTimeMin)).ToList();
            for (int i = 0, length = list.Count; i < length; i++)
            {
                var item = list[i];
                Process p = new Process();
                try
                {
                    var environment = Environment.CurrentDirectory;

                    p.StartInfo.FileName = environment + "\\phantomjs\\bin\\phantomjs.exe";

                    p.StartInfo.WorkingDirectory = environment + "\\phantomjs\\bin\\";

                    string strArg = @"{0}\phantomjs\bin\tmallsavehtml.js ""{1}"" ""{0}\{2}""";
                    p.StartInfo.Arguments = string.Format(strArg, environment, item.ListenUrl, "\\phantomjs\\htmltmall\\" + item.Id.ToString());

                    p.StartInfo.CreateNoWindow = true;
                    p.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
                    if (!p.Start())
                        throw new Exception("无法Headless浏览器.");
                    Thread.Sleep(2 * 1000);
                    string fileUrl = string.Format(@"{0}\phantomjs\htmltmall\{1}.html", environment, item.Id.ToString());
                    var fileInfo = new FileInfo(fileUrl);
                    if (!fileInfo.Exists || fileInfo.LastWriteTime < DateTime.Now.AddHours(-1))
                        Thread.Sleep(5 * 1000);//如果还没有返回就再等5秒
                    if (File.Exists(fileUrl))
                    {
                        var htmldocument = new HtmlAgilityPack.HtmlDocument();
                        htmldocument.Load(fileUrl);
                        var strHtml = htmldocument.DocumentNode.InnerHtml.ToString();
                        string reg = @"id=""J_PromoPrice""[\s\S]+?class=""tm-price"">(?<price>[^<]+)[\s\S]+?<img\s+id=""J_ImgBooth""[\s\S]+?src=""(?<imgUrl>[^""]+)";
                        if (item.ItemSource == PPism.Model.Enum.DictPPItemSource.淘宝)//淘宝和天猫正则不一样
                            reg = @"id=""J_PromoPriceNum""[\s\S]+?class=""tb-rmb-num"">(?<price>[^<]+)[\s\S]+?<img\s+id=""J_ImgBooth""[\s\S]+?src=""(?<imgUrl>[^""]+)";
                        var groups = Regex.Match(strHtml, reg).Groups;
                        var price = groups["price"].Value.Trim().ToDecimal(0);
                        var imgUrl = groups["imgUrl"].Value.Trim();
                        if (price > 0 && !string.IsNullOrEmpty(imgUrl))
                        {
                            var ppismItemBll = new PPismJob.Common.PPismItemBll();
                            var priceItem = ppismItemBll.GetPriceItem(item, price, imgUrl);
                            _priceItemRepository.Add(priceItem);
                            _ppismItemRepository.Update(item);
                        }
                    }
                }
                catch (Exception ex)
                {

                    throw ex;
                }
                finally
                {
                    p.Dispose();
                }
            }
            _repositoryContext.Commit();
        }

            /// <summary>
            /// Gets school info: name and departments name from html polish Wikipedia page.
            /// </summary>
            /// <param name="filePath">Path to locally saved html file.</param>
            /// <returns>School object with name and list of departments name.</returns>
            public static School GetSchool(string filePath)
            {
                HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();

                htmlDoc.OptionFixNestedTags = true;
                var e = htmlDoc.Encoding;
                htmlDoc.Load(filePath, Encoding.UTF8);

                var school = new School();

                if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0)
                {
                    // Handle any parse errors as required
                }
                else
                {

                    if (htmlDoc.DocumentNode != null)
                    {
                        var nodes = htmlDoc.DocumentNode.Descendants();
                        school.Name = nodes.First(x => x.Id == "firstHeading").InnerText;
                        var content = nodes.First(x => x.Id == "mw-content-text").Descendants().Where(x=>x.Name=="li").ToList();

                        foreach (var item in content)
                        {
                            if(item.InnerText.Contains("Wydział")) {
                                school.Departments.Add(item.InnerText);
                            }
                        }
                    }
                }

                return school;
            }

        public ProgramOptions Get(string configPath)
        {
            var html = new HtmlAgilityPack.HtmlDocument();
            html.Load(configPath);

            var options = new ProgramOptions();
            options.BlogUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/blogurl", "", true);
            options.BlogUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/bloguser", "", true);
            options.BlogPassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/blogpassword", "", true);
            options.DatabaseUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databaseurl", "", true);
            options.DatabaseName = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databasename", "", true);
            options.DatabaseUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databaseuser", "", true);
            options.DatabasePassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/databasepassword", "", true);

            options.FtpUrl = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftpurl", "", true);
            options.FtpUser = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftpuser", "", true);
            options.FtpPassword = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/ftppassword", "", true);

            options.ProxyAddress = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/proxyaddress", "", true);
            options.ProxyPort = XmlParse.GetIntegerNodeValue(html.DocumentNode, "/programoptions/proxyport", 0);
            options.UseProxy = XmlParse.GetBooleanNodeValue(html.DocumentNode, "/programoptions/useproxy", false);

            options.YoutubeClient = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/youtubeclient", "", true);
            options.YoutubeClientSecret = XmlParse.GetStringNodeValue(html.DocumentNode, "/programoptions/youtubeclientsecret", "", true);

            return options;
        }

 public void RocklandWebsiteShouldContainROCKLAND_offline()
 {
     var html = new HtmlAgilityPack.HtmlDocument();
     var path = System.IO.Path.GetDirectoryName(
         System.Reflection.Assembly.GetExecutingAssembly().Location) + @"\..\..\TestData";
     html.Load(path + @"\ROCKLAND - Mach an und laut!.html");
     Assert.IsTrue(html.DocumentNode.ChildNodes["html"].InnerText.Contains("ROCKLAND"));
 }

        public static List<Shows> Parse(DateTime date) //Downloading Titles of shows from specific dat
        {
            string Day = date.ToString("dd-MM-yyyy"); //Getting date in proper format for URL
            string result;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.pogdesign.co.uk/cat/day/" + Day);
            request.Method = "GET";

            using (var stream = request.GetResponse().GetResponseStream())
            using (var reader = new StreamReader(stream, Encoding.UTF8))
            {
                result = reader.ReadToEnd();
            }

            List<string> Titles = new List<string>();
            List<int> Seasons = new List<int>();
            List<int> Episodes = new List<int>();
            List<Shows> TVShow = new List<Shows>();
            List<string> Images = new List<string>();
            var document = new HtmlAgilityPack.HtmlDocument();
            document.Load(new StringReader(result));
            var titleNodes = document.DocumentNode.SelectNodes("//h4"); //Show titles are in <h4>
            var SENodes = document.DocumentNode.SelectNodes("//h5/a/span"); //Season & Episode Nodes
            var imageNodes = document.DocumentNode.SelectNodes("//div[contains(@class,'contbox ovbox')]");

            foreach (var title in titleNodes)
            {
                var text = title.InnerText;
                var wSummary = text.Replace("Summary", "");
                var resultTitle = wSummary.Remove(wSummary.Length - 2); //Deleting white spaces
                Titles.Add(resultTitle);
            }
            foreach (var SE in SENodes)
            {
                string pattern = "(?<=Season )(\\w+)"; // regex pattern for season number
                string pattern1 = "(?<=Episode )(\\w+)"; // regex pattern for episode number
                var textFromSpan = SE.InnerText;

                Match seasonMatch = Regex.Match(textFromSpan, pattern);
                Match episodeMatch = Regex.Match(textFromSpan, pattern1);

                Seasons.Add(Int32.Parse(seasonMatch.ToString()));
                Episodes.Add(Int32.Parse(episodeMatch.ToString()));
            }
            foreach (var image in imageNodes)
            {
                var url = Regex.Match(image.GetAttributeValue("style", ""), @"(?<=url\()(.*)(?=\))").Groups[1].Value;
                var output = "http://www.pogdesign.co.uk/" + url;
                Images.Add(output);

            }
            for (int i = 0; i < Titles.Count; i++)
            {
                TVShow.Add(new Shows(Titles[i], Seasons[i], Episodes[i], Images[i])); // Filing List with Shows objects
            }

            return TVShow;
        }

        static FineAntsCore.Statement ConvertHSBCHTMLFileToFineAnts(FileInfo fileInfo)
        {
            HtmlAgilityPack.HtmlDocument brokenDocument = new HtmlAgilityPack.HtmlDocument();
            brokenDocument.Load(fileInfo.FullName);
            brokenDocument.OptionOutputAsXml = true;
            string fixedXmlFileName = fileInfo.FullName + ".fixed.xml";
            brokenDocument.Save(fixedXmlFileName);
            XmlDocument document = new XmlDocument();
            document.Load(fixedXmlFileName);

            XmlNamespaceManager namespaceManager = new XmlNamespaceManager(document.NameTable);
            namespaceManager.AddNamespace("d", "http://www.w3.org/1999/xhtml");

            XmlNode closingBalanceNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[last()]/d:td[6]/d:p", namespaceManager);
            XmlNode closingBalanceSignNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[last()]/d:td[7]/d:p", namespaceManager);
            int closingBalance = moneyInPenceFromString(closingBalanceNode.InnerText.Trim());
            if (closingBalanceSignNode.InnerText.Trim() == "D") closingBalance = -closingBalance;

            XmlNode endDateNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:div[@class='extPibRow hsbcRow']/d:div[@class='hsbcPadding']/d:div[@class='hsbcTextRight']", namespaceManager);
            string endDateString = HtmlAgilityPack.HtmlEntity.DeEntitize(endDateNode.InnerText).Trim();

            System.Globalization.CultureInfo provider = System.Globalization.CultureInfo.InvariantCulture;
            DateTime endDate = DateTime.ParseExact(endDateString, "dd MMM yyyy", provider);

            XmlNode startDateNode = document.SelectSingleNode("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[1]/d:td[1]/d:p", namespaceManager);
            string startDateString = HtmlAgilityPack.HtmlEntity.DeEntitize(startDateNode.InnerText).Trim();

            DateTime startDate = dateFromDateStringFixedUsingUpperBoundDate(startDateString, endDate.AddDays(-1)).AddDays(1);

            List<FineAntsCore.Transaction> transactions = new List<FineAntsCore.Transaction>();

            XmlNodeList transactionNodes = document.SelectNodes("/span/d:html/d:body/d:div[@id='top']/d:div[@id='innerPage']/d:div[@id='wrapper']/d:div[@id='main']/d:div[@id='content']/d:div[@class='containerMain']/d:div[@class='hsbcMainContent hsbcCol']/d:div[@class='extContentHighlightPib hsbcCol']/d:table/d:tbody/d:tr[position()>1 and position()<last()]", namespaceManager);
            foreach (XmlNode node in transactionNodes)
            {
                XmlNode dateNode = node.SelectSingleNode("d:td[1]/d:p", namespaceManager);
                XmlNode typeNode = node.SelectSingleNode("d:td[2]/d:p", namespaceManager);
                XmlNode nameNode = node.SelectSingleNode("d:td[3]/d:p", namespaceManager);
                XmlNode moneyOutNode = node.SelectSingleNode("d:td[4]/d:p", namespaceManager);
                XmlNode moneyInNode = node.SelectSingleNode("d:td[5]/d:p", namespaceManager);

                string date = HtmlAgilityPack.HtmlEntity.DeEntitize(dateNode.InnerText).Trim();
                string name = HtmlAgilityPack.HtmlEntity.DeEntitize(getInnerTextIgnoringLinks(nameNode));
                string moneyIn = HtmlAgilityPack.HtmlEntity.DeEntitize(moneyInNode.InnerText).Trim();
                string moneyOut = HtmlAgilityPack.HtmlEntity.DeEntitize(moneyOutNode.InnerText).Trim();
                int money = moneyIn == "" ? -moneyInPenceFromString(moneyOut) : moneyInPenceFromString(moneyIn);

                transactions.Add(new FineAntsCore.Transaction(money, dateFromDateStringFixedUsingUpperBoundDate(date, endDate), name, ""));
            }

            // remove the temporary fixed file
            System.IO.File.Delete(fixedXmlFileName);

            FineAntsCore.Statement statement = new FineAntsCore.Statement(transactions, startDate, endDate, closingBalance);

            return statement;
        }

        public void ToClearTextTest()
        {
            HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
            htmlDoc.Load("Test.html");

            string expected = "Test \r\n    Some text\r\n     \r\n    Some more text\r\n     \r\nText here and there ";
            string converted = htmlDoc.DocumentNode.ToClearText();

            Assert.AreEqual(expected, converted);
        }

 public void SongsCanBeSortedByTimestamp()
 {
     var html = new HtmlAgilityPack.HtmlDocument();
     var path = System.IO.Path.GetDirectoryName(
         System.Reflection.Assembly.GetExecutingAssembly().Location) + @"\..\..\TestData";
     html.Load(path + @"\ROCKLAND - Mach an und laut!.html");
     var parser = new RocklandParser();
     var songs = parser.GetSongs(html);
     Assert.AreEqual("Russ Ballard", songs.First().Artist);
     songs.Sort();
     Assert.AreEqual("Russ Ballard", songs.Last().Artist);
 }

 public static List<string> GetFullShowSelection() // Getting a list of all shows in calendar
 {
     var Titles = new List<string>();
     var client = new WebClient();
     string page = client.DownloadString("http://www.pogdesign.co.uk/cat/showselect.php");
     var document = new HtmlAgilityPack.HtmlDocument();
     document.Load(new StringReader(page));
     var showNodes = document.DocumentNode.SelectNodes("//label[@class=\"label_check\"]/strong"); //Show titles are in <h4>
     foreach (var node in showNodes)
     {
         Titles.Add(node.InnerText);
     }
     return Titles;
 }

        public HtmlAgilityPack.HtmlDocument LoadDocument(string url)
        {
            var document = new HtmlAgilityPack.HtmlDocument();

            try
            {
                using (var responseStream = CreateRequest(url).GetResponse().GetResponseStream())
                {
                    document.Load(responseStream, Encoding.UTF8);
                }
            }
            catch(Exception ) 
            {
                //just do a second try
                Thread.Sleep(1000);
                using (var responseStream = CreateRequest(url).GetResponse().GetResponseStream())
                {
                    document.Load(responseStream, Encoding.UTF8);
                }
            }

            return document;
        }