C# HtmlWeb.LoadFromWebAsync方法代码示例

 private async Task<HtmlNode> LoadWebsitesDocumentNode(string url)
 {
     HtmlWeb htmlWeb = new HtmlWeb();
     HtmlDocument website = null;
     try
     {
         website = await htmlWeb.LoadFromWebAsync(url , "gsgschueler" , "vp2015_01");
     }
     catch(HtmlAgilityPack.HtmlWebException ex)
     {
        // await new Windows.UI.Popups.MessageDialog(ex.Message).ShowAsync();
     }
     HtmlNode root = website?.DocumentNode;
     return root;
 }

        public  async Task<All> GetAllSet(){
            HtmlWeb webClient = new HtmlWeb();
            //国际名校公开课 http://open.163.com/ocw/
            HtmlDocument docNode = await webClient.LoadFromWebAsync("http://localhost:8080/OpenCourse163Test.html");//
            System.Diagnostics.Debug.WriteLine("after LoadFromWebAsync");
            HtmlNode node = docNode.DocumentNode;
            // 1.<div class="m-t-bg">...</div>
            List<Catalogue> catalogues = this.GetCatalogueList(node);
            //获取一级标题列表
            HashSet<NewCatalogue> newCatalogues = new HashSet<NewCatalogue>();
            // Guid g = new Guid();
            //获取二级标题列表
            HashSet<NewCourseType> newCourseTypes = new HashSet<NewCourseType>();
            //获取课程列表
            HashSet<NewCourse> newCourses = new HashSet<NewCourse>();

            foreach (var catalogue in catalogues)
            {
                //一级
                NewCatalogue nc = new NewCatalogue
                {
                    ID = catalogue.CatalogueTitle.GetHashCode(),
                    Title = catalogue.CatalogueTitle
                };
                newCatalogues.Add(nc);
                //二级
                CourseType ct = catalogue.CourseTypes;
                NewCourseType nct = new NewCourseType
                {
                    ID = ct.CourseTypeId.GetHashCode(),
                    Title = ct.CourseTypeTitle,
                    Catalogue = nc
                };
                newCourseTypes.Add(nct);
                //课程列表
                foreach (var oc in catalogue.CourseTypes.OCourses)
                {
                    NewCourse newCourse =
                        new NewCourse
                        {
                            CourseTitle = oc.CourseTitle,
                            CourseHrefUrl = oc.CourseHrefUrl,
                            CourseImgUrl = oc.CourseImgUrl,
                            CourseUpdataProgress = oc.CourseUpdataProgress,
                            CourseType = nct

                        };
                    newCourses.Add(newCourse);
                }
            }

            return new All 
            { 
                NewCatalogueSet = newCatalogues ,
                newCourseTypeSet = newCourseTypes,
                NewCourseSet = newCourses
            };
        }

 public void ReadSite()
 {
     HtmlWeb webClient = new HtmlWeb();
     //国际名校公开课 http://open.163.com/ocw/
     Task<HtmlDocument> doc = webClient.LoadFromWebAsync("http://localhost:8080/OpenCourse163Test.html");//http://open.163.com/ocw/
     HtmlDocument docNode =doc.Result;
     HtmlNode node = docNode.DocumentNode;
     // 1.<div class="m-t-bg">...</div>
     List<Catalogue> catalogues=GetCatalogueList(node);
     printCatalogues(catalogues);
     //断言
     Assert.AreEqual(18,catalogues.Count());
 }

        /// <summary>
        /// Load content fisrt and replace a tag to javascript to raise event when user click to a link
        /// because WebView in Windows 8 is not support event Navigating
        /// </summary>
        /// <param name="url"></param>
        async void GoTo(string url)
        {
            HtmlWeb htmlWeb = new HtmlWeb();
            // Using HtmlDocument to parse content to xml linq formart
            HtmlDocument doc = await htmlWeb.LoadFromWebAsync(url);
            foreach (HtmlNode link in doc.DocumentNode.Descendants("a"))
            {
                HtmlAttribute att = link.Attributes["href"];
                att.Value = FixLink(att);
            }

            //WebViewMain.NavigateToString(doc.DocumentNode.OuterHtml);
            WebViewMain.Navigate(new Uri(url));
            WebViewMain.LoadCompleted += WebViewMain_LoadCompleted;
        }

        public static async Task<Incident> GetIncidentDetailsAsync(int incidentId)
        {
            if (incidentId < 0)
            {
                throw new ArgumentOutOfRangeException("incidentId");
            }

            // Download the webpage
            string url = String.Format(GetIncidentDetailsUrl, incidentId);
            HtmlWeb web = new HtmlWeb();

            HtmlDocument doc = await web.LoadFromWebAsync(url);

            string name = "";
            Dictionary<string, string> properties = null;

            // Find incident name
            foreach (HtmlNode headerNode in doc.DocumentNode.Descendants("h3"))
            {
                if (headerNode.HasAttributes && 
                    headerNode.Attributes.Contains("class") &&
                    headerNode.Attributes["class"].Value == "incident_h3")
                {
                    name = headerNode.InnerText;
                    break;
                }
            }

            // Find incident properties
            foreach (HtmlNode tableNode in doc.DocumentNode.Descendants("table"))
            {
                if (tableNode.HasAttributes &&
                    tableNode.Attributes.Contains("class") &&
                    tableNode.Attributes["class"].Value == "incident_table")
                {
                    properties = ParseIncidentTable(tableNode);
                    break;
                }
            }

            return new Incident(incidentId, name, properties);
        }

        /// <summary>
        ///     Retreives JSON Data from Mashie by parsing the HTML.
        ///     Because of Mashie not having an public API I had to make my own workaround.
        /// </summary>
        /// <returns>JSON String with the data.</returns>
        private static async Task<string> GetJsonDataFromMashieAsync()
        {
            //Url to page to parse.
            const string urlToFetchFrom = @"https://mpi.mashie.eu/public/menu/motala+kommun/af77367d";
            var webClient = new HtmlWeb();

            //download HTML from url.
            HtmlDocument document = await webClient.LoadFromWebAsync(urlToFetchFrom);
            //find root of HTML document.
            HtmlNode root = document.DocumentNode;
            //find all script nodes.
            var nodes = root.Descendants("script");

            //find the node that contains the "weekData" string.
            foreach (string fixedScript in from node in nodes
                                           where node.InnerHtml.Contains("weekData")
                                           select node.InnerHtml.Substring(20))
            {
                return fixedScript;
            }

            //if the parse failed
            return "Couldn't find Json data from Mashie";
        }

        /// <summary>
        /// Removing unused elements on html content
        /// </summary>
        /// <param name="url"></param>
        /// <param name="web"></param>
        /// <param name="feedItem"></param>
        /// <returns></returns>
        public async Task<string> RemoveUnusedElementsAsync(string url)
        {
            HtmlWeb web = new HtmlWeb();
            var document = await web.LoadFromWebAsync(url);

            document.DocumentNode.Descendants().Where(x => x.Id == "fb-root").ToList().ForEach(x => x.Remove());
            document.DocumentNode.Descendants().Where(x => x.Id == "header").ToList().ForEach(x => x.Remove());
            document.DocumentNode
                .Descendants(
                    "div")
                .FirstOrDefault(
                    d =>
                        d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("section-head")).Remove();
            document.DocumentNode
                .Descendants(
                    "div")
                .FirstOrDefault(
                    d =>
                        d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("container inner"))
                .Descendants("div").First().Remove();
            document.DocumentNode
                .Descendants(
                    "div")
                .FirstOrDefault(
                    d =>
                        d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("meta"))?.Remove();
            document.DocumentNode
                .Descendants(
                    "div")
                .FirstOrDefault(
                    d =>
                        d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("ssba"))?.Remove();
            document.DocumentNode.Descendants().Where(x => x.Id == "jp-relatedposts").ToList().ForEach(x => x?.Remove());
            document.DocumentNode.Descendants("aside").ToList().ForEach(x => x.Remove());
            document.DocumentNode.Descendants("footer").ToList().ForEach(x => x.Remove());

            // Modify some elements

            return document.DocumentNode.OuterHtml;
        }

 public async void DownloadPage()
 {
     var webGet = new HtmlWeb();
     var document = await webGet.LoadFromWebAsync(_url);
     DownloadedPage(document);
 }

        public static async Task<IEnumerable<Incident>> GetIncidentsAsync(int currentPage = IncidentsDefaultCurrentPage, int itemsPerPage = IncidentsDefaultItemsPerPage, IncidentsSortOrder sortOrder = IncidentsDefaultSortOrder)
        {
            if (currentPage < 0)
            {
                throw new ArgumentOutOfRangeException("currentPage");
            }

            if (itemsPerPage < 0)
            {
                throw new ArgumentOutOfRangeException("itemsPerPage");
            }

            string sortOrderString = "";
            switch (sortOrder)
            {
                case IncidentsSortOrder.Name:
                    sortOrderString = "incident_name";
                    break;
                case IncidentsSortOrder.County:
                    sortOrderString = "incident_county";
                    break;
                case IncidentsSortOrder.AdministrativeUnit:
                    sortOrderString = "incident_administrative_unit";
                    break;
                case IncidentsSortOrder.DateStarted:
                    sortOrderString = "incident_date_created";
                    break;
                case IncidentsSortOrder.DateLastUpdated:
                    sortOrderString = "incident_date_last_update";
                    break;
                case IncidentsSortOrder.Priority:
                default:
                    sortOrderString = "incident_priority";
                    break;
            }

            // Download the webpage
            string url = String.Format(GetIncidentsUrl, currentPage, itemsPerPage, sortOrderString);
            HtmlWeb web = new HtmlWeb();

            HtmlDocument doc = await web.LoadFromWebAsync(url);

            List<Incident> incidents = new List<Incident>();

            // Find incident tables
            foreach (HtmlNode tableNode in doc.DocumentNode.Descendants("table"))
            {
                if (tableNode.HasAttributes &&
                    tableNode.Attributes.Contains("class") &&
                    tableNode.Attributes["class"].Value == "incident_table")
                {
                    string name = tableNode.Attributes["title"].Value;
                    if (name != "Search for a fire")
                    {
                        int id = Incident.InvalidId;
                        Dictionary<string, string> details = ParseIncidentTable(tableNode);

                        // Parse out the hidden id
                        if (details.ContainsKey(IncidentIDKey))
                        {
                            id = Int32.Parse(details[IncidentIDKey]);
                        }

                        Incident incident = (id == Incident.InvalidId) ? new Incident(name, details) : new Incident(id, name, details);
                        incidents.Add(incident);
                    }
                }
            }

            return incidents;
        }

        /// <summary>
        ///     The scrape.
        /// </summary>
        /// <returns>
        ///     The <see cref="Task" />.
        /// </returns>
        public static async Task<BandwidthResults> Scrape()
        {
            var web = new HtmlWeb();

            IPropertySet settings = ApplicationData.Current.LocalSettings.Values;
            string siteToLoad = (string)settings["user"] == "testuser"
                                    ? "http://alexmullans.com/bandwidth.html"
                                    : "http://netreg.rose-hulman.edu/tools/networkUsage.pl";
            HtmlDocument doc =
                await
                web.LoadFromWebAsync(
                    siteToLoad, new UTF8Encoding(), (string)settings["user"], (string)settings["pass"], "rose-hulman");
            return ParseBandwidthDocument(doc);
        }

        public async Task<ViewModel.NewsItem> CreateItem(ViewModel.NewsItem result, SyndicationItem item)
        {
            result.URIToSource = item.Id;
            
            if (!string.IsNullOrEmpty(result.ContentRaw))
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(result.ContentRaw);
                foreach (HtmlNode link in doc.DocumentNode.Descendants("img"))
                {
                    result.Image = link.Attributes["src"].Value; break;
                }
                
                result.Content = HtmlEntity.DeEntitize(doc.DocumentNode.InnerText);
                result.ContentRaw = null;
            }

            HtmlDocument site = null;
            HtmlWeb web = new HtmlWeb();
            try { site = await web.LoadFromWebAsync(result.URIToSource); }
            catch { }

            if (site != null)
            {
                var contentNode = site.DocumentNode.Descendants("div").Where(A => A.Attributes["class"]?.Value?.Contains("formatted") ?? false).FirstOrDefault();

                HtmlNode imagecontainer;
                while ((imagecontainer = site.DocumentNode.Descendants("a").Where(A => A.Attributes["class"]?.Value?.Contains("golem-gallery2-nojs") ?? false).FirstOrDefault()) != null)
                {

                    string imageHtml = "<div>\n";
                    var imagesFromContainer = imagecontainer.Descendants("img");
                    foreach (var img in imagesFromContainer)
                    {
                        img.Attributes["src"].Value = img.Attributes["data-src-full"]?.Value ?? img.Attributes["data-src"]?.Value;
                        img.Attributes.Append("style", "max-width:600px");
                        imageHtml += img.OuterHtml + "<br/>\n";
                    }
                    imageHtml += "</div>\n";
                    var imagesNode = HtmlNode.CreateNode(imageHtml);
                    contentNode.ReplaceChild(imagesNode, imagecontainer);
                }


                if (contentNode != null && !string.IsNullOrEmpty(contentNode.InnerText) && !string.IsNullOrWhiteSpace(HtmlEntity.DeEntitize(contentNode.InnerText)))
                {
                    var nodes = contentNode.Elements("div").ToList();
                    foreach (var delNode in nodes)
                    {
                        contentNode.ChildNodes.Remove(delNode);
                    }

                    result.ContentRaw = string.Format(GenericHtml.HTML, contentNode.InnerHtml);
                }

                Func<string, string> getMetaContentByName = (string name) => site.DocumentNode.Descendants("meta")
                    ?.Where(A => A.Attributes["name"]?.Value?.ToLower() == name && !string.IsNullOrEmpty(A.Attributes["content"]?.Value))
                    ?.FirstOrDefault()?.Attributes["content"]?.Value;

                var twitter_image_src = getMetaContentByName("twitter:image:src");

                if (!string.IsNullOrEmpty(twitter_image_src))
                    result.Image = twitter_image_src;
            }

            return result;
        }

		public async Task<int> HTMLDownload (string stateRequested)
		{
			Log.Debug (TAG, "HTMLDownloadRunning");
			HtmlWeb htmlWeb = new HtmlWeb ();
			HtmlDocument htmlDoc = new HtmlDocument ();
			DateTime currentDateTime = DateTime.Now;
			TimeSpan currentTime = currentDateTime.TimeOfDay;
			int hourIndex = 3;
			int completeStatus = 0;

			if (regionsList.Count != 0) {
				regionsList.Clear ();
				APIList.Clear ();
			}

//			regionEntry.Clear ();
//			latestAPI.Clear();

			string hourRegion = string.Empty;
			//A problem will occur if it's 1st day of month at 12AM. Date is 0-3-2016
			string currentDay = string.Empty;
			if (currentDateTime.Day == 1) {
				currentDay = currentDateTime.Day.ToString ("D2");
			} else {
				currentDay = currentTime.Hours == 0 ? (currentDateTime.Day - 1).ToString ("D2") : currentDateTime.Day.ToString ("D2");
			}

			string date = currentDateTime.Year.ToString () + "-" + currentDateTime.Month.ToString ("D2") + "-" + currentDay;
			int currentHour = currentTime.Hours == 0 ? 24 : currentTime.Hours;

			if (currentHour > 0 && currentHour <= 6) { hourRegion = "hour1"; hourIndex += currentHour - 1; }
			else if (currentHour > 6 && currentHour <= 12) { hourRegion = "hour2"; hourIndex += (currentHour - 6 - 1); }
			else if (currentHour > 12 && currentHour <= 18) { hourRegion = "hour3"; hourIndex += (currentHour - 12 - 1);}
			else if (currentHour > 18 && currentHour <= 24) { hourRegion = "hour4"; hourIndex += (currentHour - 18 - 1);}

			string urlConstruct = "http://apims.doe.gov.my/v2/" + hourRegion + "_" + date + ".html";

			try {
				htmlDoc = await htmlWeb.LoadFromWebAsync(urlConstruct);
				var div = htmlDoc.GetElementbyId ("content");
				var table = div.Descendants ("table").ToList () [0].ChildNodes.ToList ();

				foreach (var tableEntry in table) {
					if (tableEntry.HasChildNodes) {
						var rowEntry = tableEntry.ChildNodes.ToList ();
						var stateEntry = rowEntry [0].InnerText.ToString ();
						if (stateEntry == stateRequested) {
							regionEntry.Add (rowEntry [2].InnerText.ToString ());
							latestAPI.Add (rowEntry [hourIndex].InnerText.ToString ());
						}
					}
				}
				completeStatus = 1;
			}
			catch (Exception e) {
				completeStatus = 0;
				errorString = e.Message.ToString();
			}
			Log.Debug (TAG, "HTMLDownloadFinish");
			return completeStatus;
		}