Scala Element类代码示例

//设置package包名称以及导入依赖的类
package org.oginskis.ss.tool

import net.ruippeixotog.scalascraper.browser.JsoupBrowser
import net.ruippeixotog.scalascraper.model.Element
import org.jsoup.Connection
import org.oginskis.ss.model.Flat


object FlatExtractor {

  val SS_LV_BASE ="ss.lv.base.url"

  class ExtendedJsoupBrowser extends JsoupBrowser {
    override protected[this] def defaultRequestSettings(conn: Connection): Connection = {
      super.defaultRequestSettings(conn)
      conn.followRedirects(false)
    }
  }

  val browser = new ExtendedJsoupBrowser()

  def extractFlats() : List[Flat] = {
    def extractFlats(page: Int) : List[Flat] = {
      try {
        val doc = browser.get(Properties.getProperty(SS_LV_BASE)+"/riga/centre/sell/page"
            + page + ".html")
        val rawList: Iterable[Element] = doc.body.select("[id^=\"tr_\"]")
        rawList.init.toList.map(
          entry => {
            val attr: List[Element] = entry.select(".msga2-o").toList
            val link: String = entry.select(".msg2 .d1 .am").head.attr("href")
            new Flat(Option(attr(0).text.replace("\\", "/")),
              Option(attr(1).text.trim.replace("\\", "/")),
              Option(attr(2).text.trim.toInt),
              Option(attr(3).text.replace("\\", "/")),
              Option(attr(6).text.replace(",","").replace(" €","").trim.toInt),
              Option(link))
          }) ::: extractFlats(page + 1)
      }
      catch {
        case unknown: Throwable => {
          List[Flat]()
        }
      }
    }
    extractFlats(1)
  }



} 

//设置package包名称以及导入依赖的类
package net.seabears.hockey

import java.time.{LocalDate, LocalTime, ZonedDateTime}
import java.time.format.DateTimeFormatter

import net.ruippeixotog.scalascraper.browser.JsoupBrowser
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
import net.ruippeixotog.scalascraper.dsl.DSL.Parse._
import net.ruippeixotog.scalascraper.model.Element

import net.seabears.hockey.core._
import net.seabears.hockey.util.DateUtils

object Scheduler {
  def apply(adapterFactory: Game => GameAdapter, dateStart: String, dateEnd: String, host: String)(implicit userAgentFactory: () => String, pauseFactory: () => Unit) =
    new Scheduler(adapterFactory, host, DateUtils.dates(dateStart, dateEnd), userAgentFactory, pauseFactory)
}

class Scheduler(adapterFactory: Game => GameAdapter, host: String, dates: Seq[LocalDate], userAgentFactory: () => String, pauseFactory: () => Unit) {
  private[this] val formatter = DateTimeFormatter.ofPattern("yyyyMMdd")
  private[this] val browser = new JsoupBrowser(userAgentFactory())

  def run() {
    dates.flatMap(getGames)
         .map(adapterFactory)
         .filter(_.isNew)
         .foreach(_.save)
  }

  private[this] def getGames(date: LocalDate): List[FutureGame] = {
    val dayId = date.format(formatter)
    val url = host + dayId
    println("Searching for games at " + url)
    val doc = browser.get(url)
    val tables: List[Element] = doc >> elementList("div.game-header")
    tables.map(toGame(date))
  }

  private def toGame(date: LocalDate)(element: Element): FutureGame = {
    pauseFactory()
    val away = element.select("table.game-header-table tr:nth-child(1) td.team-name").head.text
    val home = element.select("table.game-header-table tr:nth-child(3) td.team-name").head.text
    val time = element.select("ul.game-info li:nth-child(2) span:first-child").head.text
    FutureGame(Team("", home), Team("", away), parseTime(date, time))
  }

  private def parseTime(date: LocalDate, timeToParse: String) = {
    val gameTime = """^\s*(\d+:\d+\s+\w+)\s+(\w+)\s*$""".r
    val gameTime(rawTime, rawZone) = timeToParse
    val time = LocalTime.parse(rawTime, DateTimeFormatter.ofPattern("h:mm a"))
    ZonedDateTime.of(date, time, DateUtils.parseZone(rawZone))
  }
} 

//设置package包名称以及导入依赖的类
package services

import models.CouncilIn
import net.ruippeixotog.scalascraper.browser.JsoupBrowser
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.model.Element
import net.ruippeixotog.scalascraper.dsl.DSL.Extract._


  def marrickville = {
    def oneElement(e:Element) = {
      val rows = e.flatMap(_ >> elements("tr"))
      val tdrows = rows.flatMap(_ >> elements("td"))
      val tds = tdrows.map(_ >> text("td")).toList
      CouncilIn(tds(1), tds(5), tds(7), tds(3))
    }

    val browser = JsoupBrowser()
    val urlStr = "https://eproperty.marrickville.nsw.gov.au/eServices/P1/PublicNotices/AllPublicNotices.aspx?r=MC.P1.WEBGUEST&f=%24P1.ESB.PUBNOTAL.ENQ"
    val doc = browser.get(urlStr)

    // Extract the elements with name
    val grid = doc >> element("#ctl00_Content_cusApplicationResultsGrid_pnlCustomisationGrid")
    val items = grid >> elements(".grid")
    items.map(e => oneElement(e)).toList

  }

  // scrape blacktown
  def blacktown = {
    def oneElement(e:Element) = {
      val rows = e.flatMap(_ >> elements("tr"))
      val tdrows = rows.flatMap(_ >> elements("td"))
      val tds = tdrows.map(_ >> text("td")).toList
      CouncilIn(tds(1), tds(2), tds(3), tds(4))
    }

    val browser = JsoupBrowser()
    val urlStr = "http://www.blacktown.nsw.gov.au/Planning_and_Development/Development_Assessment/Development_Online/Developments_on_Notification"
    val doc = browser.get(urlStr)

    // Extract the elements with name
    val main = doc >> element("#main-content")
    val body = main >> element(".body-content")
    val items = body >> elements("table")
    items.map(e => oneElement(e)).toList

  }

}