Pasted

package com.vertigo.crawler
 
import java.util.Random
 
import com.vertigo.model.enums.{EntityType, Provider}
import com.vertigo.model.json.sync.CrawlerTask
import com.vertigo.test.util.RandomUtil
import com.wrapper.spotify.models._
import rx.lang.scala.Observable
 
import scala.collection.JavaConverters._
import scala.collection.{mutable ⇒ m}
import scala.util
 
 
/*
    Add to pom
 
 
      <plugin>
        <groupId>org.scala-tools</groupId>
        <artifactId>maven-scala-plugin</artifactId>
        <executions>
          <execution>
            <goals>
              <goal>compile</goal>
              <goal>testCompile</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <jvmArgs>
            <jvmArg>-Xms64m</jvmArg>
            <jvmArg>-Xmx1024m</jvmArg>
          </jvmArgs>
        </configuration>
      </plugin>
 
    <dependency>
      <groupId>io.reactivex</groupId>
      <artifactId>rxjava</artifactId>
    </dependency>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>2.11.8</version>
    </dependency>
    <dependency>
      <groupId>io.reactivex</groupId>
      <artifactId>rxscala_2.11</artifactId>
      <version>0.26.3</version>
    </dependency>
 */
object Example extends App {
  val random = new Random(69)
  val ru = new RandomUtil(random)
  val genres = m.Buffer(
    "aggrotech", "ebm", "electro-industrial", "futurepop",
    "ambient psychill", "psychill", "ambient", "metal",
    "crossover thrash", "death metal", "groove metal",
    "nwobhm", "speed metal", "thrash metal", "digital hardcore",
    "industrial", "industrial metal", "industrial rock",
    "neue deutsche harte", "album rock", "alternative metal",
    "classic rock", "hard rock", "nu metal", "power metal",
    "rock", "wrestling")
  val markets = m.Buffer(
    "AD", "AR", "AT", "AU", "BE", "BG", "BO", "BR", "CA", "CH",
    "CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "EC", "EE",
    "ES", "FI", "FR", "GB", "GR", "GT", "HK", "HN", "HU", "ID",
    "IE", "IS", "IT", "JP", "LI", "LT", "LU", "LV", "MC", "MT",
    "MX", "MY", "NI", "NL", "NO", "NZ", "PA", "PE", "PH", "PL",
    "PT", "PY", "SE", "SG", "SK", "SV", "TR", "TW", "US", "UY"
  )
  val va: Artist = {
    val ar = new Artist()
    ar.setId("0LyfQWJT6nXafLPZqxe9Of")
    ar.setName("Various Artists")
    ar.setFollowers({
      val f = new Followers()
      f.setHref(null)
      f.setTotal(215138)
      f
    })
    ar.setExternalUrls({
      val e = new ExternalUrls()
      e.getExternalUrls.put("spotify",
        "https://open.spotify.com/artist/" + ar.getId)
      e
    })
    ar.setGenres(Seq.empty.asJava)
    ar.setHref("https://api.spotify.com/v1/artists/" + ar.getId)
    ar.setImages(Seq.empty.asJava)
    ar.setPopularity(0)
    ar.setUri("spotify:artist:" + ar.getId)
    ar
  }
 
  def simplify(a: Album): SimpleAlbum = {
    val sa = new SimpleAlbum()
    sa.setAlbumType(a.getAlbumType)
    sa.setExternalUrls(a.getExternalUrls)
    sa.setHref(a.getHref)
    sa.setId(a.getId)
    sa.setImages(a.getImages)
    sa.setName(a.getName)
    sa.setType(a.getType)
    sa.setUri(a.getUri)
    sa.setAvailableMarkets(a.getAvailableMarkets)
    sa
  }
 
  def simplify(t: Track): SimpleTrack = {
    val sa = new SimpleTrack()
    sa.setDiscNumber(t.getDiscNumber)
    sa.setDuration(t.getDuration)
    sa.setExplicit(t.isExplicit)
    sa.setExternalUrls(t.getExternalUrls)
    sa.setHref(t.getHref)
    sa.setId(t.getId)
    sa.setName(t.getName)
    sa.setPreviewUrl(t.getPreviewUrl)
    sa.setTrackNumber(t.getTrackNumber)
    sa.setType(t.getType)
    sa.setUri(t.getUri)
    sa.setAvailableMarkets(t.getAvailableMarkets)
    sa
  }
 
  def simplify(a: Artist): SimpleArtist = {
    val sa = new SimpleArtist()
    sa.setExternalUrls(a.getExternalUrls)
    sa.setHref(a.getHref)
    sa.setId(a.getId)
    sa.setName(a.getName)
    sa.setType(a.getType)
    sa.setUri(a.getUri)
    sa
  }
 
  /**
    * Select random elements from base list.
    * Must be count <= base.size()
    */
  def listFrom[T](ru: RandomUtil,
                  base: Seq[T],
                  count: Int): Seq[T] = {
    val n = base.size
    val ts = base.toBuffer
    // remove random elements from there
    for (i ← 0 to (n - count)) {
      ts.remove(ru.nextInt(n - i))
    }
    ts
  }
 
  def generateArtist(ru: RandomUtil): Artist = {
    val a = new Artist()
    a.setId(ru.string(8))
    a.setName("artist_" + a.getId)
    a.setPopularity(ru.nextInt(100))
    a.setExternalUrls({
        val eu = new ExternalUrls()
        eu.getExternalUrls.put("spotify",
          "https://open.spotify.com/artist/" + a.getId)
        eu
      })
    a.setHref("https://api.spotify.com/v1/artists/" + a.getId)
    a.setUri("spotify:artist:" + a.getId)
    a.setGenres(listFrom(ru, genres, 3).asJava)
    a.setFollowers({
      val f = new Followers()
      f.setTotal(ru.nextInt(1000))
      f
    })
    a
  }
 
  def generateAlbum(ru: RandomUtil,
                    albumType: AlbumType,
                    creators: Seq[Artist]): Album = {
    val al = new Album()
    al.setId(ru.string(9))
    al.setName("album_" + al.getId)
    al.setAlbumType(albumType)
    al.setHref("https://api.spotify.com/v1/albums/" + al.getId)
    al.setUri("spotify:album:" + al.getId)
    al.setCopyrights({
      val c1 = new Copyright()
      c1.setText("2016 Alpha Matrix")
      c1.setText("C")
      val c2 = new Copyright()
      c2.setText("2016 Alpha Matrix")
      c2.setText("P")
      Seq(c1, c2).asJava
    })
    al.setArtists(creators.map(simplify).asJava)
    al
  }
 
  def generateTrack(ru: RandomUtil,
                    album: Album,
                    trackNumber: Int): Track = {
    val t = new Track()
    t.setId(ru.string(10))
    t.setName("track_" + t.getId)
    t.setHref("https://api.spotify.com/v1/tracks/" + t.getId)
    t.setUri("spotify:track:" + t.getId)
    t.setArtists(album.getArtists)
    t.setAlbum(simplify(album))
    t.setTrackNumber(trackNumber)
    // from 1 to 1000 second
    t.setDuration((ru.nextInt(1000) + 1) * 1000)
    t.setPopularity(ru.nextInt(10))
    t.setAvailableMarkets(album.getAvailableMarkets)
    t.setDiscNumber(0)
    t.setPreviewUrl("https://spotify.com/preview/" + t.getId)
    t
  }
 
  def randomTask(ru: RandomUtil): CrawlerTask = {
    // set only entityType and externalId,
    // the rest of them should be populated by
    val ct = new CrawlerTask()
    ct.setEntityType(EntityType.fromInt(ru.nextInt(11, 13)))
    ct.setExternalId(ru.string(10))
    ct.setProvider(Provider.Spotify)
    ct
  }
 
  def randomTasks(ru: RandomUtil,
                  sws: SpotifyWSImpl,
                  count: Int): Seq[CrawlerTask] = {
    val keys1 = sws.artistMap.keys.toBuffer
    val keys2 = sws.albumMap.keys.toBuffer
    val keys3 = sws.trackMap.keys.toBuffer
    val n1 = keys1.size
    val n2 = keys2.size
    val n3 = keys3.size
    (1 to count).map { _ ⇒
      val n: Int = ru.nextInt(n1 + n2 + n3)
      val (i, et) =
        if (n < n1) (n, EntityType.Artist)
        else if (n1 <= n && n < n2) (n - n1, EntityType.Album)
        else (n - n1 - n2, EntityType.Song)
      val eid = et match {
        case EntityType.Artist ⇒ sws.artistMap(keys1(i)).getId
        case EntityType.Album ⇒ sws.albumMap(keys2(i)).getId
        case EntityType.Song ⇒ sws.trackMap(keys3(i)).getId
      }
      val ct = new CrawlerTask()
      ct.setProvider(Provider.Spotify)
      ct.setExternalId(eid)
      ct.setEntityType(et)
      ct
    }
  }
 
  def ids(cts: Seq[CrawlerTask]): Seq[String] =
    cts.map(_.getExternalId)
 
  trait SpotifyWS {
    def getArtists(ids: Seq[String]): Seq[Artist]
 
    def getAlbums(ids: Seq[String]): Seq[Album]
 
    def getTracks(ids: Seq[String]): Seq[Track]
  }
 
  class RxWrapper(sws: SpotifyWS) {
    def obsArtists(ids: Seq[String]): Observable[Artist] = Observable.from(sws.getArtists(ids))
 
    def obsAlbums(ids: Seq[String]): Observable[Album] = Observable.from(sws.getAlbums(ids))
 
    def obsTracks(ids: Seq[String]): Observable[Track] = Observable.from(sws.getTracks(ids))
  }
 
  class SpotifyWSImpl(ru: RandomUtil) extends SpotifyWS {
    val trackMap = new m.HashMap[String, Track]()
    val albumMap = new m.HashMap[String, Album]()
    val artistMap = new m.HashMap[String, Artist]()
    // go
    initMediaCollection()
 
    override def getArtists(ids: Seq[String]): Seq[Artist] = {
      val x = ru.nextInt(100)
      if (x < 10)
        throw new RuntimeException("timeout:" + Integer.toString(x))
      else
        ids.map(artistMap.getOrElse(_, null))
    }
 
    override def getAlbums(ids: Seq[String]): Seq[Album] = {
      val x = ru.nextInt(100)
      if (x < 10)
        throw new RuntimeException("timeout:" + Integer.toString(x))
      else
        ids.map(albumMap.getOrElse(_, null))
    }
 
    override def getTracks(ids: Seq[String]): Seq[Track] = {
      val x = ru.nextInt(100)
      if (x < 10)
        throw new RuntimeException("timeout:" + Integer.toString(x))
      else
        ids.map(trackMap.getOrElse(_, null))
    }
 
    def initMediaCollection(): Unit = {
      // create all
      val artists = (0 until 100).map(_ ⇒ generateArtist(ru))
      // let each artist to have 5 albums
      // 3 exclusive, 1 with other,
      // 1 with other two
      // and many album-compilations where
      // song is made by different artist
      // e.g. album 3m6QKbHMF5GVBagWlCF1U0
      val albums1 = artists.indices.flatMap({ i ⇒
        val ar = artists(i)
        val ars = Seq(ar)
        artistMap.put(ar.getId, ar)
        val al1 = generateAlbum(ru, AlbumType.SINGLE, ars)
        val al2 = generateAlbum(ru, AlbumType.SINGLE, ars)
        val al3 = generateAlbum(ru, AlbumType.SINGLE, ars)
        Seq(al1, al2, al3)
      }).toBuffer
 
      util.Random.shuffle(artists)
 
      val albums2 = artists.view.indices.drop(1).map({ i ⇒
        val ars = Seq(artists(i - 1), artists(i))
        generateAlbum(ru, AlbumType.ALBUM, ars)
      })
      // create albums of triples
      util.Random.shuffle(artists)
      val albums3 = artists.view.indices.drop(2).map({ i ⇒
        val ars = Seq(
          artists(i - 2),
          artists(i - 1),
          artists(i)
        )
        generateAlbum(ru, AlbumType.ALBUM, ars)
      })
      // create compilations,
      // albums where the artist is "Various Artist"
      val albums4 = (0 until 200).view.map({ i ⇒
        generateAlbum(ru, AlbumType.COMPILATION, Seq(va))
      })
      // glue all the stuff together
      val albums = albums1 ++ albums2 ++ albums3 ++ albums4
 
      // now create tracks, let each album has 10 tracks
      val tracks = albums.flatMap(al ⇒
        (1 to 10).map(generateTrack(ru, al, _)))
 
      // now put the stuff into the maps
      artists.foreach(ar ⇒ artistMap.put(ar.getId, ar))
      albums.foreach(al ⇒ albumMap.put(al.getId, al))
      tracks.foreach(t ⇒ trackMap.put(t.getId, t))
    }
  }
 
  def run(): Unit = {
 
    val sws = new SpotifyWSImpl(ru)
    val rxw = new RxWrapper(sws)
 
    val tasks = randomTasks(ru, sws, 10)
    Observable.from(tasks)
        .filter(ct ⇒ ct.getEntityType == EntityType.Song)
        .tumblingBuffer(2)
        .flatMap(cts ⇒ rxw.obsTracks(ids(cts)))
        .flatMap(t ⇒ {
          // we got track, now we want the artists and album from it
          // val artists = t.getArtists
          // val album = t.getAlbum
          Observable.just(t)
        })
        .subscribe(t ⇒ println(t))
  }
 
  run()
}