package com.vertigo.crawler import java.util.Random import com.vertigo.model.enums.{EntityType, Provider} import com.vertigo.model.json.sync.CrawlerTask import com.vertigo.test.util.RandomUtil import com.wrapper.spotify.models._ import rx.lang.scala.Observable import scala.collection.JavaConverters._ import scala.collection.{mutable ⇒ m} import scala.util /* Add to pom org.scala-tools maven-scala-plugin compile testCompile -Xms64m -Xmx1024m io.reactivex rxjava org.scala-lang scala-library 2.11.8 io.reactivex rxscala_2.11 0.26.3 */ object Example extends App { val random = new Random(69) val ru = new RandomUtil(random) val genres = m.Buffer( "aggrotech", "ebm", "electro-industrial", "futurepop", "ambient psychill", "psychill", "ambient", "metal", "crossover thrash", "death metal", "groove metal", "nwobhm", "speed metal", "thrash metal", "digital hardcore", "industrial", "industrial metal", "industrial rock", "neue deutsche harte", "album rock", "alternative metal", "classic rock", "hard rock", "nu metal", "power metal", "rock", "wrestling") val markets = m.Buffer( "AD", "AR", "AT", "AU", "BE", "BG", "BO", "BR", "CA", "CH", "CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "EC", "EE", "ES", "FI", "FR", "GB", "GR", "GT", "HK", "HN", "HU", "ID", "IE", "IS", "IT", "JP", "LI", "LT", "LU", "LV", "MC", "MT", "MX", "MY", "NI", "NL", "NO", "NZ", "PA", "PE", "PH", "PL", "PT", "PY", "SE", "SG", "SK", "SV", "TR", "TW", "US", "UY" ) val va: Artist = { val ar = new Artist() ar.setId("0LyfQWJT6nXafLPZqxe9Of") ar.setName("Various Artists") ar.setFollowers({ val f = new Followers() f.setHref(null) f.setTotal(215138) f }) ar.setExternalUrls({ val e = new ExternalUrls() e.getExternalUrls.put("spotify", "https://open.spotify.com/artist/" + ar.getId) e }) ar.setGenres(Seq.empty.asJava) ar.setHref("https://api.spotify.com/v1/artists/" + ar.getId) ar.setImages(Seq.empty.asJava) ar.setPopularity(0) ar.setUri("spotify:artist:" + ar.getId) ar } def simplify(a: Album): SimpleAlbum = { val sa = new SimpleAlbum() sa.setAlbumType(a.getAlbumType) sa.setExternalUrls(a.getExternalUrls) sa.setHref(a.getHref) sa.setId(a.getId) sa.setImages(a.getImages) sa.setName(a.getName) sa.setType(a.getType) sa.setUri(a.getUri) sa.setAvailableMarkets(a.getAvailableMarkets) sa } def simplify(t: Track): SimpleTrack = { val sa = new SimpleTrack() sa.setDiscNumber(t.getDiscNumber) sa.setDuration(t.getDuration) sa.setExplicit(t.isExplicit) sa.setExternalUrls(t.getExternalUrls) sa.setHref(t.getHref) sa.setId(t.getId) sa.setName(t.getName) sa.setPreviewUrl(t.getPreviewUrl) sa.setTrackNumber(t.getTrackNumber) sa.setType(t.getType) sa.setUri(t.getUri) sa.setAvailableMarkets(t.getAvailableMarkets) sa } def simplify(a: Artist): SimpleArtist = { val sa = new SimpleArtist() sa.setExternalUrls(a.getExternalUrls) sa.setHref(a.getHref) sa.setId(a.getId) sa.setName(a.getName) sa.setType(a.getType) sa.setUri(a.getUri) sa } /** * Select random elements from base list. * Must be count <= base.size() */ def listFrom[T](ru: RandomUtil, base: Seq[T], count: Int): Seq[T] = { val n = base.size val ts = base.toBuffer // remove random elements from there for (i ← 0 to (n - count)) { ts.remove(ru.nextInt(n - i)) } ts } def generateArtist(ru: RandomUtil): Artist = { val a = new Artist() a.setId(ru.string(8)) a.setName("artist_" + a.getId) a.setPopularity(ru.nextInt(100)) a.setExternalUrls({ val eu = new ExternalUrls() eu.getExternalUrls.put("spotify", "https://open.spotify.com/artist/" + a.getId) eu }) a.setHref("https://api.spotify.com/v1/artists/" + a.getId) a.setUri("spotify:artist:" + a.getId) a.setGenres(listFrom(ru, genres, 3).asJava) a.setFollowers({ val f = new Followers() f.setTotal(ru.nextInt(1000)) f }) a } def generateAlbum(ru: RandomUtil, albumType: AlbumType, creators: Seq[Artist]): Album = { val al = new Album() al.setId(ru.string(9)) al.setName("album_" + al.getId) al.setAlbumType(albumType) al.setHref("https://api.spotify.com/v1/albums/" + al.getId) al.setUri("spotify:album:" + al.getId) al.setCopyrights({ val c1 = new Copyright() c1.setText("2016 Alpha Matrix") c1.setText("C") val c2 = new Copyright() c2.setText("2016 Alpha Matrix") c2.setText("P") Seq(c1, c2).asJava }) al.setArtists(creators.map(simplify).asJava) al } def generateTrack(ru: RandomUtil, album: Album, trackNumber: Int): Track = { val t = new Track() t.setId(ru.string(10)) t.setName("track_" + t.getId) t.setHref("https://api.spotify.com/v1/tracks/" + t.getId) t.setUri("spotify:track:" + t.getId) t.setArtists(album.getArtists) t.setAlbum(simplify(album)) t.setTrackNumber(trackNumber) // from 1 to 1000 second t.setDuration((ru.nextInt(1000) + 1) * 1000) t.setPopularity(ru.nextInt(10)) t.setAvailableMarkets(album.getAvailableMarkets) t.setDiscNumber(0) t.setPreviewUrl("https://spotify.com/preview/" + t.getId) t } def randomTask(ru: RandomUtil): CrawlerTask = { // set only entityType and externalId, // the rest of them should be populated by val ct = new CrawlerTask() ct.setEntityType(EntityType.fromInt(ru.nextInt(11, 13))) ct.setExternalId(ru.string(10)) ct.setProvider(Provider.Spotify) ct } def randomTasks(ru: RandomUtil, sws: SpotifyWSImpl, count: Int): Seq[CrawlerTask] = { val keys1 = sws.artistMap.keys.toBuffer val keys2 = sws.albumMap.keys.toBuffer val keys3 = sws.trackMap.keys.toBuffer val n1 = keys1.size val n2 = keys2.size val n3 = keys3.size (1 to count).map { _ ⇒ val n: Int = ru.nextInt(n1 + n2 + n3) val (i, et) = if (n < n1) (n, EntityType.Artist) else if (n1 <= n && n < n2) (n - n1, EntityType.Album) else (n - n1 - n2, EntityType.Song) val eid = et match { case EntityType.Artist ⇒ sws.artistMap(keys1(i)).getId case EntityType.Album ⇒ sws.albumMap(keys2(i)).getId case EntityType.Song ⇒ sws.trackMap(keys3(i)).getId } val ct = new CrawlerTask() ct.setProvider(Provider.Spotify) ct.setExternalId(eid) ct.setEntityType(et) ct } } def ids(cts: Seq[CrawlerTask]): Seq[String] = cts.map(_.getExternalId) trait SpotifyWS { def getArtists(ids: Seq[String]): Seq[Artist] def getAlbums(ids: Seq[String]): Seq[Album] def getTracks(ids: Seq[String]): Seq[Track] } class RxWrapper(sws: SpotifyWS) { def obsArtists(ids: Seq[String]): Observable[Artist] = Observable.from(sws.getArtists(ids)) def obsAlbums(ids: Seq[String]): Observable[Album] = Observable.from(sws.getAlbums(ids)) def obsTracks(ids: Seq[String]): Observable[Track] = Observable.from(sws.getTracks(ids)) } class SpotifyWSImpl(ru: RandomUtil) extends SpotifyWS { val trackMap = new m.HashMap[String, Track]() val albumMap = new m.HashMap[String, Album]() val artistMap = new m.HashMap[String, Artist]() // go initMediaCollection() override def getArtists(ids: Seq[String]): Seq[Artist] = { val x = ru.nextInt(100) if (x < 10) throw new RuntimeException("timeout:" + Integer.toString(x)) else ids.map(artistMap.getOrElse(_, null)) } override def getAlbums(ids: Seq[String]): Seq[Album] = { val x = ru.nextInt(100) if (x < 10) throw new RuntimeException("timeout:" + Integer.toString(x)) else ids.map(albumMap.getOrElse(_, null)) } override def getTracks(ids: Seq[String]): Seq[Track] = { val x = ru.nextInt(100) if (x < 10) throw new RuntimeException("timeout:" + Integer.toString(x)) else ids.map(trackMap.getOrElse(_, null)) } def initMediaCollection(): Unit = { // create all val artists = (0 until 100).map(_ ⇒ generateArtist(ru)) // let each artist to have 5 albums // 3 exclusive, 1 with other, // 1 with other two // and many album-compilations where // song is made by different artist // e.g. album 3m6QKbHMF5GVBagWlCF1U0 val albums1 = artists.indices.flatMap({ i ⇒ val ar = artists(i) val ars = Seq(ar) artistMap.put(ar.getId, ar) val al1 = generateAlbum(ru, AlbumType.SINGLE, ars) val al2 = generateAlbum(ru, AlbumType.SINGLE, ars) val al3 = generateAlbum(ru, AlbumType.SINGLE, ars) Seq(al1, al2, al3) }).toBuffer util.Random.shuffle(artists) val albums2 = artists.view.indices.drop(1).map({ i ⇒ val ars = Seq(artists(i - 1), artists(i)) generateAlbum(ru, AlbumType.ALBUM, ars) }) // create albums of triples util.Random.shuffle(artists) val albums3 = artists.view.indices.drop(2).map({ i ⇒ val ars = Seq( artists(i - 2), artists(i - 1), artists(i) ) generateAlbum(ru, AlbumType.ALBUM, ars) }) // create compilations, // albums where the artist is "Various Artist" val albums4 = (0 until 200).view.map({ i ⇒ generateAlbum(ru, AlbumType.COMPILATION, Seq(va)) }) // glue all the stuff together val albums = albums1 ++ albums2 ++ albums3 ++ albums4 // now create tracks, let each album has 10 tracks val tracks = albums.flatMap(al ⇒ (1 to 10).map(generateTrack(ru, al, _))) // now put the stuff into the maps artists.foreach(ar ⇒ artistMap.put(ar.getId, ar)) albums.foreach(al ⇒ albumMap.put(al.getId, al)) tracks.foreach(t ⇒ trackMap.put(t.getId, t)) } } def run(): Unit = { val sws = new SpotifyWSImpl(ru) val rxw = new RxWrapper(sws) val tasks = randomTasks(ru, sws, 10) Observable.from(tasks) .filter(ct ⇒ ct.getEntityType == EntityType.Song) .tumblingBuffer(2) .flatMap(cts ⇒ rxw.obsTracks(ids(cts))) .flatMap(t ⇒ { // we got track, now we want the artists and album from it // val artists = t.getArtists // val album = t.getAlbum Observable.just(t) }) .subscribe(t ⇒ println(t)) } run() }