package com.vertigo.crawler
import java.util.Random
import com.vertigo.model.enums.{EntityType, Provider}
import com.vertigo.model.json.sync.CrawlerTask
import com.vertigo.test.util.RandomUtil
import com.wrapper.spotify.models._
import rx.lang.scala.Observable
import scala.collection.JavaConverters._
import scala.collection.{mutable ⇒ m}
import scala.util
/*
Add to pom
org.scala-tools
maven-scala-plugin
compile
testCompile
-Xms64m
-Xmx1024m
io.reactivex
rxjava
org.scala-lang
scala-library
2.11.8
io.reactivex
rxscala_2.11
0.26.3
*/
object Example extends App {
val random = new Random(69)
val ru = new RandomUtil(random)
val genres = m.Buffer(
"aggrotech", "ebm", "electro-industrial", "futurepop",
"ambient psychill", "psychill", "ambient", "metal",
"crossover thrash", "death metal", "groove metal",
"nwobhm", "speed metal", "thrash metal", "digital hardcore",
"industrial", "industrial metal", "industrial rock",
"neue deutsche harte", "album rock", "alternative metal",
"classic rock", "hard rock", "nu metal", "power metal",
"rock", "wrestling")
val markets = m.Buffer(
"AD", "AR", "AT", "AU", "BE", "BG", "BO", "BR", "CA", "CH",
"CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "EC", "EE",
"ES", "FI", "FR", "GB", "GR", "GT", "HK", "HN", "HU", "ID",
"IE", "IS", "IT", "JP", "LI", "LT", "LU", "LV", "MC", "MT",
"MX", "MY", "NI", "NL", "NO", "NZ", "PA", "PE", "PH", "PL",
"PT", "PY", "SE", "SG", "SK", "SV", "TR", "TW", "US", "UY"
)
val va: Artist = {
val ar = new Artist()
ar.setId("0LyfQWJT6nXafLPZqxe9Of")
ar.setName("Various Artists")
ar.setFollowers({
val f = new Followers()
f.setHref(null)
f.setTotal(215138)
f
})
ar.setExternalUrls({
val e = new ExternalUrls()
e.getExternalUrls.put("spotify",
"https://open.spotify.com/artist/" + ar.getId)
e
})
ar.setGenres(Seq.empty.asJava)
ar.setHref("https://api.spotify.com/v1/artists/" + ar.getId)
ar.setImages(Seq.empty.asJava)
ar.setPopularity(0)
ar.setUri("spotify:artist:" + ar.getId)
ar
}
def simplify(a: Album): SimpleAlbum = {
val sa = new SimpleAlbum()
sa.setAlbumType(a.getAlbumType)
sa.setExternalUrls(a.getExternalUrls)
sa.setHref(a.getHref)
sa.setId(a.getId)
sa.setImages(a.getImages)
sa.setName(a.getName)
sa.setType(a.getType)
sa.setUri(a.getUri)
sa.setAvailableMarkets(a.getAvailableMarkets)
sa
}
def simplify(t: Track): SimpleTrack = {
val sa = new SimpleTrack()
sa.setDiscNumber(t.getDiscNumber)
sa.setDuration(t.getDuration)
sa.setExplicit(t.isExplicit)
sa.setExternalUrls(t.getExternalUrls)
sa.setHref(t.getHref)
sa.setId(t.getId)
sa.setName(t.getName)
sa.setPreviewUrl(t.getPreviewUrl)
sa.setTrackNumber(t.getTrackNumber)
sa.setType(t.getType)
sa.setUri(t.getUri)
sa.setAvailableMarkets(t.getAvailableMarkets)
sa
}
def simplify(a: Artist): SimpleArtist = {
val sa = new SimpleArtist()
sa.setExternalUrls(a.getExternalUrls)
sa.setHref(a.getHref)
sa.setId(a.getId)
sa.setName(a.getName)
sa.setType(a.getType)
sa.setUri(a.getUri)
sa
}
/**
* Select random elements from base list.
* Must be count <= base.size()
*/
def listFrom[T](ru: RandomUtil,
base: Seq[T],
count: Int): Seq[T] = {
val n = base.size
val ts = base.toBuffer
// remove random elements from there
for (i ← 0 to (n - count)) {
ts.remove(ru.nextInt(n - i))
}
ts
}
def generateArtist(ru: RandomUtil): Artist = {
val a = new Artist()
a.setId(ru.string(8))
a.setName("artist_" + a.getId)
a.setPopularity(ru.nextInt(100))
a.setExternalUrls({
val eu = new ExternalUrls()
eu.getExternalUrls.put("spotify",
"https://open.spotify.com/artist/" + a.getId)
eu
})
a.setHref("https://api.spotify.com/v1/artists/" + a.getId)
a.setUri("spotify:artist:" + a.getId)
a.setGenres(listFrom(ru, genres, 3).asJava)
a.setFollowers({
val f = new Followers()
f.setTotal(ru.nextInt(1000))
f
})
a
}
def generateAlbum(ru: RandomUtil,
albumType: AlbumType,
creators: Seq[Artist]): Album = {
val al = new Album()
al.setId(ru.string(9))
al.setName("album_" + al.getId)
al.setAlbumType(albumType)
al.setHref("https://api.spotify.com/v1/albums/" + al.getId)
al.setUri("spotify:album:" + al.getId)
al.setCopyrights({
val c1 = new Copyright()
c1.setText("2016 Alpha Matrix")
c1.setText("C")
val c2 = new Copyright()
c2.setText("2016 Alpha Matrix")
c2.setText("P")
Seq(c1, c2).asJava
})
al.setArtists(creators.map(simplify).asJava)
al
}
def generateTrack(ru: RandomUtil,
album: Album,
trackNumber: Int): Track = {
val t = new Track()
t.setId(ru.string(10))
t.setName("track_" + t.getId)
t.setHref("https://api.spotify.com/v1/tracks/" + t.getId)
t.setUri("spotify:track:" + t.getId)
t.setArtists(album.getArtists)
t.setAlbum(simplify(album))
t.setTrackNumber(trackNumber)
// from 1 to 1000 second
t.setDuration((ru.nextInt(1000) + 1) * 1000)
t.setPopularity(ru.nextInt(10))
t.setAvailableMarkets(album.getAvailableMarkets)
t.setDiscNumber(0)
t.setPreviewUrl("https://spotify.com/preview/" + t.getId)
t
}
def randomTask(ru: RandomUtil): CrawlerTask = {
// set only entityType and externalId,
// the rest of them should be populated by
val ct = new CrawlerTask()
ct.setEntityType(EntityType.fromInt(ru.nextInt(11, 13)))
ct.setExternalId(ru.string(10))
ct.setProvider(Provider.Spotify)
ct
}
def randomTasks(ru: RandomUtil,
sws: SpotifyWSImpl,
count: Int): Seq[CrawlerTask] = {
val keys1 = sws.artistMap.keys.toBuffer
val keys2 = sws.albumMap.keys.toBuffer
val keys3 = sws.trackMap.keys.toBuffer
val n1 = keys1.size
val n2 = keys2.size
val n3 = keys3.size
(1 to count).map { _ ⇒
val n: Int = ru.nextInt(n1 + n2 + n3)
val (i, et) =
if (n < n1) (n, EntityType.Artist)
else if (n1 <= n && n < n2) (n - n1, EntityType.Album)
else (n - n1 - n2, EntityType.Song)
val eid = et match {
case EntityType.Artist ⇒ sws.artistMap(keys1(i)).getId
case EntityType.Album ⇒ sws.albumMap(keys2(i)).getId
case EntityType.Song ⇒ sws.trackMap(keys3(i)).getId
}
val ct = new CrawlerTask()
ct.setProvider(Provider.Spotify)
ct.setExternalId(eid)
ct.setEntityType(et)
ct
}
}
def ids(cts: Seq[CrawlerTask]): Seq[String] =
cts.map(_.getExternalId)
trait SpotifyWS {
def getArtists(ids: Seq[String]): Seq[Artist]
def getAlbums(ids: Seq[String]): Seq[Album]
def getTracks(ids: Seq[String]): Seq[Track]
}
class RxWrapper(sws: SpotifyWS) {
def obsArtists(ids: Seq[String]): Observable[Artist] = Observable.from(sws.getArtists(ids))
def obsAlbums(ids: Seq[String]): Observable[Album] = Observable.from(sws.getAlbums(ids))
def obsTracks(ids: Seq[String]): Observable[Track] = Observable.from(sws.getTracks(ids))
}
class SpotifyWSImpl(ru: RandomUtil) extends SpotifyWS {
val trackMap = new m.HashMap[String, Track]()
val albumMap = new m.HashMap[String, Album]()
val artistMap = new m.HashMap[String, Artist]()
// go
initMediaCollection()
override def getArtists(ids: Seq[String]): Seq[Artist] = {
val x = ru.nextInt(100)
if (x < 10)
throw new RuntimeException("timeout:" + Integer.toString(x))
else
ids.map(artistMap.getOrElse(_, null))
}
override def getAlbums(ids: Seq[String]): Seq[Album] = {
val x = ru.nextInt(100)
if (x < 10)
throw new RuntimeException("timeout:" + Integer.toString(x))
else
ids.map(albumMap.getOrElse(_, null))
}
override def getTracks(ids: Seq[String]): Seq[Track] = {
val x = ru.nextInt(100)
if (x < 10)
throw new RuntimeException("timeout:" + Integer.toString(x))
else
ids.map(trackMap.getOrElse(_, null))
}
def initMediaCollection(): Unit = {
// create all
val artists = (0 until 100).map(_ ⇒ generateArtist(ru))
// let each artist to have 5 albums
// 3 exclusive, 1 with other,
// 1 with other two
// and many album-compilations where
// song is made by different artist
// e.g. album 3m6QKbHMF5GVBagWlCF1U0
val albums1 = artists.indices.flatMap({ i ⇒
val ar = artists(i)
val ars = Seq(ar)
artistMap.put(ar.getId, ar)
val al1 = generateAlbum(ru, AlbumType.SINGLE, ars)
val al2 = generateAlbum(ru, AlbumType.SINGLE, ars)
val al3 = generateAlbum(ru, AlbumType.SINGLE, ars)
Seq(al1, al2, al3)
}).toBuffer
util.Random.shuffle(artists)
val albums2 = artists.view.indices.drop(1).map({ i ⇒
val ars = Seq(artists(i - 1), artists(i))
generateAlbum(ru, AlbumType.ALBUM, ars)
})
// create albums of triples
util.Random.shuffle(artists)
val albums3 = artists.view.indices.drop(2).map({ i ⇒
val ars = Seq(
artists(i - 2),
artists(i - 1),
artists(i)
)
generateAlbum(ru, AlbumType.ALBUM, ars)
})
// create compilations,
// albums where the artist is "Various Artist"
val albums4 = (0 until 200).view.map({ i ⇒
generateAlbum(ru, AlbumType.COMPILATION, Seq(va))
})
// glue all the stuff together
val albums = albums1 ++ albums2 ++ albums3 ++ albums4
// now create tracks, let each album has 10 tracks
val tracks = albums.flatMap(al ⇒
(1 to 10).map(generateTrack(ru, al, _)))
// now put the stuff into the maps
artists.foreach(ar ⇒ artistMap.put(ar.getId, ar))
albums.foreach(al ⇒ albumMap.put(al.getId, al))
tracks.foreach(t ⇒ trackMap.put(t.getId, t))
}
}
def run(): Unit = {
val sws = new SpotifyWSImpl(ru)
val rxw = new RxWrapper(sws)
val tasks = randomTasks(ru, sws, 10)
Observable.from(tasks)
.filter(ct ⇒ ct.getEntityType == EntityType.Song)
.tumblingBuffer(2)
.flatMap(cts ⇒ rxw.obsTracks(ids(cts)))
.flatMap(t ⇒ {
// we got track, now we want the artists and album from it
// val artists = t.getArtists
// val album = t.getAlbum
Observable.just(t)
})
.subscribe(t ⇒ println(t))
}
run()
}