Merge lp:~elopio/u1-test-utils/musicsearch into lp:u1-test-utils

[Leo Arias]

Added the recipe to start a cerati music search server.

The cerati server is the missing point for our in dash payments.
With this branch, we add a known and small catalogue of songs that will appear on the dash and will redirect us to our local servers for more details and purchase.

[Vincent Ladeuil (vila)]

\o/

review: Approve

=== modified file 'setup_vm/NOTES'
--- setup_vm/NOTES	2013-04-23 15:22:48 +0000
+++ setup_vm/NOTES	2013-05-27 15:15:33 +0000
@@ -99,7 +99,8 @@
 
 13. Set up the Music Search server:
 
-  TODO. Do we need it?
+  # This server runs on the same U1 server machine.
+  $ ssh [e-mail] ~/bin/run-music-search
 
 13. Install the Curucú server (Only needed for Smart Scopes tests):
 

=== modified file 'setup_vm/u1/install'
--- setup_vm/u1/install	2013-05-11 23:44:48 +0000
+++ setup_vm/u1/install	2013-05-27 15:15:33 +0000
@@ -38,6 +38,11 @@
 
 [url]
 openid_sso_server: {sso.url}
+u1ms_search_base: {music_search.url}/v1/search
+
+[musicsearch]
+host: {music_search.address}
+port: {music_search.port}
 
 EOF
 # XXX The secrets file is overlayed, so we can't use the config file.
@@ -78,3 +83,92 @@
 # TODO ask on #u1-ops if there's a better way.
 sed -i 's/development-lazr.conf/local.conf/g' utilities/supervisor-dev.conf.tpl
 sed -i 's/development-appserver-lazr.conf/local.conf/g' utilities/supervisor-dev.conf.tpl
+
+# Create the music search catalogue.
+# The import script will get the full catalogue since last monday, and updates for the rest of the days.
+today=`date +%a`
+if [ "$today" = "Mon" ] 
+then
+    monday=`date +%Y%m%d`
+else
+    monday=`date -dlast-monday +%Y%m%d`
+fi
+# 496 is the id of the world shop. The other ids can be found on the cerati branch, cerati/parameters.py:MUSIC_PROVIDER_SHOP_IDS.
+# TODO should we add them? --elopio - 20130520
+monday_shop_dir=working_dir/$monday/shop-496
+mkdir -p $monday_shop_dir
+cat <<EOF >$monday_shop_dir/artist-full.csv
+artistId,name,popularity,tags,image,url
+32,Jimi Hendrix,0.63,"rock,world,spanish,pop-rock,1990s,1960s,2000s",http://cdn.7static.com/static/img/artistimages/00/000/000/0000000032_<\$size\$>.jpg,http://www.7digital.com/artist/jimi-hendrix/
+
+EOF
+gzip $monday_shop_dir/artist-full.csv
+
+cat <<EOF >$monday_shop_dir/artist-updates.csv
+action,artistId,name,popularity,tags,image,url
+
+EOF
+gzip $monday_shop_dir/artist-updates.csv
+
+cat <<EOF >$monday_shop_dir/release-full.csv
+releaseId,title,version,artistId,artistAppearsAs,barcode,type,year,explicitContent,trackCount,duration,tags,licensorId,image,dateAdded,releaseDate,labelId,labelName,formats,price,rrp,url
+1541733,Jimi Hendrix,,32,Jimi Hendrix,8718011752963,Album,2007,false,22,6906,rock,335,http://cdn.7static.com/static/img/sleeveart/00/015/417/0001541733_<\$size\$>.jpg,2012-01-06T10:58:52.093Z,2007-12-10T00:00:00Z,92464,Weton,"17,33,",9.99,9.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix
+
+EOF
+gzip $monday_shop_dir/release-full.csv
+
+cat <<EOF >$monday_shop_dir/release-updates.csv
+action,releaseId,title,version,artistId,artistAppearsAs,barcode,type,year,explicitContent,trackCount,duration,tags,licensorId,image,dateAdded,releaseDate,labelId,labelName,formats,price,rrp,url
+
+EOF
+gzip $monday_shop_dir/release-updates.csv
+
+cat <<EOF >$monday_shop_dir/track-full.csv
+trackId,title,version,type,isrc,explicitContent,trackNumber,discNumber,artistId,artistAppearsAs,releaseId,duration,formats,price,rrp,url,popularity,streamingReleaseDate
[phone],Foxy Lady,,Audio,NLHR50913714,false,1,1,32,The Jimi Hendrix Experience,1541733,366,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=01,0.35,2007-12-10T00:00:00Z
[phone],Sunshine Of Your Love,,Audio,NLHR50913719,false,2,1,32,The Jimi Hendrix Experience,1541733,409,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=02,0.27,2007-12-10T00:00:00Z
[phone],Room Full Of Mirrors,,Audio,NLHR50913727,false,3,1,32,The Jimi Hendrix Experience,1541733,174,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=03,0.3,2007-12-10T00:00:00Z
[phone],Fire,,Audio,NLHR50913721,false,7,1,32,The Jimi Hendrix Experience,1541733,234,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=07,0.32,2007-12-10T00:00:00Z
[phone],Outside Woman Blues,,Audio,NLHR50913711,false,9,1,32,Jimi Hendrix,1541733,466,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=09,0.27,2007-12-10T00:00:00Z
[phone],Purple Haze,,Audio,NLHR50913717,false,11,1,32,The Jimi Hendrix Experience,1541733,181,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=11,0.33,2007-12-10T00:00:00Z
[phone],Voodoo Chile (Slight Return),,Audio,NLHR50913722,false,12,1,32,The Jimi Hendrix Experience,1541733,549,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=12,0.35,2007-12-10T00:00:00Z
[phone],Bleeding Heart (Blues In C Sharp),,Audio,NLHR50913726,false,13,1,32,The Jimi Hendrix Experience,1541733,319,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=13,0.3,2007-12-10T00:00:00Z
[phone],Morrison's Lament,,Audio,NLHR50913709,false,15,1,32,Jimi Hendrix,1541733,485,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=15,0.27,2007-12-10T00:00:00Z
[phone],Little Wing,,Audio,NLHR50913731,false,17,1,32,The Jimi Hendrix Experience,1541733,194,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=17,0.32,2007-12-10T00:00:00Z
[phone],Wild Thing,,Audio,NLHR50913710,false,18,1,32,The Jimi Hendrix Experience,1541733,215,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=18,0.27,2007-12-10T00:00:00Z
[phone],Wake Up This Morning And Found Yourself Dead,,Audio,NLHR50913720,false,19,1,32,Jimi Hendrix,1541733,478,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=19,0.27,2007-12-10T00:00:00Z
[phone],Tomorrow Never Knows / Uranus Rock,,Audio,NLHR50913729,false,21,1,32,Jimi Hendrix,1541733,493,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=21,0.32,2007-12-10T00:00:00Z
[phone],Red House,,Audio,NLHR50913730,false,22,1,32,Jimi Hendrix,1541733,636,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=22,0.23,2007-12-10T00:00:00Z
[phone],Hot Trigger (A.k.a. Not Trigger Walking With Bessie Mae),,Audio,NLHR50913716,false,4,1,1195740,Lonnie Youngblood And The So-Called Jimi Hendrix Tapes,1541733,233,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=04,0.27,2007-12-10T00:00:00Z
[phone],Let The Good Times Roll,,Audio,NLHR50913725,false,6,1,1195740,Lonnie Youngblood And The So-Called Jimi Hendrix Tapes,1541733,365,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=06,0.27,2007-12-10T00:00:00Z
[phone],Freedom And You,,Audio,NLHR50913715,false,8,1,1195740,Lonnie Youngblood And The So-Called Jimi Hendrix Tapes,1541733,145,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=08,0.27,2007-12-10T00:00:00Z
[phone],Good Feeling,,Audio,NLHR50913724,false,16,1,1195740,Lonnie Youngblood And The So-Called Jimi Hendrix Tapes,1541733,255,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=16,0.27,2007-12-10T00:00:00Z
[phone],Wipe The Sweat Part 2,,Audio,NLHR50913728,false,5,1,1195742,Jimi Hendrix feat. Lonnie Youngblood,1541733,169,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=05,0.27,2007-12-10T00:00:00Z
[phone],Soul Food (That's What I Like),,Audio,NLHR50913713,false,10,1,1195742,Jimi Hendrix feat. Lonnie Youngblood,1541733,210,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=10,0.27,2007-12-10T00:00:00Z
[phone],She's A Fox,,Audio,NLHR50913732,false,14,1,1195742,Jimi Hendrix feat. Lonnie Youngblood,1541733,160,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=14,0.3,2007-12-10T00:00:00Z
[phone],Go Go Shoes,,Audio,NLHR50913718,false,20,1,1195742,Jimi Hendrix feat. Lonnie Youngblood,1541733,170,"17,33",0.99,0.99,http://xw.7digital.com/artist/jimi-hendrix/release/jimi-hendrix?h=20,0.27,2007-12-10T00:00:00Z
+
+EOF
+gzip $monday_shop_dir/track-full.csv
+
+cat <<EOF >$monday_shop_dir/track-updates.csv
+action,trackId,title,version,type,isrc,explicitContent,trackNumber,discNumber,artistId,artistAppearsAs,releaseId,duration,formats,price,rrp,url,popularity,streamingReleaseDate
+
+EOF
+gzip $monday_shop_dir/track-updates.csv
+
+# Install the music search server.
+cd ~
+# Get the branch.
+bzr branch lp:cerati {music_search.src_dir}
+# Setup the environment.
+cd {music_search.src_dir}
+cat <<EOF > configs/local-lazr.conf
+[general]
+environment_name: development
+base_url: {music_search.url}
+web_music_store_url: {u1.url}
+
+EOF
+
+SOURCEDEPS_DIR=../ubunet-sourcedeps/ make

=== added file 'setup_vm/u1/run-music-search'
--- setup_vm/u1/run-music-search	1970-01-01 00:00:00 +0000
+++ setup_vm/u1/run-music-search	2013-05-27 15:15:33 +0000
@@ -0,0 +1,18 @@
+#!/bin/sh -ex
+
+cd ~/{u1.src_dir}
+make start-solr
+# Import the 7digital catalog.
+today=`date +%a`
+if [ "$today" = "Mon" ] 
+then
+    monday=`date +%Y%m%d`
+else
+    monday=`date -dlast-monday +%Y%m%d`
+fi
+# 496 is the id of the world shop. The other ids can be found on the cerati branch, cerati/parameters.py:MUSIC_PROVIDER_SHOP_IDS.
+# TODO should we add them? --elopio - 20130520
+./utilities/seven_digital/manage.py import_7d_to_solr canonical {music_search.token} --shop-ids 496 --solr-port `ps fax | grep jetty.port | tail -n 1 | sed 's/.*jetty.port=\([0-9]*\).*/\1/'` --limit 1000 --date $monday
+cd ~/{music_search.src_dir}
+echo `ps fax | grep jetty.port | tail -n 1 | sed 's/.*jetty.port=\([0-9]*\).*/\1/'` > tmp/jetty.port
+PYTHONPATH=.:lib:$PYTHONPATH U1CONFIG=configs/local-lazr.conf gunicorn cerati.views:app -b 0.0.0.0:{music_search.port}

=== modified file 'setup_vm/unity/install-sources'
--- setup_vm/unity/install-sources	2013-05-11 02:05:59 +0000
+++ setup_vm/unity/install-sources	2013-05-27 15:15:33 +0000
@@ -38,5 +38,8 @@
 
 [ubuntuone]
 ubuntuone_server_url = {u1.url}
+
+[musicsearch]
+musicsearch_server_url = {music_search.url}
     
 EOF

=== modified file 'setup_vm/vms.conf'
--- setup_vm/vms.conf	2013-05-17 17:08:15 +0000
+++ setup_vm/vms.conf	2013-05-27 15:15:33 +0000
@@ -2,6 +2,8 @@
 # sso.address=sso.local
 # pay.address=pay.local
 # u1.address=u1.local
+# The token is stored on the cerati branch, cerati/parameters.py:MUSIC_PROVIDER_OAUTH_CONSUMER_SECRET
+# music_search.token=the-7digital-token
 # ppa.ubuntuone-hackers.password
 
 sso.src_dir=canonical-identity-provider
@@ -20,6 +22,11 @@
 u1.port=8003
 u1.url=http://{u1.address}:{u1.port}
 
+music_search.src_dir=cerati
+music_search.port=8004
+music_search.address={u1.address}
+music_search.url=http://{music_search.address}:{music_search.port}
+
 [precise-server-pristine]
 vm.name=precise-server-pristine
 vm.release=precise
@@ -49,10 +56,10 @@
 vm.release=precise
 vm.backing=precise-server-pristine.qcow2
 vm.apt_sources={ppa.ubuntuone_hackers}
-vm.packages=openjdk-7-jre,ubuntuone-developer-dependencies
+vm.packages=openjdk-7-jre, ubuntuone-developer-dependencies, jetty, libjetty-extra-java, solr-jetty, gunicorn
 vm.ubuntu_script=u1/install
 vm.update=True
-vm.uploaded_scripts=u1/run
+vm.uploaded_scripts=u1/run, u1/run-music-search
 
 [raring-desktop-pristine]
 vm.name=raring-desktop-pristine
@@ -70,7 +77,7 @@
 vm.release=raring
 vm.backing=raring-desktop-pristine.qcow2
 vm.apt_sources=deb http://ppa.launchpad.net/ubuntuone/dashpurchase-testing/ubuntu {vm.release} main|4BD0ECAE,deb http://ppa.launchpad.net/vila/selenium/ubuntu {vm.release} main|5703355D,ppa:ubuntuone/nightlies
-vm.packages=python-selenium, python-sst, ubuntuone-dev-tools, libautopilot-qt, python-requests, python-mock, python-django, python-django-configglue, python-piston-mini-client, xvfb
+vm.packages=python-selenium, python-sst, ubuntuone-dev-tools, libautopilot-qt, python-requests, python-mock, python-django, python-django-configglue, python-piston-mini-client, python-dateutil, xvfb
 vm.uploaded_scripts=unity/run-sso-client, unity/run-unity-lens-music
 # TODO unity/run-syncdaemon. We don't yet have the hermetic filesync server.
 vm.update=True