Merge lp:~laurentf/arte+7recorder/arte+7recorder into lp:arte+7recorder

[beudbeud]

[beudbeud (beudbeud)]

review: Approve

=== modified file 'arte7recorder/Catalog.py'
--- arte7recorder/Catalog.py	2010-10-06 09:25:40 +0000
+++ arte7recorder/Catalog.py	2013-11-13 20:38:32 +0000
@@ -93,6 +93,8 @@
             video = dict()
             for h in i.findAll('h2'):
                 for a in h.findAll('a'):
+                    if a.string == None:
+                        continue
                     video['targetURL'] = self.ARTE_WEB_ROOT + a['href']
                     video['targetURL'] = video['targetURL'].replace("/fr/", lang)
                     video['bigTitle'] = unescape_html( a.string )

=== modified file 'arte7recorder/arte7recorder.py'
--- arte7recorder/arte7recorder.py	2010-10-06 09:25:40 +0000
+++ arte7recorder/arte7recorder.py	2013-11-13 20:38:32 +0000
@@ -16,6 +16,7 @@
 import gettext
 import pygtk
 import BeautifulSoup as BS
+import json
 pygtk.require('2.0')
 
 from Catalog import Catalog, unescape_html, get_lang
@@ -39,18 +40,94 @@
 def get_rtmp_url( url_page, quality ):
     page_soup = BS.BeautifulSoup( urllib2.urlopen(url_page).read() )
 
-    movie_object = page_soup.find("object", classid="clsid:d27cdb6e-ae6d-11cf-96b8-444553540000")
-    movie = movie_object.find("param", {"name":"movie"})
-    movie_url = "http" + unescape_xml(movie['value'].split("http")[-1])
-
-    xml_soup = BS.BeautifulStoneSoup( urllib2.urlopen(movie_url).read() )
-    movie_url = xml_soup.find("video", {'lang': get_lang()})['ref']
-
-    xml_soup = BS.BeautifulStoneSoup( urllib2.urlopen(movie_url).read() )
-    base_soup = xml_soup.find("urls")
-    movie_url = base_soup.find("url", {"quality": quality}).string
+    vc = page_soup.find("div", {"class":"video-container"})
+    vc_url = vc['arte_vp_url']
+    #json_soup = BS.BeautifulStoneSoup
+    json_tree = json.load( urllib2.urlopen(vc_url) )
+    #obj = json_tree["videoJsonPlayer"]["VSR"]["RTMP_SQ_1"]
+    #movie_url = obj["streamer"]+obj["url"]
+    obj = json_tree["videoJsonPlayer"]["VSR"]["HTTP_REACH_EQ_1"]
+    movie_url = obj["url"]
+    ##FIXME!!! implement quality and language!!
+    #print "-->",json_tree.videoJsonPlayer.VSR
+    #movie_object = page_soup.find("object", classid="clsid:d27cdb6e-ae6d-11cf-96b8-444553540000")
+    #movie = movie_object.find("param", {"name":"movie"})
+    #movie_url = "http" + unescape_xml(movie['value'].split("http")[-1])
+
+    #xml_soup = BS.BeautifulStoneSoup( urllib2.urlopen(movie_url).read() )
+    #movie_url = xml_soup.find("video", {'lang': get_lang()})['ref']
+
+    #xml_soup = BS.BeautifulStoneSoup( urllib2.urlopen(movie_url).read() )
+    #base_soup = xml_soup.find("urls")
+    #movie_url = base_soup.find("url", {"quality": quality}).string
     return movie_url
 
+def http_download( link, destination = "/dev/null", try_resume = True, resuming = False ):
+    global subprocess_pid
+    #print link, destination
+    some_dl_done = False
+    need_more_dl = True
+
+    if try_resume and os.path.isfile( destination ):
+        for percent in http_download(link, destination, False, True ):
+            if percent != -1:
+                some_dl_done = True
+                need_more_dl = percent != 100.0
+                yield percent
+            else:
+                break
+
+    max_skip_cnt = 15
+    cmd_dl = 'wget "%s" -O "%s"' % (link, destination)
+    cmd_resume = 'wget -c "%s" -O "%s"' % (link, destination)
+    SECONDS_TO_WAIT = 5
+    #percent_re = re.compile("\((.+)%\)$")
+    percent_re = re.compile("([0-9,]+)%")
+
+    ret_code = None
+    if some_dl_done or resuming:
+        cmd = cmd_resume
+    else:
+        cmd = cmd_dl
+    while need_more_dl:
+        stderr_buff = ""
+        whole_stderr_buff = ""
+        p = subprocess.Popen( cmd, shell=True, stderr=subprocess.PIPE, close_fds=True)
+        subprocess_pid = p.pid + 1
+        while ret_code is None:
+            fds_read, fds_write, fds_exception = select.select([p.stderr],[], [], SECONDS_TO_WAIT)
+            if len(fds_read) == 1:
+                #print p.stderr.read(100),fds_read,fds_write
+                c = p.stderr.read(1)
+                whole_stderr_buff += c
+                if c in ("\n","\r"):
+                    match = percent_re.search( stderr_buff )
+                    if max_skip_cnt == 0:
+                        yield -1.0
+                    if match is not None:
+                        max_skip_cnt = 15
+                        yield float(match.group(1))
+                    else:
+                        max_skip_cnt -= 1
+                    stderr_buff = ""
+                else:
+                    stderr_buff += c
+            ret_code = p.poll()
+        whole_stderr_buff += p.stderr.read()
+        subprocess_pid = None
+        if ret_code == 0:
+            yield 100.0
+            break
+        elif ret_code == 2:
+            cmd = cmd_resume
+        else:
+            print ret_code
+            print whole_stderr_buff
+            print
+            yield -1.0
+        ret_code = None
+
+
 def rtmp_download( link, destination = "/dev/null", try_resume = True, resuming =False ):
     global subprocess_pid
     some_dl_done = False
@@ -125,7 +202,7 @@
   wmvRE = re.compile('availableFormats.*=.*"(.*HQ.*wmv.*)"')
   mmsRE = re.compile('"(mms.*)"')
   resumeRE = re.compile('<p class="text">([^<]*)<')
-  dureeRE = re.compile('[^0-9]*([0-9]+)(mn|min)')
+  dureeRE = re.compile('[^0-9]*([0-9]+) (mn|min)')
         
   def __init__(self):
       self.staticon = gtk.StatusIcon() 
@@ -308,13 +385,15 @@
                   break
               url_page = n[2]
               self.nom_emi = n[0]
-              self.nom_fichier = self.nom_emi + "-" + n[1] + '.flv'
+              #self.nom_fichier = self.nom_emi + "-" + n[1] + '.flv'
+              self.nom_fichier = self.nom_emi + "-" + n[1] + '.mp4'
               self.nom_fichier = self.nom_fichier.replace("/", "-")
               self.liststore2.set_value(self.treeiter, 3, _('Download...'))
               try:
                   rtmp_url = get_rtmp_url( url_page, quality = "hd"  )
                   signal_fin = False
-                  for percent in rtmp_download( rtmp_url, self.directory + "/" + self.nom_fichier.replace("'", "_") ):
+                  #for percent in rtmp_download( rtmp_url, self.directory + "/" + self.nom_fichier.replace("'", "_") ):
+                  for percent in http_download( rtmp_url, self.directory + "/" + self.nom_fichier.replace("'", "_") ):
                       if percent == -1.0:
                           raise IOError()
                       signal_fin = percent == 100.0
@@ -333,6 +412,7 @@
                   self.erreur = self.builder.get_object("error_dialog")
                   self.error_text = self.builder.get_object("error_text")
                   self.error_text.set_text(_("There are problem with your internet connection"))
+                  self.liststore2.set_value(self.treeiter, 3, _('Error while downloading'))
                   self.erreur.run()
                   self.result = self.erreur.run()
                   self.erreur.destroy()
@@ -411,18 +491,26 @@
       page = urllib2.urlopen(data_url).read()
       #data_resume = self.resumeRE.search(page).group(1).replace('\n', '').strip()
       soup = BS.BeautifulSoup( page )
-      base_node = soup.find('div', {"class":"recentTracksCont"})
+      #base_node = soup.find('div', {"class":"recentTracksCont"})
+      base_node = soup.find('meta', {"name":"description"})
       data_resume = u""
-      for i in base_node.findAll('p'):
-          if len(data_resume) != 0:
-              data_resume += "\n"
-          #print data_resume.replace("\n","\\n"), i.string
-          try:
-              data_resume += unescape_html(i.string)
-              if i["class"] == "accroche":
-                  data_resume += "\n"
-          except:
-              pass
+      #for i in base_node.findAll('p'):
+      #    if len(data_resume) != 0:
+      #        data_resume += "\n"
+      #    #print data_resume.replace("\n","\\n"), i.string
+      #    try:
+      #        data_resume += unescape_html(i.string)
+      #        if i["class"] == "accroche":
+      #            data_resume += "\n"
+      #    except:
+      #        pass
+      try:
+          res = base_node.get('content')
+      except:
+          pass
+      if len(data_resume) != 0:
+          data_resume += "\n"
+      data_resume += unescape_html(res)
       self.textbuffer1 = self.builder.get_object("textbuffer1")
       self.textbuffer1.set_text(data_resume)
       data_time = self.dureeRE.search(page).group(1)
@@ -488,7 +576,7 @@
 if __name__ == "__main__":
   catalog = Catalog()
   datalist = open('/tmp/database', 'w')
-  print >> datalist, '\n'.join(['%s;%s;%s;%s' % (video[Catalog.TITLE_TAG], video[Catalog.DATE_TAG], video[Catalog.URL_TAG], video[Catalog.IMAGE_TAG]) for video in  catalog.videos])
+  print >> datalist, '\n'.join(['%s;%s;%s;%s' % (video[Catalog.TITLE_TAG], video[Catalog.DATE_TAG], video[Catalog.URL_TAG], video[Catalog.IMAGE_TAG]) for video in  catalog.videos if Catalog.TITLE_TAG in video])
   datalist.close()
   app = GUI()
   gtk.main()