Merge lp:~clement-grimal/qarte/update-arte-website into lp:qarte/qarte-3

[kokoklems]

Fix the json parsing, but the thumbnails are broken.
I also disabled the remove_expired_videos function, as the field used in the json to get the expiration date is not available anymore.

Unmerged revisions

181. By kokoklems on 2017-07-11

Fix fetch of the json data after ARTE website update. Thumbnails and expiration dates are broken!

=== modified file 'artetv.py'
--- artetv.py	2017-04-26 15:52:49 +0000
+++ artetv.py	2017-07-11 15:22:11 +0000
@@ -137,6 +137,8 @@
                 print(video)
             self.videos.append(TVItem(video))
 
+        # lgg.warning('DDD :: ' + str(len(self.videos)))
+
         for video in self.old_videos:
             # The two first pages contains some identical videos
             if not video.idx in idxs:
@@ -144,7 +146,8 @@
                 idxs.append(video.idx)
 
         self.indexes = idxs[:]
-        Thread(target=self.remove_expired_videos).start()
+        # TODO: rights_end field no longer available
+        # Thread(target=self.remove_expired_videos).start()
 
     def sort_recents(self):
         """Sort the recent videos.
@@ -853,6 +856,7 @@
 
     def get_next_page(self):
         url = self.url + next(self.day_before)
+        # lgg.warning(url)
         items = self.get_json(self.fetch_page(url))
         videos = []
         indexes = []
@@ -862,36 +866,37 @@
                 if video:
                     videos.append(video)
                     indexes.append(video['id'])
+        # lgg.warning('D :: ' + str(len(videos)))
         return videos, indexes
 
     def get_json(self, content):
         if content:
             try:
-                content = content.split('"tvguide":{"broadcasts":')[1]
-                content = content.split('"currentDate":')[0]
-                content = content.lstrip().rstrip(', \n')
-                return json.loads(content)
+                # lgg.warning(content)
+                content = content.split('window.__INITIAL_STATE__ = ')[1]
+                content = content.split(';')[0]
+                data = json.loads(content)
+                items = data['page']['zones'][1]['teasers']
+                # lgg.warning(items)
+                return items
             except Exception as why:
                 lgg.warning("Read json error: %s" % why)
 
     def clean_item(self, item):
         video = {}
         is_live = False
-        for v in item["videos"]:
-            if v["kindLabel"] not in ("BANDE-ANNONCE", "EXTRAIT"):
-                is_live = True
-        if not is_live:
+        # lgg.warning(item)
+        if item["kind"] in ("BANDE-ANNONCE", "EXTRAIT"):
             return False
         try:
             video["rights_end"] = v["videoRightsEnd"]
         except:
             video["rights_end"] = ""
         try:
-            video['scheduled_on'] = item["broadcastBeginRounded"]
+            video['scheduled_on'] = item['beginsAt']
         except:
             video['scheduled_on'] = ""
 
-        item = item["program"]
         try:
             video["url"] = item["url"]
             video['id'] = item['programId']
@@ -899,20 +904,22 @@
             # No url, no video
             return False
 
-        video["subtitle"] = item.get("shortDescription", "")
+        video["subtitle"] = item.get("subtitle", "")
         video["headerText"] = item.get("headerText", "")
         video["title"] = item.get("title", "No title")
         video["description"] = item.get("fullDescription", "")
-        video["duration"] = item.get("durationSeconds", "")
+        video["duration"] = item.get("duration", 0)
         video["teaser"] = item.get("teaserText", "")
         video["programId"] = item.get("programId", "")
         video['adult'] = None
         video['views'] = 0
-        thumb = item.get("mainImage", "")
-        if thumb:
-            video["thumbnail_url"] = thumb.get("url", "")
-            video["credit"] = thumb.get("copyright", "")
-        
+        try:
+            video["thumbnail_url"] = item['images'][-1]['url']
+            video["credit"] = '' #item['images'][-1]['copyright']
+        except:
+            video["thumbnail_url"] = ''
+            video["credit"] = ''
+
         return video
 
 
@@ -971,6 +978,8 @@
             return datetime.now()
 
     def format_duration(self, value):
+        if not value:
+            value = 0
         min_, sec = divmod(value, 60)
         if min_ < 60:
             return "%s min. %s sec." %(min_, sec)