You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

252 lines
9.0 KiB

10 years ago
10 years ago
10 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import itertools
  6. from .common import InfoExtractor
  7. from ..compat import (
  8. compat_str,
  9. compat_urllib_request,
  10. )
  11. from ..utils import (
  12. ExtractorError,
  13. int_or_none,
  14. orderedSet,
  15. str_to_int,
  16. unescapeHTML,
  17. )
  18. class DailymotionBaseInfoExtractor(InfoExtractor):
  19. @staticmethod
  20. def _build_request(url):
  21. """Build a request with the family filter disabled"""
  22. request = compat_urllib_request.Request(url)
  23. request.add_header('Cookie', 'family_filter=off; ff=off')
  24. return request
  25. class DailymotionIE(DailymotionBaseInfoExtractor):
  26. """Information Extractor for Dailymotion"""
  27. _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
  28. IE_NAME = 'dailymotion'
  29. _FORMATS = [
  30. ('stream_h264_ld_url', 'ld'),
  31. ('stream_h264_url', 'standard'),
  32. ('stream_h264_hq_url', 'hq'),
  33. ('stream_h264_hd_url', 'hd'),
  34. ('stream_h264_hd1080_url', 'hd180'),
  35. ]
  36. _TESTS = [
  37. {
  38. 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
  39. 'md5': '2137c41a8e78554bb09225b8eb322406',
  40. 'info_dict': {
  41. 'id': 'x2iuewm',
  42. 'ext': 'mp4',
  43. 'uploader': 'IGN',
  44. 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
  45. 'upload_date': '20150306',
  46. }
  47. },
  48. # Vevo video
  49. {
  50. 'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
  51. 'info_dict': {
  52. 'title': 'Roar (Official)',
  53. 'id': 'USUV71301934',
  54. 'ext': 'mp4',
  55. 'uploader': 'Katy Perry',
  56. 'upload_date': '20130905',
  57. },
  58. 'params': {
  59. 'skip_download': True,
  60. },
  61. 'skip': 'VEVO is only available in some countries',
  62. },
  63. # age-restricted video
  64. {
  65. 'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
  66. 'md5': '0d667a7b9cebecc3c89ee93099c4159d',
  67. 'info_dict': {
  68. 'id': 'xyh2zz',
  69. 'ext': 'mp4',
  70. 'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
  71. 'uploader': 'HotWaves1012',
  72. 'age_limit': 18,
  73. }
  74. }
  75. ]
  76. def _real_extract(self, url):
  77. video_id = self._match_id(url)
  78. url = 'https://www.dailymotion.com/video/%s' % video_id
  79. # Retrieve video webpage to extract further information
  80. request = self._build_request(url)
  81. webpage = self._download_webpage(request, video_id)
  82. # Extract URL, uploader and title from webpage
  83. self.report_extraction(video_id)
  84. # It may just embed a vevo video:
  85. m_vevo = re.search(
  86. r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
  87. webpage)
  88. if m_vevo is not None:
  89. vevo_id = m_vevo.group('id')
  90. self.to_screen('Vevo video detected: %s' % vevo_id)
  91. return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
  92. age_limit = self._rta_search(webpage)
  93. video_upload_date = None
  94. mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
  95. if mobj is not None:
  96. video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
  97. embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
  98. embed_request = self._build_request(embed_url)
  99. embed_page = self._download_webpage(
  100. embed_request, video_id, 'Downloading embed page')
  101. info = self._search_regex(r'var info = ({.*?}),$', embed_page,
  102. 'video info', flags=re.MULTILINE)
  103. info = json.loads(info)
  104. if info.get('error') is not None:
  105. msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
  106. raise ExtractorError(msg, expected=True)
  107. formats = []
  108. for (key, format_id) in self._FORMATS:
  109. video_url = info.get(key)
  110. if video_url is not None:
  111. m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
  112. if m_size is not None:
  113. width, height = map(int_or_none, (m_size.group(1), m_size.group(2)))
  114. else:
  115. width, height = None, None
  116. formats.append({
  117. 'url': video_url,
  118. 'ext': 'mp4',
  119. 'format_id': format_id,
  120. 'width': width,
  121. 'height': height,
  122. })
  123. if not formats:
  124. raise ExtractorError('Unable to extract video URL')
  125. # subtitles
  126. video_subtitles = self.extract_subtitles(video_id, webpage)
  127. view_count = str_to_int(self._search_regex(
  128. r'video_views_count[^>]+>\s+([\d\.,]+)',
  129. webpage, 'view count', fatal=False))
  130. title = self._og_search_title(webpage, default=None)
  131. if title is None:
  132. title = self._html_search_regex(
  133. r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage,
  134. 'title')
  135. return {
  136. 'id': video_id,
  137. 'formats': formats,
  138. 'uploader': info['owner.screenname'],
  139. 'upload_date': video_upload_date,
  140. 'title': title,
  141. 'subtitles': video_subtitles,
  142. 'thumbnail': info['thumbnail_url'],
  143. 'age_limit': age_limit,
  144. 'view_count': view_count,
  145. }
  146. def _get_subtitles(self, video_id, webpage):
  147. try:
  148. sub_list = self._download_webpage(
  149. 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
  150. video_id, note=False)
  151. except ExtractorError as err:
  152. self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
  153. return {}
  154. info = json.loads(sub_list)
  155. if (info['total'] > 0):
  156. sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list'])
  157. return sub_lang_list
  158. self._downloader.report_warning('video doesn\'t have subtitles')
  159. return {}
  160. class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
  161. IE_NAME = 'dailymotion:playlist'
  162. _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
  163. _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
  164. _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
  165. _TESTS = [{
  166. 'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
  167. 'info_dict': {
  168. 'title': 'SPORT',
  169. 'id': 'xv4bw_nqtv_sport',
  170. },
  171. 'playlist_mincount': 20,
  172. }]
  173. def _extract_entries(self, id):
  174. video_ids = []
  175. for pagenum in itertools.count(1):
  176. request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
  177. webpage = self._download_webpage(request,
  178. id, 'Downloading page %s' % pagenum)
  179. video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
  180. if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
  181. break
  182. return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
  183. for video_id in orderedSet(video_ids)]
  184. def _real_extract(self, url):
  185. mobj = re.match(self._VALID_URL, url)
  186. playlist_id = mobj.group('id')
  187. webpage = self._download_webpage(url, playlist_id)
  188. return {
  189. '_type': 'playlist',
  190. 'id': playlist_id,
  191. 'title': self._og_search_title(webpage),
  192. 'entries': self._extract_entries(playlist_id),
  193. }
  194. class DailymotionUserIE(DailymotionPlaylistIE):
  195. IE_NAME = 'dailymotion:user'
  196. _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
  197. _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
  198. _TESTS = [{
  199. 'url': 'https://www.dailymotion.com/user/nqtv',
  200. 'info_dict': {
  201. 'id': 'nqtv',
  202. 'title': 'Rémi Gaillard',
  203. },
  204. 'playlist_mincount': 100,
  205. }]
  206. def _real_extract(self, url):
  207. mobj = re.match(self._VALID_URL, url)
  208. user = mobj.group('user')
  209. webpage = self._download_webpage(url, user)
  210. full_user = unescapeHTML(self._html_search_regex(
  211. r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
  212. webpage, 'user'))
  213. return {
  214. '_type': 'playlist',
  215. 'id': user,
  216. 'title': full_user,
  217. 'entries': self._extract_entries(user),
  218. }