You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

251 lines
8.9 KiB

10 years ago
10 years ago
10 years ago
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import itertools
  6. from .common import InfoExtractor
  7. from ..compat import (
  8. compat_str,
  9. compat_urllib_request,
  10. )
  11. from ..utils import (
  12. ExtractorError,
  13. int_or_none,
  14. orderedSet,
  15. str_to_int,
  16. unescapeHTML,
  17. )
  18. class DailymotionBaseInfoExtractor(InfoExtractor):
  19. @staticmethod
  20. def _build_request(url):
  21. """Build a request with the family filter disabled"""
  22. request = compat_urllib_request.Request(url)
  23. request.add_header('Cookie', 'family_filter=off; ff=off')
  24. return request
  25. class DailymotionIE(DailymotionBaseInfoExtractor):
  26. """Information Extractor for Dailymotion"""
  27. _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
  28. IE_NAME = 'dailymotion'
  29. _FORMATS = [
  30. ('stream_h264_ld_url', 'ld'),
  31. ('stream_h264_url', 'standard'),
  32. ('stream_h264_hq_url', 'hq'),
  33. ('stream_h264_hd_url', 'hd'),
  34. ('stream_h264_hd1080_url', 'hd180'),
  35. ]
  36. _TESTS = [
  37. {
  38. 'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
  39. 'md5': '2137c41a8e78554bb09225b8eb322406',
  40. 'info_dict': {
  41. 'id': 'x2iuewm',
  42. 'ext': 'mp4',
  43. 'uploader': 'IGN',
  44. 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
  45. }
  46. },
  47. # Vevo video
  48. {
  49. 'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
  50. 'info_dict': {
  51. 'title': 'Roar (Official)',
  52. 'id': 'USUV71301934',
  53. 'ext': 'mp4',
  54. 'uploader': 'Katy Perry',
  55. 'upload_date': '20130905',
  56. },
  57. 'params': {
  58. 'skip_download': True,
  59. },
  60. 'skip': 'VEVO is only available in some countries',
  61. },
  62. # age-restricted video
  63. {
  64. 'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
  65. 'md5': '0d667a7b9cebecc3c89ee93099c4159d',
  66. 'info_dict': {
  67. 'id': 'xyh2zz',
  68. 'ext': 'mp4',
  69. 'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
  70. 'uploader': 'HotWaves1012',
  71. 'age_limit': 18,
  72. }
  73. }
  74. ]
  75. def _real_extract(self, url):
  76. video_id = self._match_id(url)
  77. url = 'http://www.dailymotion.com/video/%s' % video_id
  78. # Retrieve video webpage to extract further information
  79. request = self._build_request(url)
  80. webpage = self._download_webpage(request, video_id)
  81. # Extract URL, uploader and title from webpage
  82. self.report_extraction(video_id)
  83. # It may just embed a vevo video:
  84. m_vevo = re.search(
  85. r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
  86. webpage)
  87. if m_vevo is not None:
  88. vevo_id = m_vevo.group('id')
  89. self.to_screen('Vevo video detected: %s' % vevo_id)
  90. return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
  91. age_limit = self._rta_search(webpage)
  92. video_upload_date = None
  93. mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
  94. if mobj is not None:
  95. video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
  96. embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
  97. embed_request = self._build_request(embed_url)
  98. embed_page = self._download_webpage(
  99. embed_request, video_id, 'Downloading embed page')
  100. info = self._search_regex(r'var info = ({.*?}),$', embed_page,
  101. 'video info', flags=re.MULTILINE)
  102. info = json.loads(info)
  103. if info.get('error') is not None:
  104. msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
  105. raise ExtractorError(msg, expected=True)
  106. formats = []
  107. for (key, format_id) in self._FORMATS:
  108. video_url = info.get(key)
  109. if video_url is not None:
  110. m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
  111. if m_size is not None:
  112. width, height = map(int_or_none, (m_size.group(1), m_size.group(2)))
  113. else:
  114. width, height = None, None
  115. formats.append({
  116. 'url': video_url,
  117. 'ext': 'mp4',
  118. 'format_id': format_id,
  119. 'width': width,
  120. 'height': height,
  121. })
  122. if not formats:
  123. raise ExtractorError('Unable to extract video URL')
  124. # subtitles
  125. video_subtitles = self.extract_subtitles(video_id, webpage)
  126. view_count = str_to_int(self._search_regex(
  127. r'video_views_count[^>]+>\s+([\d\.,]+)',
  128. webpage, 'view count', fatal=False))
  129. title = self._og_search_title(webpage, default=None)
  130. if title is None:
  131. title = self._html_search_regex(
  132. r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage,
  133. 'title')
  134. return {
  135. 'id': video_id,
  136. 'formats': formats,
  137. 'uploader': info['owner.screenname'],
  138. 'upload_date': video_upload_date,
  139. 'title': title,
  140. 'subtitles': video_subtitles,
  141. 'thumbnail': info['thumbnail_url'],
  142. 'age_limit': age_limit,
  143. 'view_count': view_count,
  144. }
  145. def _get_subtitles(self, video_id, webpage):
  146. try:
  147. sub_list = self._download_webpage(
  148. 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
  149. video_id, note=False)
  150. except ExtractorError as err:
  151. self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
  152. return {}
  153. info = json.loads(sub_list)
  154. if (info['total'] > 0):
  155. sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list'])
  156. return sub_lang_list
  157. self._downloader.report_warning('video doesn\'t have subtitles')
  158. return {}
  159. class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
  160. IE_NAME = 'dailymotion:playlist'
  161. _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
  162. _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'
  163. _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
  164. _TESTS = [{
  165. 'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
  166. 'info_dict': {
  167. 'title': 'SPORT',
  168. 'id': 'xv4bw_nqtv_sport',
  169. },
  170. 'playlist_mincount': 20,
  171. }]
  172. def _extract_entries(self, id):
  173. video_ids = []
  174. for pagenum in itertools.count(1):
  175. request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
  176. webpage = self._download_webpage(request,
  177. id, 'Downloading page %s' % pagenum)
  178. video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
  179. if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
  180. break
  181. return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
  182. for video_id in orderedSet(video_ids)]
  183. def _real_extract(self, url):
  184. mobj = re.match(self._VALID_URL, url)
  185. playlist_id = mobj.group('id')
  186. webpage = self._download_webpage(url, playlist_id)
  187. return {
  188. '_type': 'playlist',
  189. 'id': playlist_id,
  190. 'title': self._og_search_title(webpage),
  191. 'entries': self._extract_entries(playlist_id),
  192. }
  193. class DailymotionUserIE(DailymotionPlaylistIE):
  194. IE_NAME = 'dailymotion:user'
  195. _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
  196. _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
  197. _TESTS = [{
  198. 'url': 'https://www.dailymotion.com/user/nqtv',
  199. 'info_dict': {
  200. 'id': 'nqtv',
  201. 'title': 'Rémi Gaillard',
  202. },
  203. 'playlist_mincount': 100,
  204. }]
  205. def _real_extract(self, url):
  206. mobj = re.match(self._VALID_URL, url)
  207. user = mobj.group('user')
  208. webpage = self._download_webpage(url, user)
  209. full_user = unescapeHTML(self._html_search_regex(
  210. r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
  211. webpage, 'user'))
  212. return {
  213. '_type': 'playlist',
  214. 'id': user,
  215. 'title': full_user,
  216. 'entries': self._extract_entries(user),
  217. }