You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

228 lines
8.8 KiB

  1. import re
  2. import json
  3. import itertools
  4. from .common import InfoExtractor
  5. from .subtitles import SubtitlesInfoExtractor
  6. from ..utils import (
  7. compat_urllib_request,
  8. compat_str,
  9. get_element_by_attribute,
  10. get_element_by_id,
  11. orderedSet,
  12. ExtractorError,
  13. )
  14. class DailymotionBaseInfoExtractor(InfoExtractor):
  15. @staticmethod
  16. def _build_request(url):
  17. """Build a request with the family filter disabled"""
  18. request = compat_urllib_request.Request(url)
  19. request.add_header('Cookie', 'family_filter=off')
  20. request.add_header('Cookie', 'ff=off')
  21. return request
  22. class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
  23. """Information Extractor for Dailymotion"""
  24. _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
  25. IE_NAME = u'dailymotion'
  26. _FORMATS = [
  27. (u'stream_h264_ld_url', u'ld'),
  28. (u'stream_h264_url', u'standard'),
  29. (u'stream_h264_hq_url', u'hq'),
  30. (u'stream_h264_hd_url', u'hd'),
  31. (u'stream_h264_hd1080_url', u'hd180'),
  32. ]
  33. _TESTS = [
  34. {
  35. u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
  36. u'file': u'x33vw9.mp4',
  37. u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
  38. u'info_dict': {
  39. u"uploader": u"Amphora Alex and Van .",
  40. u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
  41. }
  42. },
  43. # Vevo video
  44. {
  45. u'url': u'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
  46. u'file': u'USUV71301934.mp4',
  47. u'info_dict': {
  48. u'title': u'Roar (Official)',
  49. u'uploader': u'Katy Perry',
  50. u'upload_date': u'20130905',
  51. },
  52. u'params': {
  53. u'skip_download': True,
  54. },
  55. u'skip': u'VEVO is only available in some countries',
  56. },
  57. # age-restricted video
  58. {
  59. u'url': u'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
  60. u'file': u'xyh2zz.mp4',
  61. u'md5': u'0d667a7b9cebecc3c89ee93099c4159d',
  62. u'info_dict': {
  63. u'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
  64. u'uploader': 'HotWaves1012',
  65. u'age_limit': 18,
  66. }
  67. }
  68. ]
  69. def _real_extract(self, url):
  70. # Extract id and simplified title from URL
  71. mobj = re.match(self._VALID_URL, url)
  72. video_id = mobj.group(1).split('_')[0].split('?')[0]
  73. url = 'http://www.dailymotion.com/video/%s' % video_id
  74. # Retrieve video webpage to extract further information
  75. request = self._build_request(url)
  76. webpage = self._download_webpage(request, video_id)
  77. # Extract URL, uploader and title from webpage
  78. self.report_extraction(video_id)
  79. # It may just embed a vevo video:
  80. m_vevo = re.search(
  81. r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
  82. webpage)
  83. if m_vevo is not None:
  84. vevo_id = m_vevo.group('id')
  85. self.to_screen(u'Vevo video detected: %s' % vevo_id)
  86. return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
  87. video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
  88. # Looking for official user
  89. r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
  90. webpage, 'video uploader', fatal=False)
  91. age_limit = self._rta_search(webpage)
  92. video_upload_date = None
  93. mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
  94. if mobj is not None:
  95. video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
  96. embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
  97. embed_page = self._download_webpage(embed_url, video_id,
  98. u'Downloading embed page')
  99. info = self._search_regex(r'var info = ({.*?}),$', embed_page,
  100. 'video info', flags=re.MULTILINE)
  101. info = json.loads(info)
  102. if info.get('error') is not None:
  103. msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
  104. raise ExtractorError(msg, expected=True)
  105. formats = []
  106. for (key, format_id) in self._FORMATS:
  107. video_url = info.get(key)
  108. if video_url is not None:
  109. m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
  110. if m_size is not None:
  111. width, height = m_size.group(1), m_size.group(2)
  112. else:
  113. width, height = None, None
  114. formats.append({
  115. 'url': video_url,
  116. 'ext': 'mp4',
  117. 'format_id': format_id,
  118. 'width': width,
  119. 'height': height,
  120. })
  121. if not formats:
  122. raise ExtractorError(u'Unable to extract video URL')
  123. # subtitles
  124. video_subtitles = self.extract_subtitles(video_id)
  125. if self._downloader.params.get('listsubtitles', False):
  126. self._list_available_subtitles(video_id)
  127. return
  128. return {
  129. 'id': video_id,
  130. 'formats': formats,
  131. 'uploader': video_uploader,
  132. 'upload_date': video_upload_date,
  133. 'title': self._og_search_title(webpage),
  134. 'subtitles': video_subtitles,
  135. 'thumbnail': info['thumbnail_url'],
  136. 'age_limit': age_limit,
  137. }
  138. def _get_available_subtitles(self, video_id):
  139. try:
  140. sub_list = self._download_webpage(
  141. 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
  142. video_id, note=False)
  143. except ExtractorError as err:
  144. self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
  145. return {}
  146. info = json.loads(sub_list)
  147. if (info['total'] > 0):
  148. sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
  149. return sub_lang_list
  150. self._downloader.report_warning(u'video doesn\'t have subtitles')
  151. return {}
  152. class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
  153. IE_NAME = u'dailymotion:playlist'
  154. _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
  155. _MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
  156. _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
  157. def _extract_entries(self, id):
  158. video_ids = []
  159. for pagenum in itertools.count(1):
  160. request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
  161. webpage = self._download_webpage(request,
  162. id, u'Downloading page %s' % pagenum)
  163. playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
  164. video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
  165. if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
  166. break
  167. return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
  168. for video_id in orderedSet(video_ids)]
  169. def _real_extract(self, url):
  170. mobj = re.match(self._VALID_URL, url)
  171. playlist_id = mobj.group('id')
  172. webpage = self._download_webpage(url, playlist_id)
  173. return {'_type': 'playlist',
  174. 'id': playlist_id,
  175. 'title': get_element_by_id(u'playlist_name', webpage),
  176. 'entries': self._extract_entries(playlist_id),
  177. }
  178. class DailymotionUserIE(DailymotionPlaylistIE):
  179. IE_NAME = u'dailymotion:user'
  180. _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
  181. _MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/user/.+?".*?>.*?</a>.*?</div>'
  182. _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
  183. def _real_extract(self, url):
  184. mobj = re.match(self._VALID_URL, url)
  185. user = mobj.group('user')
  186. webpage = self._download_webpage(url, user)
  187. full_user = self._html_search_regex(
  188. r'<a class="label" href="/%s".*?>(.*?)</' % re.escape(user),
  189. webpage, u'user', flags=re.DOTALL)
  190. return {
  191. '_type': 'playlist',
  192. 'id': user,
  193. 'title': full_user,
  194. 'entries': self._extract_entries(user),
  195. }