You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

418 lines
16 KiB

11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
10 years ago
11 years ago
11 years ago
10 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
11 years ago
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import json
  5. import hashlib
  6. import uuid
  7. from .common import InfoExtractor
  8. from ..compat import (
  9. compat_urllib_parse,
  10. compat_urllib_request,
  11. )
  12. from ..utils import (
  13. ExtractorError,
  14. int_or_none,
  15. unified_strdate,
  16. )
  17. class SmotriIE(InfoExtractor):
  18. IE_DESC = 'Smotri.com'
  19. IE_NAME = 'smotri'
  20. _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
  21. _NETRC_MACHINE = 'smotri'
  22. _TESTS = [
  23. # real video id 2610366
  24. {
  25. 'url': 'http://smotri.com/video/view/?id=v261036632ab',
  26. 'md5': '2a7b08249e6f5636557579c368040eb9',
  27. 'info_dict': {
  28. 'id': 'v261036632ab',
  29. 'ext': 'mp4',
  30. 'title': 'катастрофа с камер видеонаблюдения',
  31. 'uploader': 'rbc2008',
  32. 'uploader_id': 'rbc08',
  33. 'upload_date': '20131118',
  34. 'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
  35. },
  36. },
  37. # real video id 57591
  38. {
  39. 'url': 'http://smotri.com/video/view/?id=v57591cb20',
  40. 'md5': '830266dfc21f077eac5afd1883091bcd',
  41. 'info_dict': {
  42. 'id': 'v57591cb20',
  43. 'ext': 'flv',
  44. 'title': 'test',
  45. 'uploader': 'Support Photofile@photofile',
  46. 'uploader_id': 'support-photofile',
  47. 'upload_date': '20070704',
  48. 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
  49. },
  50. },
  51. # video-password, not approved by moderator
  52. {
  53. 'url': 'http://smotri.com/video/view/?id=v1390466a13c',
  54. 'md5': 'f6331cef33cad65a0815ee482a54440b',
  55. 'info_dict': {
  56. 'id': 'v1390466a13c',
  57. 'ext': 'mp4',
  58. 'title': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
  59. 'uploader': 'timoxa40',
  60. 'uploader_id': 'timoxa40',
  61. 'upload_date': '20100404',
  62. 'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
  63. },
  64. 'params': {
  65. 'videopassword': 'qwerty',
  66. },
  67. 'skip': 'Video is not approved by moderator',
  68. },
  69. # video-password
  70. {
  71. 'url': 'http://smotri.com/video/view/?id=v6984858774#',
  72. 'md5': 'f11e01d13ac676370fc3b95b9bda11b0',
  73. 'info_dict': {
  74. 'id': 'v6984858774',
  75. 'ext': 'mp4',
  76. 'title': 'Дача Солженицина ПАРОЛЬ 223322',
  77. 'uploader': 'psavari1',
  78. 'uploader_id': 'psavari1',
  79. 'upload_date': '20081103',
  80. 'thumbnail': 're:^https?://.*\.jpg$',
  81. },
  82. 'params': {
  83. 'videopassword': '223322',
  84. },
  85. },
  86. # age limit + video-password, not approved by moderator
  87. {
  88. 'url': 'http://smotri.com/video/view/?id=v15408898bcf',
  89. 'md5': '91e909c9f0521adf5ee86fbe073aad70',
  90. 'info_dict': {
  91. 'id': 'v15408898bcf',
  92. 'ext': 'flv',
  93. 'title': 'этот ролик не покажут по ТВ',
  94. 'uploader': 'zzxxx',
  95. 'uploader_id': 'ueggb',
  96. 'upload_date': '20101001',
  97. 'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
  98. 'age_limit': 18,
  99. },
  100. 'params': {
  101. 'videopassword': '333'
  102. },
  103. 'skip': 'Video is not approved by moderator',
  104. },
  105. # age limit + video-password
  106. {
  107. 'url': 'http://smotri.com/video/view/?id=v7780025814',
  108. 'md5': 'b4599b068422559374a59300c5337d72',
  109. 'info_dict': {
  110. 'id': 'v7780025814',
  111. 'ext': 'mp4',
  112. 'title': 'Sexy Beach (пароль 123)',
  113. 'uploader': 'вАся',
  114. 'uploader_id': 'asya_prosto',
  115. 'upload_date': '20081218',
  116. 'thumbnail': 're:^https?://.*\.jpg$',
  117. 'age_limit': 18,
  118. },
  119. 'params': {
  120. 'videopassword': '123'
  121. },
  122. },
  123. # swf player
  124. {
  125. 'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
  126. 'md5': '31099eeb4bc906712c5f40092045108d',
  127. 'info_dict': {
  128. 'id': 'v9188090500',
  129. 'ext': 'mp4',
  130. 'title': 'Shakira - Don\'t Bother',
  131. 'uploader': 'HannahL',
  132. 'uploader_id': 'lisaha95',
  133. 'upload_date': '20090331',
  134. 'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
  135. },
  136. },
  137. ]
  138. @classmethod
  139. def _extract_url(cls, webpage):
  140. mobj = re.search(
  141. r'<embed[^>]src=(["\'])(?P<url>http://pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=v.+?\1)',
  142. webpage)
  143. if mobj is not None:
  144. return mobj.group('url')
  145. mobj = re.search(
  146. r'''(?x)<div\s+class="video_file">http://smotri\.com/video/download/file/[^<]+</div>\s*
  147. <div\s+class="video_image">[^<]+</div>\s*
  148. <div\s+class="video_id">(?P<id>[^<]+)</div>''', webpage)
  149. if mobj is not None:
  150. return 'http://smotri.com/video/view/?id=%s' % mobj.group('id')
  151. def _search_meta(self, name, html, display_name=None):
  152. if display_name is None:
  153. display_name = name
  154. return self._html_search_meta(name, html, display_name)
  155. def _real_extract(self, url):
  156. video_id = self._match_id(url)
  157. video_form = {
  158. 'ticket': video_id,
  159. 'video_url': '1',
  160. 'frame_url': '1',
  161. 'devid': 'LoadupFlashPlayer',
  162. 'getvideoinfo': '1',
  163. }
  164. video_password = self._downloader.params.get('videopassword', None)
  165. if video_password:
  166. video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
  167. request = compat_urllib_request.Request(
  168. 'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
  169. request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  170. video = self._download_json(request, video_id, 'Downloading video JSON')
  171. video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
  172. if not video_url:
  173. if video.get('_moderate_no'):
  174. raise ExtractorError(
  175. 'Video %s has not been approved by moderator' % video_id, expected=True)
  176. if video.get('error'):
  177. raise ExtractorError('Video %s does not exist' % video_id, expected=True)
  178. if video.get('_pass_protected') == 1:
  179. msg = ('Invalid video password' if video_password
  180. else 'This video is protected by a password, use the --video-password option')
  181. raise ExtractorError(msg, expected=True)
  182. title = video['title']
  183. thumbnail = video['_imgURL']
  184. upload_date = unified_strdate(video['added'])
  185. uploader = video['userNick']
  186. uploader_id = video['userLogin']
  187. duration = int_or_none(video['duration'])
  188. # Video JSON does not provide enough meta data
  189. # We will extract some from the video web page instead
  190. webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
  191. webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')
  192. # Warning if video is unavailable
  193. warning = self._html_search_regex(
  194. r'<div class="videoUnModer">(.*?)</div>', webpage,
  195. 'warning message', default=None)
  196. if warning is not None:
  197. self._downloader.report_warning(
  198. 'Video %s may not be available; smotri said: %s ' %
  199. (video_id, warning))
  200. # Adult content
  201. if re.search('EroConfirmText">', webpage) is not None:
  202. self.report_age_confirmation()
  203. confirm_string = self._html_search_regex(
  204. r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
  205. webpage, 'confirm string')
  206. confirm_url = webpage_url + '&confirm=%s' % confirm_string
  207. webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
  208. adult_content = True
  209. else:
  210. adult_content = False
  211. view_count = self._html_search_regex(
  212. 'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
  213. webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
  214. return {
  215. 'id': video_id,
  216. 'url': video_url,
  217. 'title': title,
  218. 'thumbnail': thumbnail,
  219. 'uploader': uploader,
  220. 'upload_date': upload_date,
  221. 'uploader_id': uploader_id,
  222. 'duration': duration,
  223. 'view_count': int_or_none(view_count),
  224. 'age_limit': 18 if adult_content else 0,
  225. }
  226. class SmotriCommunityIE(InfoExtractor):
  227. IE_DESC = 'Smotri.com community videos'
  228. IE_NAME = 'smotri:community'
  229. _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)'
  230. _TEST = {
  231. 'url': 'http://smotri.com/community/video/kommuna',
  232. 'info_dict': {
  233. 'id': 'kommuna',
  234. 'title': 'КПРФ',
  235. },
  236. 'playlist_mincount': 4,
  237. }
  238. def _real_extract(self, url):
  239. mobj = re.match(self._VALID_URL, url)
  240. community_id = mobj.group('communityid')
  241. url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id
  242. rss = self._download_xml(url, community_id, 'Downloading community RSS')
  243. entries = [self.url_result(video_url.text, 'Smotri')
  244. for video_url in rss.findall('./channel/item/link')]
  245. description_text = rss.find('./channel/description').text
  246. community_title = self._html_search_regex(
  247. '^Видео сообщества "([^"]+)"$', description_text, 'community title')
  248. return self.playlist_result(entries, community_id, community_title)
  249. class SmotriUserIE(InfoExtractor):
  250. IE_DESC = 'Smotri.com user videos'
  251. IE_NAME = 'smotri:user'
  252. _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)'
  253. _TESTS = [{
  254. 'url': 'http://smotri.com/user/inspector',
  255. 'info_dict': {
  256. 'id': 'inspector',
  257. 'title': 'Inspector',
  258. },
  259. 'playlist_mincount': 9,
  260. }]
  261. def _real_extract(self, url):
  262. mobj = re.match(self._VALID_URL, url)
  263. user_id = mobj.group('userid')
  264. url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id
  265. rss = self._download_xml(url, user_id, 'Downloading user RSS')
  266. entries = [self.url_result(video_url.text, 'Smotri')
  267. for video_url in rss.findall('./channel/item/link')]
  268. description_text = rss.find('./channel/description').text
  269. user_nickname = self._html_search_regex(
  270. '^Видео режиссера (.*)$', description_text,
  271. 'user nickname')
  272. return self.playlist_result(entries, user_id, user_nickname)
  273. class SmotriBroadcastIE(InfoExtractor):
  274. IE_DESC = 'Smotri.com broadcasts'
  275. IE_NAME = 'smotri:broadcast'
  276. _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*'
  277. def _real_extract(self, url):
  278. mobj = re.match(self._VALID_URL, url)
  279. broadcast_id = mobj.group('broadcastid')
  280. broadcast_url = 'http://' + mobj.group('url')
  281. broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
  282. if re.search('>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None:
  283. raise ExtractorError(
  284. 'Broadcast %s does not exist' % broadcast_id, expected=True)
  285. # Adult content
  286. if re.search('EroConfirmText">', broadcast_page) is not None:
  287. (username, password) = self._get_login_info()
  288. if username is None:
  289. raise ExtractorError(
  290. 'Erotic broadcasts allowed only for registered users, '
  291. 'use --username and --password options to provide account credentials.',
  292. expected=True)
  293. login_form = {
  294. 'login-hint53': '1',
  295. 'confirm_erotic': '1',
  296. 'login': username,
  297. 'password': password,
  298. }
  299. request = compat_urllib_request.Request(
  300. broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
  301. request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  302. broadcast_page = self._download_webpage(
  303. request, broadcast_id, 'Logging in and confirming age')
  304. if re.search('>Неверный логин или пароль<', broadcast_page) is not None:
  305. raise ExtractorError('Unable to log in: bad username or password', expected=True)
  306. adult_content = True
  307. else:
  308. adult_content = False
  309. ticket = self._html_search_regex(
  310. r"window\.broadcast_control\.addFlashVar\('file'\s*,\s*'([^']+)'\)",
  311. broadcast_page, 'broadcast ticket')
  312. url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
  313. broadcast_password = self._downloader.params.get('videopassword', None)
  314. if broadcast_password:
  315. url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
  316. broadcast_json_page = self._download_webpage(
  317. url, broadcast_id, 'Downloading broadcast JSON')
  318. try:
  319. broadcast_json = json.loads(broadcast_json_page)
  320. protected_broadcast = broadcast_json['_pass_protected'] == 1
  321. if protected_broadcast and not broadcast_password:
  322. raise ExtractorError(
  323. 'This broadcast is protected by a password, use the --video-password option',
  324. expected=True)
  325. broadcast_offline = broadcast_json['is_play'] == 0
  326. if broadcast_offline:
  327. raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)
  328. rtmp_url = broadcast_json['_server']
  329. mobj = re.search(r'^rtmp://[^/]+/(?P<app>.+)/?$', rtmp_url)
  330. if not mobj:
  331. raise ExtractorError('Unexpected broadcast rtmp URL')
  332. broadcast_playpath = broadcast_json['_streamName']
  333. broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
  334. broadcast_thumbnail = broadcast_json['_imgURL']
  335. broadcast_title = self._live_title(broadcast_json['title'])
  336. broadcast_description = broadcast_json['description']
  337. broadcaster_nick = broadcast_json['nick']
  338. broadcaster_login = broadcast_json['login']
  339. rtmp_conn = 'S:%s' % uuid.uuid4().hex
  340. except KeyError:
  341. if protected_broadcast:
  342. raise ExtractorError('Bad broadcast password', expected=True)
  343. raise ExtractorError('Unexpected broadcast JSON')
  344. return {
  345. 'id': broadcast_id,
  346. 'url': rtmp_url,
  347. 'title': broadcast_title,
  348. 'thumbnail': broadcast_thumbnail,
  349. 'description': broadcast_description,
  350. 'uploader': broadcaster_nick,
  351. 'uploader_id': broadcaster_login,
  352. 'age_limit': 18 if adult_content else 0,
  353. 'ext': 'flv',
  354. 'play_path': broadcast_playpath,
  355. 'player_url': 'http://pics.smotri.com/broadcast_play.swf',
  356. 'app': broadcast_app,
  357. 'rtmp_live': True,
  358. 'rtmp_conn': rtmp_conn,
  359. 'is_live': True,
  360. }