]>
git.gir.st - subscriptionfeed.git/blob - app/common/innertube.py
1 # functions that deal with parsing data from youtube's internal API ("innertube")
3 from urllib
.parse
import parse_qs
, urlparse
5 def listfind(obj
, key
):
7 given a list of dicts, where one dict contains a given key, return said key.
9 return next(iter([ obj
[key
] for obj
in obj
if key
in obj
.keys() ]),{})
10 def listget(obj
, index
, fallback
=None):
11 return next(iter(obj
[index
:]), fallback
)
13 def prepare_searchresults(yt_results
):
14 contents
= listfind(yt_results
, 'response') \
16 .get('twoColumnSearchResultsRenderer',{})\
17 .get('primaryContents',{})\
18 .get('sectionListRenderer',{})\
20 contents
= listfind(contents
, 'itemSectionRenderer').get('contents',[])
22 return parse_result_items(contents
)
23 def prepare_infocards(metadata
):
24 cards
= metadata
.get('cards',{}).get('cardCollectionRenderer',{}).get('cards',[])
25 return [parse_infocard(card
) for card
in cards
if card
is not None]
27 def prepare_endcards(metadata
):
28 endsc
= metadata
.get('endscreen',{}).get('endscreenRenderer',{}).get('elements',[])
29 return [parse_endcard(card
) for card
in endsc
if card
is not None]
32 output
= {str(e
['height']): e
['url'] for e
in thumbs
}
33 largest
=next(iter(sorted(output
.keys(),reverse
=True,key
=int)),None)
34 return {**output
, 'largest': largest
}
37 # externals URLs are redirected through youtube.com/redirect, but we
38 # may encounter internal URLs, too
39 return parse_qs(urlparse(url
).query
).get('q',[url
])[0]
41 def toInt(s
, fallback
=0):
45 return int(''.join(filter(str.isdigit
, s
)))
49 # Remove left-/rightmost word from string:
50 delL
= lambda s
: s
.partition(' ')[2]
53 if s
is None: # missing from autogen'd music, some livestreams
55 # Some livestreams have "Streamed 7 hours ago"
56 s
= s
.replace("Streamed ","")
57 # Now, everything should be in the form "1 year ago"
58 value
, unit
, _
= s
.split(" ")
62 ).get(unit
, unit
[0]) # first letter otherwise (e.g. year(s) => y)
64 return f
"{value}{suffix}"
66 def log_unknown_card(data
):
69 from flask
import request
71 except: source
= "unknown"
72 with
open("/tmp/innertube.err", "a") as f
:
73 f
.write(f
"\n/***** {source} *****/\n")
74 json
.dump(data
, f
, indent
=2)
76 def parse_result_items(items
):
77 # TODO: use .get() for most non-essential attributes
79 parses youtube search response into an easier to use format.
83 key
= next(iter(item
.keys()), None)
85 if key
== 'videoRenderer':
86 is_live
= listfind(content
.get('badges',[]), 'metadataBadgeRenderer').get('style') == 'BADGE_STYLE_TYPE_LIVE_NOW'
87 results
.append({'type': 'VIDEO', 'content': {
88 'video_id': content
['videoId'],
89 'title': content
['title']['runs'][0]['text'],
90 'author': content
['longBylineText']['runs'][0]['text'] or \
91 content
['shortBylineText']['runs'][0]['text'],
92 'channel_id': content
['ownerText']['runs'][0] \
93 ['navigationEndpoint']['browseEndpoint']['browseId'],
94 'length': content
.get('lengthText',{}).get('simpleText') \
95 if not is_live
else 'LIVE', # "44:07", "1:41:50"
96 'views': toInt(content
.get('viewCountText',{}).get('simpleText') or # "123,456 views"
97 listget(content
.get('viewCountText',{}).get('runs'),0,{}).get('text')), # "1,234 watching"
98 'published': age(content
.get('publishedTimeText',{}).get('simpleText')),
100 elif key
== 'playlistRenderer':
101 results
.append({'type': 'PLAYLIST', 'content': {
102 'playlist_id': content
['navigationEndpoint']['watchEndpoint']['playlistId'],
103 'video_id': content
['navigationEndpoint']['watchEndpoint']['videoId'],
104 'title': content
['title']['simpleText'],
105 'author': content
['longBylineText']['runs'][0]['text'] or
106 content
['shortBylineText']['runs'][0]['text'],
107 'channel_id': content
['longBylineText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId'], # OR .shortBylineText
108 'n_videos': toInt(content
['videoCount']),
110 elif key
== 'radioRenderer': # "Mix" playlists
111 results
.append({'type': 'PLAYLIST', 'content': {
112 'playlist_id': content
['playlistId'],
113 'video_id': content
['navigationEndpoint']['watchEndpoint']['videoId'],
114 'title': content
['title']['simpleText'],
115 'author': content
['longBylineText']['simpleText'] or \
116 content
['shortBylineText']['simpleText'] , # always "YouTube"
118 'n_videos': content
['videoCountShortText']['runs'][0]['text'] or \
119 content
['videoCountText']['runs'][0]['text'],
120 # videoCountShortText: "50+"; videoCountText: "50+ videos"
122 elif key
== 'channelRenderer':
123 results
.append({'type': 'CHANNEL', 'content': {
124 'channel_id': content
['channelId'],
125 'title': content
['title']['simpleText'],
126 'icons': mkthumbs(content
['thumbnail']['thumbnails']),
127 'subscribers': content
['subscriberCountText']['simpleText'], # "2.47K subscribers"
129 elif key
== 'shelfRenderer':
130 results
.extend([item
for item
in
131 parse_result_items(content
['content']['verticalListRenderer']['items'])
133 elif key
== 'movieRenderer': # movies to buy/rent
135 elif key
== 'horizontalCardListRenderer':
136 # suggested searches: .cards[].searchRefinementCardRenderer.query.runs[].text
139 log_unknown_card(item
)
142 def parse_infocard(card
):
144 parses a single infocard into a format that's easier to handle.
146 card
= card
['cardRenderer']
147 ctype
= list(card
['content'].keys())[0]
148 content
= card
['content'][ctype
]
149 if ctype
== "pollRenderer":
150 return {'type': "POLL", 'content': {
151 'question': content
['question']['simpleText'],
152 'answers': [(a
['text']['simpleText'],a
['numVotes']) \
153 for a
in content
['choices']],
155 elif ctype
== "videoInfoCardContentRenderer":
156 is_live
= content
.get('badge',{}).get('liveBadgeRenderer') is not None
157 return {'type': "VIDEO", 'content': {
158 'video_id': content
['action']['watchEndpoint']['videoId'],
159 'title': content
['videoTitle']['simpleText'],
160 'author': delL(content
['channelName']['simpleText']),
161 'length': content
.get('lengthString',{}).get('simpleText') \
162 if not is_live
else "LIVE", # "23:03"
163 'views': toInt(content
.get('viewCountText',{}).get('simpleText')),
164 # XXX: views sometimes "Starts: July 31, 2020 at 1:30 PM"
166 elif ctype
== "playlistInfoCardContentRenderer":
167 return {'type': "PLAYLIST", 'content': {
168 'playlist_id': content
['action']['watchEndpoint']['playlistId'],
169 'video_id': content
['action']['watchEndpoint']['videoId'],
170 'title': content
['playlistTitle']['simpleText'],
171 'author': delL(content
['channelName']['simpleText']),
172 'n_videos': toInt(content
['playlistVideoCount']['simpleText']),
174 elif ctype
== "simpleCardContentRenderer" and \
175 'urlEndpoint' in content
['command']:
176 return {'type': "WEBSITE", 'content': {
177 'url': clean_url(content
['command']['urlEndpoint']['url']),
178 'domain': content
['displayDomain']['simpleText'],
179 'title': content
['title']['simpleText'],
180 # XXX: no thumbnails for infocards
182 elif ctype
== "collaboratorInfoCardContentRenderer":
183 return {'type': "CHANNEL", 'content': {
184 'channel_id': content
['endpoint']['browseEndpoint']['browseId'],
185 'title': content
['channelName']['simpleText'],
186 'icons': mkthumbs(content
['channelAvatar']['thumbnails']),
187 'subscribers': content
.get('subscriberCountText',{}).get('simpleText',''), # "545K subscribers"
190 log_unknown_card(card
)
193 def parse_endcard(card
):
195 parses a single endcard into a format that's easier to handle.
197 card
= card
.get('endscreenElementRenderer', card
) #only sometimes nested
198 ctype
= card
['style']
199 if ctype
== "CHANNEL":
200 return {'type': ctype
, 'content': {
201 'channel_id': card
['endpoint']['browseEndpoint']['browseId'],
202 'title': card
['title']['simpleText'],
203 'icons': mkthumbs(card
['image']['thumbnails']),
205 elif ctype
== "VIDEO":
206 return {'type': ctype
, 'content': {
207 'video_id': card
['endpoint']['watchEndpoint']['videoId'],
208 'title': card
['title']['simpleText'],
209 'length': card
['videoDuration']['simpleText'], # '12:21'
210 'views': toInt(card
['metadata']['simpleText']),
211 # XXX: no channel name
213 elif ctype
== "PLAYLIST":
214 return {'type': ctype
, 'content': {
215 'playlist_id': card
['endpoint']['watchEndpoint']['playlistId'],
216 'video_id': card
['endpoint']['watchEndpoint']['videoId'],
217 'title': card
['title']['simpleText'],
218 'author': delL(card
['metadata']['simpleText']),
219 'n_videos': toInt(card
['playlistLength']['simpleText']),
221 elif ctype
== "WEBSITE" or ctype
== "CREATOR_MERCHANDISE":
222 url
= clean_url(card
['endpoint']['urlEndpoint']['url'])
223 return {'type': "WEBSITE", 'content': {
225 'domain': urlparse(url
).netloc
,
226 'title': card
['title']['simpleText'],
227 'icons': mkthumbs(card
['image']['thumbnails']),
230 log_unknown_card(card
)