]>
git.gir.st - subscriptionfeed.git/blob - app/common/innertube.py
1 # functions that deal with parsing data from youtube's internal API ("innertube")
3 from urllib
.parse
import parse_qs
, urlparse
7 given a list of dicts, where one dict contains a given key, return said key.
9 return [ obj
[key
] for obj
in obj
if key
in obj
.keys() ]
10 def listget(obj
, index
, fallback
=None):
11 return next(iter(obj
[index
:]), fallback
)
12 flatten
= lambda l
: [item
for sublist
in l
for item
in sublist
] # https://stackoverflow.com/a/952952
13 first
= lambda l
: next(iter(l
),{})
14 listfind
= lambda obj
,key
: first(findall(obj
,key
))
16 def prepare_searchresults(yt_results
):
17 contents
= listfind(yt_results
, 'response') \
19 .get('twoColumnSearchResultsRenderer',{})\
20 .get('primaryContents',{})\
21 .get('sectionListRenderer',{})\
23 contents
= flatten([c
.get('contents',[]) for c
in findall(contents
, 'itemSectionRenderer')])
25 return parse_result_items(contents
)
27 def prepare_infocards(metadata
):
28 cards
= metadata
.get('cards',{}).get('cardCollectionRenderer',{}).get('cards',[])
29 return [parse_infocard(card
) for card
in cards
if card
is not None]
31 def prepare_endcards(metadata
):
32 endsc
= metadata
.get('endscreen',{}).get('endscreenRenderer',{}).get('elements',[])
33 return [parse_endcard(card
) for card
in endsc
if card
is not None]
36 output
= {str(e
['height']): e
['url'] for e
in thumbs
}
37 largest
=next(iter(sorted(output
.keys(),reverse
=True,key
=int)),None)
38 return {**output
, 'largest': largest
}
41 # externals URLs are redirected through youtube.com/redirect, but we
42 # may encounter internal URLs, too
43 return parse_qs(urlparse(url
).query
).get('q',[url
])[0]
45 def toInt(s
, fallback
=0):
49 return int(''.join(filter(str.isdigit
, s
)))
53 # Remove left-/rightmost word from string:
54 delL
= lambda s
: s
.partition(' ')[2]
57 if s
is None: # missing from autogen'd music, some livestreams
59 # Some livestreams have "Streamed 7 hours ago"
60 s
= s
.replace("Streamed ","")
61 # Now, everything should be in the form "1 year ago"
62 value
, unit
, _
= s
.split(" ")
66 ).get(unit
, unit
[0]) # first letter otherwise (e.g. year(s) => y)
68 return f
"{value}{suffix}"
70 def log_unknown_card(data
):
73 from flask
import request
75 except: source
= "unknown"
76 with
open("/tmp/innertube.err", "a") as f
:
77 f
.write(f
"\n/***** {source} *****/\n")
78 json
.dump(data
, f
, indent
=2)
80 def parse_result_items(items
):
81 # TODO: use .get() for most non-essential attributes
83 parses youtube search response into an easier to use format.
87 key
= next(iter(item
.keys()), None)
89 if key
== 'videoRenderer':
90 is_live
= listfind(content
.get('badges',[]), 'metadataBadgeRenderer').get('style') == 'BADGE_STYLE_TYPE_LIVE_NOW'
91 results
.append({'type': 'VIDEO', 'content': {
92 'video_id': content
['videoId'],
93 'title': content
['title']['runs'][0]['text'],
94 'author': content
['longBylineText']['runs'][0]['text'] or \
95 content
['shortBylineText']['runs'][0]['text'],
96 'channel_id': content
['ownerText']['runs'][0] \
97 ['navigationEndpoint']['browseEndpoint']['browseId'],
98 'length': content
.get('lengthText',{}).get('simpleText') \
99 if not is_live
else 'LIVE', # "44:07", "1:41:50"
100 'views': toInt(content
.get('viewCountText',{}).get('simpleText') or # "123,456 views"
101 listget(content
.get('viewCountText',{}).get('runs'),0,{}).get('text')), # "1,234 watching"
102 'published': age(content
.get('publishedTimeText',{}).get('simpleText')),
104 elif key
== 'playlistRenderer':
105 results
.append({'type': 'PLAYLIST', 'content': {
106 'playlist_id': content
['navigationEndpoint']['watchEndpoint']['playlistId'],
107 'video_id': content
['navigationEndpoint']['watchEndpoint']['videoId'],
108 'title': content
['title']['simpleText'],
109 'author': content
['longBylineText']['runs'][0]['text'] or
110 content
['shortBylineText']['runs'][0]['text'],
111 'channel_id': content
['longBylineText']['runs'][0]['navigationEndpoint']['browseEndpoint']['browseId'], # OR .shortBylineText
112 'n_videos': toInt(content
['videoCount']),
114 elif key
== 'radioRenderer': # "Mix" playlists
115 results
.append({'type': 'PLAYLIST', 'content': {
116 'playlist_id': content
['playlistId'],
117 'video_id': content
['navigationEndpoint']['watchEndpoint']['videoId'],
118 'title': content
['title']['simpleText'],
119 'author': content
['longBylineText']['simpleText'] or \
120 content
['shortBylineText']['simpleText'] , # always "YouTube"
122 'n_videos': content
['videoCountShortText']['runs'][0]['text'] or \
123 content
['videoCountText']['runs'][0]['text'],
124 # videoCountShortText: "50+"; videoCountText: "50+ videos"
126 elif key
== 'channelRenderer':
127 results
.append({'type': 'CHANNEL', 'content': {
128 'channel_id': content
['channelId'],
129 'title': content
['title']['simpleText'],
130 'icons': mkthumbs(content
['thumbnail']['thumbnails']),
131 'subscribers': content
['subscriberCountText']['simpleText'], # "2.47K subscribers"
133 elif key
== 'shelfRenderer':
134 results
.extend([item
for item
in
135 parse_result_items(content
['content']['verticalListRenderer']['items'])
137 elif key
== 'movieRenderer': # movies to buy/rent
139 elif key
== 'carouselAdRenderer': # haha, no.
141 elif key
== 'horizontalCardListRenderer':
142 # suggested searches: .cards[].searchRefinementCardRenderer.query.runs[].text
145 log_unknown_card(item
)
148 def parse_infocard(card
):
150 parses a single infocard into a format that's easier to handle.
152 card
= card
['cardRenderer']
153 ctype
= list(card
['content'].keys())[0]
154 content
= card
['content'][ctype
]
155 if ctype
== "pollRenderer":
156 return {'type': "POLL", 'content': {
157 'question': content
['question']['simpleText'],
158 'answers': [(a
['text']['simpleText'],a
['numVotes']) \
159 for a
in content
['choices']],
161 elif ctype
== "videoInfoCardContentRenderer":
162 is_live
= content
.get('badge',{}).get('liveBadgeRenderer') is not None
163 return {'type': "VIDEO", 'content': {
164 'video_id': content
['action']['watchEndpoint']['videoId'],
165 'title': content
['videoTitle']['simpleText'],
166 'author': delL(content
['channelName']['simpleText']),
167 'length': content
.get('lengthString',{}).get('simpleText') \
168 if not is_live
else "LIVE", # "23:03"
169 'views': toInt(content
.get('viewCountText',{}).get('simpleText')),
170 # XXX: views sometimes "Starts: July 31, 2020 at 1:30 PM"
172 elif ctype
== "playlistInfoCardContentRenderer":
173 return {'type': "PLAYLIST", 'content': {
174 'playlist_id': content
['action']['watchEndpoint']['playlistId'],
175 'video_id': content
['action']['watchEndpoint']['videoId'],
176 'title': content
['playlistTitle']['simpleText'],
177 'author': delL(content
['channelName']['simpleText']),
178 'n_videos': toInt(content
['playlistVideoCount']['simpleText']),
180 elif ctype
== "simpleCardContentRenderer" and \
181 'urlEndpoint' in content
['command']:
182 return {'type': "WEBSITE", 'content': {
183 'url': clean_url(content
['command']['urlEndpoint']['url']),
184 'domain': content
['displayDomain']['simpleText'],
185 'title': content
['title']['simpleText'],
186 # XXX: no thumbnails for infocards
188 elif ctype
== "collaboratorInfoCardContentRenderer":
189 return {'type': "CHANNEL", 'content': {
190 'channel_id': content
['endpoint']['browseEndpoint']['browseId'],
191 'title': content
['channelName']['simpleText'],
192 'icons': mkthumbs(content
['channelAvatar']['thumbnails']),
193 'subscribers': content
.get('subscriberCountText',{}).get('simpleText',''), # "545K subscribers"
196 log_unknown_card(card
)
199 def parse_endcard(card
):
201 parses a single endcard into a format that's easier to handle.
203 card
= card
.get('endscreenElementRenderer', card
) #only sometimes nested
204 ctype
= card
['style']
205 if ctype
== "CHANNEL":
206 return {'type': ctype
, 'content': {
207 'channel_id': card
['endpoint']['browseEndpoint']['browseId'],
208 'title': card
['title']['simpleText'],
209 'icons': mkthumbs(card
['image']['thumbnails']),
211 elif ctype
== "VIDEO":
212 return {'type': ctype
, 'content': {
213 'video_id': card
['endpoint']['watchEndpoint']['videoId'],
214 'title': card
['title']['simpleText'],
215 'length': card
['videoDuration']['simpleText'], # '12:21'
216 'views': toInt(card
['metadata']['simpleText']),
217 # XXX: no channel name
219 elif ctype
== "PLAYLIST":
220 return {'type': ctype
, 'content': {
221 'playlist_id': card
['endpoint']['watchEndpoint']['playlistId'],
222 'video_id': card
['endpoint']['watchEndpoint']['videoId'],
223 'title': card
['title']['simpleText'],
224 'author': delL(card
['metadata']['simpleText']),
225 'n_videos': toInt(card
['playlistLength']['simpleText']),
227 elif ctype
== "WEBSITE" or ctype
== "CREATOR_MERCHANDISE":
228 url
= clean_url(card
['endpoint']['urlEndpoint']['url'])
229 return {'type': "WEBSITE", 'content': {
231 'domain': urlparse(url
).netloc
,
232 'title': card
['title']['simpleText'],
233 'icons': mkthumbs(card
['image']['thumbnails']),
236 log_unknown_card(card
)