# functions that deal with parsing data from youtube's internal API ("innertube")

from urllib.parse import parse_qs, urlparse
import re

def findall(obj, key):
    """
    given a list of dicts, where one dict contains a given key, return said key.
    """
    if obj is None: return []
    return [ obj[key] for obj in obj if key in obj.keys() ]
def listget(obj, index, fallback=None):
    if obj is None: return fallback
    return next(iter(obj[index:]), fallback)
flatten = lambda l: [item for sublist in l for item in sublist] # https://stackoverflow.com/a/952952
first = lambda l: next(iter(l),{})
listfind = lambda obj,key: first(findall(obj,key))

class G:
    """
    null-coalescing version of dict.get() that also works on lists.

    the | operator is overloaded to achieve similar looking code to jq(1) filters.
    the first found key is used: dict(foo=1)|G('bar','foo') returns 1.
    """
    def __init__(self, *keys):
        self.keys = keys
    def __ror__(self, other):
        for key in self.keys:
            try:    return other[key]
            except: continue
        return None
    class _Text:
        """ parses youtube's .runs[].text and .simpleText variants """
        def __ror__(self, other): # Note: only returning runs[0], not concat'ing all!
            return other|G('simpleText') or other|G('runs')|G(0)|G('text')
    text = _Text()
class Select:
    """ |Select('foo') returns the first foo in list, |Select(all='foo') returns all foos. """
    def __init__(self, key=None, *, all=None):
        self.key = key or all
        self.all = all
    def __ror__(self, other):
        try: items = [ other[self.key] for other in other if self.key in other.keys() ]
        except: items = []
        return items if self.all else items|G(0)
class A:
    """ apply """
    def __init__(self, f, *args):
        self.f = f
        self.args = args
    def __ror__(self, other):
        return self.f(other, *self.args)
    class _Int:
        def __ror__(self, other):
            try:    return int(''.join(filter(str.isdigit, other)))
            except: return None
    int = _Int()


def prepare_searchresults(yt_results):
    contents = ( # from continuation token
        yt_results
        |G('onResponseReceivedCommands')
        |Select('appendContinuationItemsAction')
        |G('continuationItems')
    ) or ( # from page 1
        yt_results
        |G('contents')
        |G('twoColumnSearchResultsRenderer')
        |G('primaryContents')
        |G('sectionListRenderer')
        |G('contents')
    )
    items = flatten([c.get('contents',[]) for c in contents|Select(all='itemSectionRenderer')])
    items, extra = parse_result_items(items)
    more = contents|Select("continuationItemRenderer")|G("continuationEndpoint")|G("continuationCommand")|G("token")
    estimatedResults = yt_results|G("estimatedResults")

    return items, extra, more

def prepare_infocards(metadata):
    cards = metadata.get('cards',{}).get('cardCollectionRenderer',{}).get('cards',[])
    return list(filter(None, map(parse_infocard, cards)))

def prepare_endcards(metadata):
    endsc = metadata.get('endscreen',{}).get('endscreenRenderer',{}).get('elements',[])
    return list(filter(None, map(parse_endcard, endsc)))

def prepare_channel(response, channel_id):
    meta1 = response|G('metadata')|G('channelMetadataRenderer')
    meta2 = response|G('microformat')|G('microformatDataRenderer')
    title = meta1|G('title') or meta2|G('title')
    descr = meta1|G('description') or meta2|G('description') # meta2.description is capped at 160chars
    thumb = mkthumbs((meta2|G('thumbnail') or meta1|G('avatar'))|G('thumbnails') or {}) # .avatar ~ 900px

    contents = response|G('continuationContents')
    if not contents: # overran end of list
        return title, descr, thumb, [], False

    unparsed = contents|G('gridContinuation')|G('items') or \
            contents|G('sectionListContinuation')|G('contents') or []
    items, extra = parse_channel_items(unparsed, channel_id, title)
    more = (
        contents
        |G('gridContinuation', 'sectionListContinuation')
        |G('continuations')
        |Select('nextContinuationData')
        |G('continuation')
    )

    return title, descr, thumb, items, more

def prepare_playlist(result):
    contents = result['continuationContents']
    unparsed = contents['playlistVideoListContinuation'].get('contents',[])
    more = (
        contents
        |G('playlistVideoListContinuation')
        |G('continuations')
        |Select('nextContinuationData')
        |G('continuation')
    )

    return list(filter(None, map(parse_playlist, unparsed))), more

def mkthumbs(thumbs):
    output = {str(e['height']): e['url'] for e in thumbs}
    largest=next(iter(sorted(output.keys(),reverse=True,key=int)),None)
    return {**output, 'largest': largest}

def clean_url(url):
    # externals URLs are redirected through youtube.com/redirect, but we
    # may encounter internal URLs, too
    return parse_qs(urlparse(url).query).get('q',[url])[0]

def toInt(s, fallback=0):
    if s is None:
        return fallback
    try:
        return int(''.join(filter(str.isdigit, s)))
    except ValueError:
        return fallback

# Remove left-/rightmost word from string:
delL = lambda s: s.partition(' ')[2]

def age(s):
    if s is None: # missing from autogen'd music, some livestreams
        return None
    # Some livestreams have "Streamed 7 hours ago"
    s = s.replace("Streamed ","")
    # Now, everything should be in the form "1 year ago"
    value, unit, _ = s.split(" ")
    suffix = dict(
        minute='min',
        minutes='min',
    ).get(unit, unit[0]) # first letter otherwise (e.g. year(s) => y)

    return f"{value}{suffix}"

def log_unknown_card(data):
    import json
    try:
        from flask import request
        source = request.url
    except: source = "unknown"
    with open("/tmp/innertube.err", "a") as f:
        f.write(f"\n/***** {source} *****/\n")
        json.dump(data, f, indent=2)

def parse_result_items(items):
    # TODO: use .get() for most non-essential attributes
    """
    parses youtube search response into an easier to use format.
    """
    results = []
    extras = []
    for item in items:
        key = next(iter(item.keys()), None)
        content = item[key]
        if key == 'videoRenderer':
            results.append({'type': 'VIDEO', 'content': {
                'video_id': content['videoId'],
                'title': content['title']|G.text,
                'author': content|G('longBylineText','shortBylineText')|G.text,
                'channel_id': content|G('ownerText')|G('runs')|G(0) \
                    |G('navigationEndpoint')|G('browseEndpoint')|G('browseId'),
                'length': content|G('lengthText')|G.text, # "44:07", "1:41:50"
                'views': content|G('viewCountText')|G.text|A.int or 0, # "1,234 {views|watching}", absent on 0 views
                'published': content|G('publishedTimeText')|G('simpleText')|A(age),
                'live': content|G('badges')|Select('metadataBadgeRenderer')|G('style')=='BADGE_STYLE_TYPE_LIVE_NOW',
            }})
        elif key in ['playlistRenderer', 'radioRenderer', 'showRenderer']: # radio == "Mix" playlist, show == normal playlist, specially displayed
            results.append({'type': 'PLAYLIST', 'content': {
                'playlist_id': content['navigationEndpoint']|G('watchEndpoint')|G('playlistId') or \
                    content|G('playlistId'), # COURSE/"learning playlist"
                'video_id': content['navigationEndpoint']|G('watchEndpoint')|G('videoId') or \
                    videoid_from_thumbnail(content), # learning playlist
                'title': content['title']|G.text,
                # Note: learning playlists have no author/channel_id
                'author': content|G('longBylineText','shortBylineText')|G.text,
                'channel_id': content|G('longBylineText','shortBylineText')|G('runs')|G(0) \
                    |G('navigationEndpoint')|G('browseEndpoint')|G('browseId'),
                'n_videos': content|G('videoCount')|A.int or \
                    content|G('videoCountShortText','videoCountText')|G.text, # "Mix" playlists
            }})
        elif key == 'channelRenderer':
            results.append({'type': 'CHANNEL', 'content': {
                'channel_id': content['channelId'],
                'title': content['title']|G.text,
                'icons': content['thumbnail']['thumbnails']|A(mkthumbs),
                'subscribers': content|G('subscriberCountText')|G('simpleText'), # "2.47K subscribers"
            }})
        elif key == 'shelfRenderer':
            subkey = next(iter(content['content'].keys()), None) #verticalListRenderer/horizontalMovieListRenderer
            r, e = parse_result_items(content['content'][subkey]['items'])
            results.extend(r)
            extras.extend(e)
        elif key in ['movieRenderer', 'gridMovieRenderer']: # movies to buy/rent
            pass # gMR.{videoId,title.runs[].text,lengthText.simpleText}
        elif key in ['carouselAdRenderer','searchPyvRenderer','promotedSparklesTextSearchRenderer']: # haha, no.
            pass
        elif key == 'horizontalCardListRenderer':
            # suggested searches: .cards[].searchRefinementCardRenderer.query.runs[].text
            pass
        elif key == 'emergencyOneboxRenderer': # suicide prevention hotline
            pass
        elif key in ['clarificationRenderer', 'infoPanelContainerRenderer']: # COVID-19/conspiracy theory infos
            pass
        elif key == 'webAnswerRenderer': # "Result from the web"
            pass
        elif key == 'didYouMeanRenderer' or key == 'showingResultsForRenderer':
            extras.append({
                'type': 'spelling',
                'query': content['correctedQueryEndpoint']['searchEndpoint']['query'], # non-misspelled query
                'autocorrected': key == 'showingResultsForRenderer',
            })
        elif key == 'messageRenderer': # "No more results"
            extras.append({
                'type': 'message',
                'message': content|G('title','text')|G.text,
            })
        elif key == 'backgroundPromoRenderer': # e.g. "no results"
            extras.append({
                'type': content['icon']['iconType'],
                'message': content['title']|G.text,
            })
        else:
            log_unknown_card(item)
    return results, extras

def parse_infocard(card):
    """
    parses a single infocard into a format that's easier to handle.
    """
    card = card['cardRenderer']
    ctype = list(card['content'].keys())[0]
    content = card['content'][ctype]
    if ctype == "pollRenderer":
        return {'type': "POLL", 'content': {
            'question': content['question']['simpleText'],
            'answers': [(a['text']['simpleText'],a['numVotes']) \
                for a in content['choices']],
        }}
    elif ctype == "videoInfoCardContentRenderer":
        is_live = content.get('badge',{}).get('liveBadgeRenderer') is not None
        return {'type': "VIDEO", 'content': {
            'video_id': content['action']['watchEndpoint']['videoId'],
            'title': content['videoTitle']['simpleText'],
            'author': delL(content['channelName']['simpleText']),
            'length': content.get('lengthString',{}).get('simpleText') \
                if not is_live else "LIVE", # "23:03"
            'views': toInt(content.get('viewCountText',{}).get('simpleText')),
                # XXX: views sometimes "Starts: July 31, 2020 at 1:30 PM"
        }}
    elif ctype == "playlistInfoCardContentRenderer":
        return {'type': "PLAYLIST", 'content': {
            'playlist_id': content['action']['watchEndpoint']['playlistId'],
            'video_id': content['action']['watchEndpoint']['videoId'],
            'title': content['playlistTitle']['simpleText'],
            'author': delL(content['channelName']['simpleText']),
            'n_videos': toInt(content['playlistVideoCount']['simpleText']),
        }}
    elif ctype == "simpleCardContentRenderer" and \
            'urlEndpoint' in content['command']:
        return {'type': "WEBSITE", 'content': {
            'url': clean_url(content['command']['urlEndpoint']['url']),
            'domain': content['displayDomain']['simpleText'],
            'title': content['title']['simpleText'],
            # XXX: no thumbnails for infocards
        }}
    elif ctype == "collaboratorInfoCardContentRenderer":
        return {'type': "CHANNEL", 'content': {
            'channel_id': content['endpoint']['browseEndpoint']['browseId'],
            'title': content['channelName']['simpleText'],
            'icons': mkthumbs(content['channelAvatar']['thumbnails']),
            'subscribers': content.get('subscriberCountText',{}).get('simpleText',''), # "545K subscribers"
        }}
    else:
        log_unknown_card(card)
        return None

def parse_endcard(card):
    """
    parses a single endcard into a format that's easier to handle.
    """
    card = card.get('endscreenElementRenderer', card) #only sometimes nested
    ctype = card['style']
    if ctype == "CHANNEL":
        return {'type': ctype, 'content': {
            'channel_id': card['endpoint']['browseEndpoint']['browseId'],
            'title': card['title']|G.text,
            'icons': mkthumbs(card['image']['thumbnails']),
        }}
    elif ctype == "VIDEO":
        if not 'endpoint' in card: return None # title == "This video is unavailable."
        return {'type': ctype, 'content': {
            'video_id': card['endpoint']['watchEndpoint']['videoId'],
            'title': card['title']|G.text,
            'length': card['videoDuration']|G.text,  # '12:21'
            'views': toInt(card['metadata']|G.text),
            # XXX: no channel name
        }}
    elif ctype == "PLAYLIST":
        return {'type': ctype, 'content': {
            'playlist_id': card['endpoint']['watchEndpoint']['playlistId'],
            'video_id': card['endpoint']['watchEndpoint']['videoId'],
            'title': card['title']|G.text,
            'author': delL(card['metadata']|G.text),
            'n_videos': toInt(card['playlistLength']|G.text),
        }}
    elif ctype == "WEBSITE" or ctype == "CREATOR_MERCHANDISE":
        url = clean_url(card['endpoint']['urlEndpoint']['url'])
        return {'type': "WEBSITE", 'content': {
            'url': url,
            'domain': urlparse(url).netloc,
            'title': card['title']|G.text,
            'icons': mkthumbs(card['image']['thumbnails']),
        }}
    else:
        log_unknown_card(card)
        return None

def videoid_from_thumbnail(content):
    # learning playlist; example: PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5 (/user/enyay/playlists)
    return re.match(r"https?://i.ytimg.com/vi/([-_0-9a-zA-Z]{11})|()",
        listget(listget(content.get('thumbnails',[]),0,{}).get('thumbnails',[]),0,{}).get('url','')
    ).group(1)

def parse_channel_items(items, channel_id, author):
    result = []
    extra = []
    for item in items:
        key = next(iter(item.keys()), None)
        content = item[key]
        if key in ["gridVideoRenderer", "videoRenderer", "videoCardRenderer"]:
            # only videoCardRenderer (topic channels) has author and channel, others fall back to supplied ones.
            result.append({'type': 'VIDEO', 'content': {
                'video_id': content['videoId'],
                'title': content['title'].get('simpleText') or content['title'].get('runs',[{}])[0].get('text'),
                'author': content.get('bylineText',{}).get('runs',[{}])[0].get('text') or author,
                'channel_id': content.get('bylineText',{}).get('runs',[{}])[0] \
                    .get('navigationEndpoint',{}).get('browseEndpoint',{}).get('browseId') or channel_id,
                'length': (content.get('lengthText',{}).get('simpleText') or # topic channel
                    listfind(content.get('thumbnailOverlays',[]),'thumbnailOverlayTimeStatusRenderer')
                        .get('text',{}).get('simpleText')),
                # topic channel: .metadataText.simpleText = "22M views \u00b7 2 months ago"
                'views': toInt(content.get('viewCountText',{}).get('simpleText')),
                'published': age(content.get('publishedTimeText',{}).get('simpleText')),
            }})
        elif key == "gridPlaylistRenderer" or key == "playlistRenderer":
            result.append({'type': 'PLAYLIST', 'content': {
                'playlist_id': content['navigationEndpoint'].get('watchEndpoint',{}).get('playlistId') or content.get('playlistId'),
                'video_id': content['navigationEndpoint'].get('watchEndpoint',{}).get('videoId',{}) or videoid_from_thumbnail(content),
                'title': (content['title'].get('simpleText') or # playlistRenderer
                    content['title']['runs'][0]['text']), # gridPlaylistRenderer
                'author': author,
                'channel_id': channel_id,
                'n_videos': toInt(content.get('videoCount') or # playlistRenderer
                    content.get('videoCountShortText',{}).get('simpleText') or # grid(1)
                    content.get('videoCountText',{}).get('runs',[{}])[0].get('text')), # grid(2)
            }})
        elif key == "showRenderer":
            result.append({'type': 'PLAYLIST', 'content': {
                'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'],
                'video_id': content['navigationEndpoint']['watchEndpoint']['videoId'],
                'title': content['title']['simpleText'],
                'author': author,
                'channel_id': channel_id,
                'n_videos': None,
            }})
        elif key in ["gridShowRenderer"]:
            result.append({'type': 'PLAYLIST', 'content': {
                'playlist_id': (content|G('navigationEndpoint')
                    |G('browseEndpoint')|G('browseId'))[2:],
                #^: playlistId prefixed with 'VL', which must be removed
                'video_id': None,
                'title': content|G('title')|G.text,
                'author': author,
                'channel_id': channel_id,
                'n_videos': content|G('thumbnailOverlays')|G(0)
                    |G('thumbnailOverlayBottomPanelRenderer')|G('text')|G.text,
            }})
        elif key in ["itemSectionRenderer", "gridRenderer", "horizontalCardListRenderer", "horizontalListRenderer"]:
                newkey = {
                    "itemSectionRenderer": 'contents',
                    "gridRenderer": 'items',
                    "horizontalCardListRenderer": 'cards',
                    "horizontalListRenderer": 'items',
                }.get(key)
                r, e = parse_channel_items(content[newkey], channel_id, author)
                result.extend(r)
                extra.extend(e)
        elif key == "shelfRenderer":
                r, e = parse_channel_items([content['content']], channel_id, author)
                result.extend(r)
                extra.extend(e)
        elif key == "messageRenderer":
            # e.g. {'messageRenderer': {'text': {'runs': [{'text': 'This channel has no playlists.'}]}}}
            pass
        elif key == "gameCardRenderer":
            pass
        elif key == "gridChannelRenderer":
            pass # don't care; related channels, e.g. on UCMsgXPD3wzzt8RxHJmXH7hQ
        else:
            log_unknown_card(item)

    return result, extra

def parse_playlist(item):
    key = next(iter(item.keys()), None)
    content = item[key]
    if key == "playlistVideoRenderer":
        if not content.get('isPlayable', False):
            return None # private or deleted video

        return {'type': 'VIDEO', 'content': {
            'video_id': content['videoId'],
            'title': (content['title'].get('simpleText') or # playable videos
                content['title'].get('runs',[{}])[0].get('text')), # "[Private video]"
            'playlist_id': content['navigationEndpoint']['watchEndpoint']['playlistId'],
            'index': content['navigationEndpoint']['watchEndpoint'].get('index',0), #or int(content['index']['simpleText']) (absent on course intros; e.g. PL96C35uN7xGJu6skU4TBYrIWxggkZBrF5)
            # rest is missing from unplayable videos:
            'author': content.get('shortBylineText',{}).get('runs',[{}])[0].get('text'),
            'channel_id':content.get('shortBylineText',{}).get('runs',[{}])[0].get('navigationEndpoint',{}).get('browseEndpoint',{}).get('browseId'),
            'length': (content.get("lengthText",{}).get("simpleText") or # "8:51"
                int(content.get("lengthSeconds", 0))), # "531"
            'starttime': content['navigationEndpoint']['watchEndpoint'].get('startTimeSeconds'),
        }}
    else:
        raise Exception(item) # XXX TODO