From 7ddb60edbec29e4d88bb4b2f8f427869acea8220 Mon Sep 17 00:00:00 2001
From: girst <tobi@isticktoit.net>
Date: Mon, 18 Oct 2021 01:43:41 +0200
Subject: [PATCH] use continuation token instead of manually paginated results

this is how youtube does it, and this also works for channel?sort=oldest
---
 app/browse/__init__.py                | 17 ++++++++---------
 app/browse/templates/channel.html.j2  |  3 +--
 app/browse/templates/playlist.html.j2 |  5 +----
 app/common/innertube.py               | 21 ++++++++++++++++-----
 4 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/app/browse/__init__.py b/app/browse/__init__.py
index 22c19b5..3abb294 100644
--- a/app/browse/__init__.py
+++ b/app/browse/__init__.py
@@ -48,16 +48,13 @@ def search():
 def channel(channel_id, subpage="videos"):
     token = getattr(current_user, 'token', 'guest')
     if subpage == "videos":
-        page = request.args.get('page', 1, type=int)
         sort_by = request.args.get('sort') or "newest"
         query = None
     elif subpage == "playlists":
-        page = request.args.get('page', 1, type=int)
         sort_by = request.args.get('sort', "modified")
         query = None
     elif subpage == "search":
         query = request.args.get('q')
-        page = request.args.get('page', 1, type=int)
         sort_by = None
     else: # we don't support /home, /about, ..., so redirect to /videos.
         return redirect(url_for('.channel', channel_id=channel_id))
@@ -66,7 +63,10 @@ def channel(channel_id, subpage="videos"):
     if not re.match(r"(UC[A-Za-z0-9_-]{22})", channel_id):
         return redirect(url_for('.channel_redirect', user=channel_id))
 
-    result = fetch_ajax(make_channel_params(channel_id, subpage, page, sort_by, query, v3=(subpage != "search")))
+    # if we don't have a continuation, we create parameters for page 1 manually:
+    continuation = request.args.get('continuation') or \
+        make_channel_params(channel_id, subpage, 1, sort_by, query, v3=(subpage != "search"))
+    result = fetch_ajax(continuation)
     error = find_and_parse_error(result)
 
     if result is None: # if fetching from innertube failed, fall back to xmlfeed:
@@ -107,7 +107,6 @@ def channel(channel_id, subpage="videos"):
         channel_img=thumb,
         channel_desc=descr,
         is_subscribed=is_subscribed,
-        page=page,
         has_more=more)
 
 @frontend.route('/user/<user>/')
@@ -135,15 +134,16 @@ def playlist():
     playlist_id = request.args.get('list')
     if not playlist_id:
         raise BadRequest("No playlist ID")
-    page = request.args.get('page', 1, type=int)
 
     xmlfeed = fetch_xml("playlist_id", playlist_id)
     if not xmlfeed:
         raise NotFound("Unable to fetch playlist")
     title, author, _, channel_id, _ = parse_xml(xmlfeed)
 
-    offset = (page-1)*100 # each call returns 100 items
-    result = fetch_ajax(make_playlist_params(playlist_id, offset))
+    # if we don't have a continuation, we create parameters for page 1 manually:
+    continuation = request.args.get('continuation') or \
+        make_playlist_params(playlist_id, 0)
+    result = fetch_ajax(continuation)
     error = find_and_parse_error(result)
 
     if result is None:
@@ -161,7 +161,6 @@ def playlist():
         author=author,
         channel_id=channel_id,
         rows=rows,
-        page=page,
         has_more=more)
 
 @frontend.route('/<something>', strict_slashes=False)
diff --git a/app/browse/templates/channel.html.j2 b/app/browse/templates/channel.html.j2
index e7076a4..7d0c9c7 100644
--- a/app/browse/templates/channel.html.j2
+++ b/app/browse/templates/channel.html.j2
@@ -41,7 +41,6 @@
 {% if not rows|length %}no more results{% endif %}
 
 <div class="pagination-container">
-	{{ macros.pagination("previous", {'page':(-1,1)}, -1) if page > 1 }}
-	{{ macros.pagination("next", {'page':(+1,1)}, +1) if has_more}}
+	{{ macros.pagination("more", {'continuation': has_more}, +1) if has_more}}
 </div>
 {% endblock %}
diff --git a/app/browse/templates/playlist.html.j2 b/app/browse/templates/playlist.html.j2
index 7508202..656d513 100644
--- a/app/browse/templates/playlist.html.j2
+++ b/app/browse/templates/playlist.html.j2
@@ -16,9 +16,6 @@
 {% if not rows|length %}no more results{% endif %}
 
 <div class="pagination-container">
-{% if page is not none %}
-	{{ macros.pagination("previous", {'page':(-1,1)}, -1) if page > 1 }}
-	{{ macros.pagination("next", {'page':(+1,1)}, +1) if has_more }}
-{% endif %}
+	{{ macros.pagination("more", {'continuation': has_more}, +1) if has_more}}
 </div>
 {% endblock %}
diff --git a/app/common/innertube.py b/app/common/innertube.py
index 057939a..c94235c 100644
--- a/app/common/innertube.py
+++ b/app/common/innertube.py
@@ -91,17 +91,28 @@ def prepare_channel(response, channel_id):
     unparsed = contents|G('gridContinuation')|G('items') or \
             contents|G('sectionListContinuation')|G('contents') or []
     items, extra = parse_channel_items(unparsed, channel_id, title)
-    has_more = 'continuations' in \
-            (contents|G('gridContinuation', 'sectionListContinuation') or {})
+    more = (
+        contents
+        |G('gridContinuation', 'sectionListContinuation')
+        |G('continuations')
+        |Select('nextContinuationData')
+        |G('continuation')
+    )
 
-    return title, descr, thumb, items, has_more
+    return title, descr, thumb, items, more
 
 def prepare_playlist(result):
     contents = result['continuationContents']
     unparsed = contents['playlistVideoListContinuation'].get('contents',[])
-    has_more = 'continuations' in contents.get('playlistVideoListContinuation')
+    more = (
+        contents
+        |G('playlistVideoListContinuation')
+        |G('continuations')
+        |Select('nextContinuationData')
+        |G('continuation')
+    )
 
-    return list(filter(None, map(parse_playlist, unparsed))), has_more
+    return list(filter(None, map(parse_playlist, unparsed))), more
 
 def mkthumbs(thumbs):
     output = {str(e['height']): e['url'] for e in thumbs}
-- 
2.39.3