ticker

   1 #!/bin/sh
   2
   3 # XXX: workaround for not installed binaries
   4 export PATH="$PATH:."
   5
   6 :<<'DOCS'
   7         DerStandard live ticker -- extractor
   8         Extracts headines, text and embedded content from a derStandard.at
   9         Liveticker and outputs it as tab seperated fields.
  10         Give it an article ID (2000xxxxxxxxx) as argv[1]. Only displays
  11         editorial content (no user's comments).
  12
  13         requries: curl, jq, websocat (https://github.com/vi/websocat)
  14
  15         TODO:
  16          * detect if live (/jetzt/api/ .hmrcs?)
  17            NOTE: .M[0].M == "updateReportMetaDataWithReadonly"
  18                  -> .M[0].A[0].rcs -> 0=live, 1=nachlese
  19
  20         Copyright (C) 2019 Tobias Girstmair; GNU GPL v3
  21 DOCS
  22
  23 curl="curl -s -L -AGoogleBot/2.1"
  24 ticker_id=${1}
  25 backlog=${2:-9999}
  26 test -z "$ticker_id" && {
  27         echo "Usage: $0 TICKER-ID [BACKLOG]" >&2
  28         echo "try:      $0 2000103942403" >&2
  29         exit 1
  30 }
  31
  32 # trim ticker to only contain ID (if URL supplied):
  33 ticker_id=${ticker_id#*/livebericht/}
  34 ticker_id=${ticker_id%%[-/]*}
  35
  36 mkurl() { # params: 1=protocol 2=endpoint 3=ticker_id (if any) 4=token (if any)
  37         echo "${1}://live.derstandard.at/jetzt/signalr/hub/${2}
  38                 ?transport=webSockets&clientProtocol=1.5&lbid=${3}
  39                 &v=1.0.7082.29332&connectionToken=${4}
  40                 &connectionData=%5B%7B%22name%22%3A%22reporthub%22%7D%5D
  41                 &tid=2" | tr -d ' \t\n'
  42 }
  43
  44 # load old items; oldest first:
  45 test "$backlog" -gt 0 &&
  46 $curl -s "https://derstandard.at/jetzt/api/redcontent?id=${ticker_id}&ps=$backlog" |
  47     jq -r '.rcs | reverse | .[] |
  48         ["addRedContentItem", .id, .ctd, .hl, .cm, .td.pq.hl, .td.pq.cm, .td.epu, .td.mu, .td.ec],
  49         ["updateVotes", .id, .vp, .vn] | @tsv'
  50
  51 token=$($curl -s $(mkurl "https" "negotiate") | jq -r '.ConnectionToken | @uri')
  52 # send after first response from websocket (hacky):
  53 mkdir -p logs # XXX: temporary: save livetickers for later analysis
  54 (sleep 1; $curl -sf `mkurl "https" "start" "$ticker_id" "$token"`>/dev/null) &
  55 websocat $(mkurl "wss" "connect" "$ticker_id" "$token") |
  56     tee "logs/$ticker_id-`date +%s`" |
  57     jq --unbuffered -r '.M[0] | .M as $type |
  58         if $type == "addRedContentItem" or $type == "updateRedContentItem" then
  59                 .A[0] | [$type, .RedContentId, .CreateDate, .Headline, .Text, .PostingQuote.headline,
  60                         .PostingQuote.text, .ExternalProviderUri, .MediaUrl, .EmbedCode]
  61         elif $type == "updateVotes" then
  62                 .A[0] | .[] | [$type, .id, .pvc, .nvc]
  63         else
  64                 []
  65         end | @tsv'