Replace tokens with guest accounts, swap endpoints

.github/workflows/run-tests.yml

@@ -10,25 +10,34 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: buildjet-2vcpu-ubuntu-2204
+    strategy:
+      matrix:
+        nim:
+          - "1.6.10"
+          - "1.6.x"
+          - "2.0.x"
+          - "devel"
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Cache nimble
         id: cache-nimble
-        uses: actions/cache@v3
+        uses: buildjet/cache@v3
         with:
           path: ~/.nimble
-          key: nimble-${{ hashFiles('*.nimble') }}
-          restore-keys: "nimble-"
+          key: ${{ matrix.nim }}-nimble-${{ hashFiles('*.nimble') }}
+          restore-keys: |
+            ${{ matrix.nim }}-nimble-
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
           cache: "pip"
       - uses: jiro4989/setup-nim-action@v1
         with:
-          nim-version: "1.x"
+          nim-version: ${{ matrix.nim }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - run: nimble build -d:release -Y
       - run: pip install seleniumbase
       - run: seleniumbase install chromedriver
@@ -37,9 +46,11 @@ jobs:
         run: |
           sudo apt install libsass-dev -y
           cp nitter.example.conf nitter.conf
+          sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
           nimble md
           nimble scss
+          echo '${{ secrets.GUEST_ACCOUNTS }}' > ./guest_accounts.jsonl
       - name: Run tests
         run: |
           ./nitter &
-          pytest -n4 tests
+          pytest -n8 tests

.gitignore

@@ -10,4 +10,5 @@ nitter
 /public/css/style.css
 /public/md/*.html
 nitter.conf
+guest_accounts.json*
 dump.rdb

Dockerfile.arm64

@@ -1,7 +1,7 @@
 FROM alpine:3.18 as nim
 LABEL maintainer="[email protected]"
 
-RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.14-r0" nimble pcre
+RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2"
 
 WORKDIR /src/nitter
 
@@ -15,9 +15,11 @@ RUN nimble build -d:danger -d:lto -d:strip \
 
 FROM alpine:3.18
 WORKDIR /src/
-RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
+RUN apk --no-cache add pcre ca-certificates openssl1.1-compat
 COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
 EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter

config.nims

@@ -7,12 +7,7 @@
 
 # disable annoying warnings
 warning("GcUnsafe2", off)
+warning("HoleEnumConv", off)
 hint("XDeclaredButNotUsed", off)
 hint("XCannotRaiseY", off)
 hint("User", off)
-
-const
-  nimVersion = (major: NimMajor, minor: NimMinor, patch: NimPatch)
-
-when nimVersion >= (1, 6, 0):
-  warning("HoleEnumConv", off)

nitter.example.conf

@@ -23,7 +23,7 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints (/.tokens)
+enableDebug = false  # enable request logs and debug endpoints (/.accounts)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
 tokenCount = 10

nitter.nimble

@@ -14,16 +14,16 @@ requires "nim >= 1.6.10"
 requires "jester#baca3f"
 requires "karax#5cf360c"
 requires "sass#7dfdd03"
-requires "nimcrypto#4014ef9"
+requires "nimcrypto#a079df9"
 requires "markdown#158efe3"
 requires "packedjson#9e6fbb6"
 requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#ca5989a"
 requires "flatty#e668085"
-requires "jsony#ea811be"
-
+requires "jsony#1de1f08"
+requires "oauth#b8c163b"
 
 # Tasks
 

src/api.nim

@@ -33,23 +33,6 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
     js = await fetch(url ? params, apiId)
   result = parseGraphTimeline(js, "user", after)
 
-# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
-#   if id.len == 0: return
-#   let
-#     ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-#     url = oldUserTweets / (id & ".json") ? ps
-#   result = parseTimeline(await fetch(url, Api.timeline), after)
-
-proc getUserTimeline*(id: string; after=""): Future[Profile] {.async.} =
-  var ps = genParams({"id": id})
-  if after.len > 0:
-    ps.add ("down_cursor", after)
-
-  let
-    url = legacyUserTweets ? ps
-    js = await fetch(url, Api.userTimeline)
-  result = parseUserTimeline(js, after)
-
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
@@ -112,10 +95,10 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
   if after.len > 0:
     result.replies = await getReplies(id, after)
 
-proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
+proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   let q = genQueryParam(query)
   if q.len == 0 or q == emptyQuery:
-    return Profile(tweets: Timeline(query: query, beginning: true))
+    return Timeline(query: query, beginning: true)
 
   var
     variables = %*{
@@ -129,44 +112,29 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
   if after.len > 0:
     variables["cursor"] = % after
   let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = Profile(tweets: parseGraphSearch(await fetch(url, Api.search), after))
-  result.tweets.query = query
-
-proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
-  var q = genQueryParam(query)
-
-  if q.len == 0 or q == emptyQuery:
-    return Timeline(query: query, beginning: true)
-
-  if after.len > 0:
-    q &= " max_id:" & after
-
-  let url = tweetSearch ? genParams({
-    "q": q ,
-    "modules": "status",
-    "result_type": "recent",
-  })
-
-  result = parseTweetSearch(await fetch(url, Api.search), after)
+  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
   result.query = query
 
-proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
   if query.text.len == 0:
     return Result[User](query: query, beginning: true)
 
-  var url = userSearch ? {
-    "q": query.text,
-    "skip_status": "1",
-    "count": "20",
-    "page": page
-  }
+  var
+    variables = %*{
+      "rawQuery": query.text,
+      "count": 20,
+      "product": "People",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+    result.beginning = false
 
-  result = parseUsers(await fetchRaw(url, Api.userSearch))
+  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphSearch[User](await fetch(url, Api.search), after)
   result.query = query
-  if page.len == 0:
-    result.bottom = "2"
-  elif page.allCharsInSet(Digits):
-    result.bottom = $(parseInt(page) + 1)
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return