locsec/twscrape
1
Форкнуть 0
зеркало из https://github.com/viginum-datalab/twscrape.git synced 2025-10-28 20:54:24 +02:00
Код Задачи Пакеты Проекты Релизы Вики Активность

fix: decode zstd response

Просмотр исходного кода
Этот коммит содержится в:
viginum-datalab 2025-01-15 17:08:16 +01:00
родитель 393cfbc425
Коммит 6431dac7d0
2 изменённых файлов: 26 добавлений и 19 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

31
twscrape/api.py
Просмотреть файл

@ -1,7 +1,9 @@
from contextlib import aclosing
from httpx import Response
from json import loads
from typing_extensions import deprecated
from zstd import decompress
from .accounts_pool import AccountsPool
from .logger import set_log_level
@ -126,7 +128,8 @@ class API:
if rep is None:
return
obj = rep.json()
encoding: str | None = rep.headers.get("content-encoding")
obj = loads(decompress(rep.content)) if encoding == "zstd" else rep.json()
els = get_by_path(obj, "entries") or []
els = [
x
@ -138,7 +141,7 @@ class API:
]
cur = self._get_cursor(obj, cursor_type)
rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit)
rep, cnt, active = self._is_end(obj, queue, els, cur, cnt, limit)
if rep is None:
return
@ -169,7 +172,7 @@ class API:
async def search(self, q: str, limit=-1, kv=None):
async with aclosing(self.search_raw(q, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
yield x
# user_by_id
@ -261,7 +264,7 @@ class API:
async def tweet_replies(self, twid: int, limit=-1, kv=None):
async with aclosing(self.tweet_replies_raw(twid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
if x.inReplyToTweetId == twid:
yield x
@ -278,7 +281,7 @@ class API:
async def followers(self, uid: int, limit=-1, kv=None):
async with aclosing(self.followers_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# verified_followers
@ -296,7 +299,7 @@ class API:
async def verified_followers(self, uid: int, limit=-1, kv=None):
async with aclosing(self.verified_followers_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# following
@ -311,7 +314,7 @@ class API:
async def following(self, uid: int, limit=-1, kv=None):
async with aclosing(self.following_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# subscriptions
@ -326,7 +329,7 @@ class API:
async def subscriptions(self, uid: int, limit=-1, kv=None):
async with aclosing(self.subscriptions_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# retweeters
@ -341,7 +344,7 @@ class API:
async def retweeters(self, twid: int, limit=-1, kv=None):
async with aclosing(self.retweeters_raw(twid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# favoriters
@ -358,7 +361,7 @@ class API:
async def favoriters(self, twid: int, limit=-1, kv=None):
async with aclosing(self.favoriters_raw(twid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_users(rep.json(), limit):
for x in parse_users(rep, limit):
yield x
# user_tweets
@ -381,7 +384,7 @@ class API:
async def user_tweets(self, uid: int, limit=-1, kv=None):
async with aclosing(self.user_tweets_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
yield x
# user_tweets_and_replies
@ -404,7 +407,7 @@ class API:
async def user_tweets_and_replies(self, uid: int, limit=-1, kv=None):
async with aclosing(self.user_tweets_and_replies_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
yield x
# user_media
@ -476,7 +479,7 @@ class API:
async def liked_tweets(self, uid: int, limit=-1, kv=None):
async with aclosing(self.liked_tweets_raw(uid, limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
yield x
# Get current user bookmarks
@ -502,5 +505,5 @@ class API:
async def bookmarks(self, limit=-1, kv=None):
async with aclosing(self.bookmarks_raw(limit=limit, kv=kv)) as gen:
async for rep in gen:
for x in parse_tweets(rep.json(), limit):
for x in parse_tweets(rep, limit):
yield x

14
twscrape/queue_client.py
Просмотреть файл

@ -1,6 +1,7 @@
import json
from json import JSONDecodeError, dumps, loads
import os
from typing import Any
from zstd import decompress
import httpx
from httpx import AsyncClient, Response
@ -56,8 +57,10 @@ def dump_rep(rep: Response):
msg.append("\n")
try:
msg.append(json.dumps(rep.json(), indent=2))
except json.JSONDecodeError:
encoding: str | None = rep.headers.get("content-encoding")
obj = loads(decompress(rep.content)) if encoding == "zstd" else rep.json()
msg.append(dumps(obj, indent=2))
except JSONDecodeError:
msg.append(rep.text)
txt = "\n".join(msg)
@ -120,8 +123,9 @@ class QueueClient:
dump_rep(rep)
try:
res = rep.json()
except json.JSONDecodeError:
encoding: str | None = rep.headers.get("content-encoding")
res = loads(decompress(rep.content)) if encoding == "zstd" else rep.json()
except JSONDecodeError:
res: Any = {"_raw": rep.text}
limit_remaining = int(rep.headers.get("x-rate-limit-remaining", -1))