diff --git a/.github/example.png b/.github/example.png new file mode 100644 index 0000000..c2f1f79 Binary files /dev/null and b/.github/example.png differ diff --git a/readme.md b/readme.md index 83c4673..4c16630 100644 --- a/readme.md +++ b/readme.md @@ -13,7 +13,7 @@ Twitter GraphQL API implementation with [SNScrape](https://github.com/JustAnotherArchivist/snscrape) data models.
- example of cli usage + example of cli usage
## Install diff --git a/tests/test_parser.py b/tests/test_parser.py index cf3a920..43de777 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -4,7 +4,7 @@ from typing import Callable from twscrape import API, gather from twscrape.logger import set_log_level -from twscrape.models import Tweet, User, parse_tweet +from twscrape.models import Tweet, User, UserRef, parse_tweet BASE_DIR = os.path.dirname(__file__) DATA_DIR = os.path.join(BASE_DIR, "mocked-data") @@ -49,18 +49,33 @@ def mock_rep(fn: Callable, filename: str, as_generator=False): def check_tweet(doc: Tweet | None): assert doc is not None - assert doc.id is not None - assert doc.id_str is not None assert isinstance(doc.id, int) assert isinstance(doc.id_str, str) - assert doc.id == int(doc.id_str) + assert str(doc.id) == doc.id_str assert doc.url is not None assert doc.id_str in doc.url assert doc.user is not None + assert isinstance(doc.conversationId, int) + assert isinstance(doc.conversationIdStr, str) + assert str(doc.conversationId) == doc.conversationIdStr + + if doc.inReplyToTweetId is not None: + assert isinstance(doc.inReplyToTweetId, int) + assert isinstance(doc.inReplyToTweetIdStr, str) + assert str(doc.inReplyToTweetId) == doc.inReplyToTweetIdStr + + if doc.inReplyToUser: + check_user_ref(doc.inReplyToUser) + + if doc.mentionedUsers: + for x in doc.mentionedUsers: + check_user_ref(x) + obj = doc.dict() assert doc.id == obj["id"] + assert doc.id_str == obj["id_str"] assert doc.user.id == obj["user"]["id"] assert "url" in obj @@ -104,10 +119,9 @@ def check_tweet(doc: Tweet | None): def check_user(doc: User): assert doc.id is not None - assert doc.id_str is not None assert isinstance(doc.id, int) assert isinstance(doc.id_str, str) - assert doc.id == int(doc.id_str) + assert str(doc.id) == doc.id_str assert doc.username is not None assert doc.descriptionLinks is not None @@ -127,6 +141,19 @@ def check_user(doc: User): assert str(doc.id) in txt +def check_user_ref(doc: UserRef): + assert isinstance(doc.id, int) + assert isinstance(doc.id_str, str) + assert str(doc.id) == doc.id_str + + assert doc.username is not None + assert doc.displayname is not None + + obj = doc.dict() + assert doc.id == obj["id"] + assert doc.id_str == obj["id_str"] + + async def test_search(): api = API() mock_rep(api.search_raw, "raw_search", as_generator=True) diff --git a/twscrape/models.py b/twscrape/models.py index de47ae4..fd5b194 100644 --- a/twscrape/models.py +++ b/twscrape/models.py @@ -84,13 +84,19 @@ class TextLink(JSONTrait): @dataclass class UserRef(JSONTrait): id: int + id_str: str username: str displayname: str _type: str = "snscrape.modules.twitter.UserRef" @staticmethod def parse(obj: dict): - return UserRef(id=int(obj["id_str"]), username=obj["screen_name"], displayname=obj["name"]) + return UserRef( + id=int(obj["id_str"]), + id_str=obj["id_str"], + username=obj["screen_name"], + displayname=obj["name"], + ) @dataclass @@ -163,6 +169,7 @@ class Tweet(JSONTrait): likeCount: int quoteCount: int conversationId: int + conversationIdStr: str hashtags: list[str] cashtags: list[str] mentionedUsers: list[UserRef] @@ -173,6 +180,7 @@ class Tweet(JSONTrait): place: Optional[Place] = None coordinates: Optional[Coordinates] = None inReplyToTweetId: int | None = None + inReplyToTweetIdStr: str | None = None inReplyToUser: UserRef | None = None source: str | None = None sourceUrl: str | None = None @@ -217,6 +225,7 @@ class Tweet(JSONTrait): likeCount=obj["favorite_count"], quoteCount=obj["quote_count"], conversationId=int(obj["conversation_id_str"]), + conversationIdStr=obj["conversation_id_str"], hashtags=[x["text"] for x in get_or(obj, "entities.hashtags", [])], cashtags=[x["text"] for x in get_or(obj, "entities.symbols", [])], mentionedUsers=[UserRef.parse(x) for x in get_or(obj, "entities.user_mentions", [])], @@ -229,6 +238,7 @@ class Tweet(JSONTrait): place=Place.parse(obj["place"]) if obj.get("place") else None, coordinates=Coordinates.parse(obj), inReplyToTweetId=int_or(obj, "in_reply_to_status_id_str"), + inReplyToTweetIdStr=get_or(obj, "in_reply_to_status_id_str"), inReplyToUser=_get_reply_user(obj, res), source=obj.get("source", None), sourceUrl=_get_source_url(obj),