Этот коммит содержится в:
Vlad Pronsky 2024-02-10 21:51:24 +02:00
родитель d579c5fe9e
Коммит 6114ebed39
20 изменённых файлов: 62393 добавлений и 57397 удалений

3
.github/workflows/stale-issues.yml поставляемый
Просмотреть файл

@ -1,4 +1,5 @@
name: 'Close stale issues and PRs'
name: close stale issues
on:
workflow_dispatch:
schedule:

Просмотреть файл

@ -62,10 +62,11 @@ update-mocks:
twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_followers.json
twscrape following --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_following.json
twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_details.json
twscrape tweet_replies --limit 1 --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_replies.json
twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_retweeters.json
twscrape favoriters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_favoriters.json
twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets.json
twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets_and_replies.json
twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/raw_search.json
twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/raw_list_timeline.json
twscrape likes --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_likes.json
twscrape liked_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_likes.json

Просмотреть файл

@ -78,6 +78,9 @@ async def main():
await gather(api.retweeters(tweet_id, limit=20)) # list[User]
await gather(api.favoriters(tweet_id, limit=20)) # list[User]
# Note: this method have small pagination from X side, like 5 tweets per query
await gather(api.tweet_replies(tweet_id, limit=20)) # list[Tweet]
# get user by login
user_login = "xdevelopers"
await api.user_by_login(user_login) # User
@ -234,6 +237,7 @@ twscrape --db test-accounts.db <command>
```sh
twscrape search "QUERY" --limit=20
twscrape tweet_details TWEET_ID
twscrape tweet_replies TWEET_ID --limit=20
twscrape retweeters TWEET_ID --limit=20
twscrape favoriters TWEET_ID --limit=20
twscrape user_by_id USER_ID

Просмотреть файл

@ -108,8 +108,8 @@
}
},
"fast_followers_count": 0,
"favourites_count": 6002,
"followers_count": 897,
"favourites_count": 6001,
"followers_count": 898,
"friends_count": 2634,
"has_custom_timelines": true,
"is_translator": false,
@ -117,7 +117,7 @@
"location": "India",
"media_count": 380,
"name": "Tarun Gupta #ngIndia",
"normal_followers_count": 897,
"normal_followers_count": 898,
"pinned_tweet_ids_str": [
"1754277413373292641"
],
@ -126,7 +126,7 @@
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1736141291011158016/d5syLOws_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "TarunDevops",
"statuses_count": 2322,
"statuses_count": 2320,
"translator_type": "none",
"url": "https://t.co/H5vY4SfIeh",
"verified": false,
@ -421,7 +421,7 @@
"fast_followers_count": 0,
"favourites_count": 8,
"followers_count": 1,
"friends_count": 50,
"friends_count": 51,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 0,
@ -479,7 +479,7 @@
},
"fast_followers_count": 0,
"favourites_count": 6392,
"followers_count": 737,
"followers_count": 736,
"friends_count": 4980,
"has_custom_timelines": true,
"is_translator": false,
@ -487,7 +487,7 @@
"location": "Other Universe.",
"media_count": 74,
"name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54",
"normal_followers_count": 737,
"normal_followers_count": 736,
"pinned_tweet_ids_str": [
"1746638470255157399"
],
@ -885,23 +885,23 @@
}
},
"fast_followers_count": 0,
"favourites_count": 98727,
"followers_count": 17859,
"friends_count": 19631,
"favourites_count": 98732,
"followers_count": 17863,
"friends_count": 19636,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2,
"location": "Aracaju Sergipe Brasil",
"media_count": 365,
"name": "Pedro Vasconcelos🎶",
"normal_followers_count": 17859,
"normal_followers_count": 17863,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/81217433/1672402393",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1684590879594491905/0DyAIFwY_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "psvas5",
"statuses_count": 40314,
"statuses_count": 40317,
"translator_type": "none",
"url": "https://t.co/JDYYzUaorM",
"verified": false,
@ -1017,15 +1017,15 @@
},
"fast_followers_count": 0,
"favourites_count": 270,
"followers_count": 130,
"friends_count": 573,
"followers_count": 131,
"friends_count": 572,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 0,
"location": "New Jersey.USA",
"media_count": 1,
"name": "Elena",
"normal_followers_count": 130,
"normal_followers_count": 131,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706",
@ -1076,7 +1076,7 @@
},
"fast_followers_count": 0,
"favourites_count": 476,
"followers_count": 90,
"followers_count": 89,
"friends_count": 341,
"has_custom_timelines": false,
"is_translator": false,
@ -1084,7 +1084,7 @@
"location": "๊USA",
"media_count": 1,
"name": "rajibelena",
"normal_followers_count": 90,
"normal_followers_count": 89,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988",
@ -1207,7 +1207,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 1198,
"favourites_count": 1197,
"followers_count": 19,
"friends_count": 116,
"has_custom_timelines": false,

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -99,7 +99,7 @@
},
"fast_followers_count": 0,
"favourites_count": 15,
"followers_count": 22,
"followers_count": 21,
"friends_count": 32,
"has_custom_timelines": false,
"is_translator": false,
@ -107,7 +107,7 @@
"location": "",
"media_count": 0,
"name": "Christopher Cameron",
"normal_followers_count": 22,
"normal_followers_count": 21,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706929807328706560/1705679588",
@ -158,7 +158,7 @@
},
"fast_followers_count": 0,
"favourites_count": 22,
"followers_count": 5,
"followers_count": 4,
"friends_count": 49,
"has_custom_timelines": false,
"is_translator": false,
@ -166,7 +166,7 @@
"location": "",
"media_count": 0,
"name": "Elijah Montgomery",
"normal_followers_count": 5,
"normal_followers_count": 4,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706876370268422144/1705678757",
@ -217,7 +217,7 @@
},
"fast_followers_count": 0,
"favourites_count": 15,
"followers_count": 8,
"followers_count": 7,
"friends_count": 29,
"has_custom_timelines": false,
"is_translator": false,
@ -225,7 +225,7 @@
"location": "",
"media_count": 0,
"name": "Joseph Griffin",
"normal_followers_count": 8,
"normal_followers_count": 7,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706571709682503680/1705673806",
@ -452,7 +452,7 @@
},
"fast_followers_count": 0,
"favourites_count": 6392,
"followers_count": 737,
"followers_count": 736,
"friends_count": 4980,
"has_custom_timelines": true,
"is_translator": false,
@ -460,7 +460,7 @@
"location": "Other Universe.",
"media_count": 74,
"name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54",
"normal_followers_count": 737,
"normal_followers_count": 736,
"pinned_tweet_ids_str": [
"1746638470255157399"
],
@ -513,15 +513,15 @@
},
"fast_followers_count": 0,
"favourites_count": 270,
"followers_count": 130,
"friends_count": 573,
"followers_count": 131,
"friends_count": 572,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 0,
"location": "New Jersey.USA",
"media_count": 1,
"name": "Elena",
"normal_followers_count": 130,
"normal_followers_count": 131,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706",
@ -572,7 +572,7 @@
},
"fast_followers_count": 0,
"favourites_count": 476,
"followers_count": 90,
"followers_count": 89,
"friends_count": 341,
"has_custom_timelines": false,
"is_translator": false,
@ -580,7 +580,7 @@
"location": "๊USA",
"media_count": 1,
"name": "rajibelena",
"normal_followers_count": 90,
"normal_followers_count": 89,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988",
@ -631,15 +631,15 @@
},
"fast_followers_count": 0,
"favourites_count": 258,
"followers_count": 151,
"friends_count": 1627,
"followers_count": 152,
"friends_count": 1659,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 0,
"location": "",
"media_count": 18,
"name": "deepak kumar",
"normal_followers_count": 151,
"normal_followers_count": 152,
"pinned_tweet_ids_str": [
"1712957054028005406"
],
@ -952,7 +952,7 @@
"fast_followers_count": 0,
"favourites_count": 3448,
"followers_count": 232,
"friends_count": 1221,
"friends_count": 1222,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 8,
@ -1109,23 +1109,23 @@
}
},
"fast_followers_count": 0,
"favourites_count": 6792,
"followers_count": 523,
"friends_count": 4998,
"favourites_count": 6825,
"followers_count": 526,
"friends_count": 5001,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 4,
"location": "California",
"media_count": 812,
"media_count": 817,
"name": "Yogi Vedd",
"normal_followers_count": 523,
"normal_followers_count": 526,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/961612928532275200/1518108233",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/961618203209228288/c_H53G3j_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "Yogi39Yogi",
"statuses_count": 8704,
"statuses_count": 8733,
"translator_type": "none",
"url": "https://t.co/FkxeNl3zwQ",
"verified": false,

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

741
tests/mocked-data/raw_tweet_replies.json Обычный файл
Просмотреть файл

@ -0,0 +1,741 @@
{
"data": {
"threaded_conversation_with_injections_v2": {
"instructions": [
{
"type": "TimelineAddEntries",
"entries": [
{
"entryId": "tweet-1649191520250245121",
"sortIndex": "7574180516604530686",
"content": {
"entryType": "TimelineTimelineItem",
"__typename": "TimelineTimelineItem",
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191520250245121",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"edit_control": {
"edit_tweet_ids": [
"1649191520250245121"
],
"editable_until_msecs": "1682034673000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "677243",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 69,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:13 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
98
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [],
"user_mentions": []
},
"favorite_count": 434,
"favorited": false,
"full_text": "Today, we are deprecating our Premium v1.1 API, including Premium Search and Account Activity API.",
"is_quote_status": false,
"lang": "en",
"quote_count": 114,
"reply_count": 101,
"retweet_count": 221,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191520250245121"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread",
"hasModeratedReplies": false
}
}
},
{
"entryId": "conversationthread-1649191521323995138",
"sortIndex": "7574180516604530676",
"content": {
"entryType": "TimelineTimelineModule",
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191521323995138",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191521323995138",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"edit_control": {
"edit_tweet_ids": [
"1649191521323995138"
],
"editable_until_msecs": "1682034673000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "189234",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 13,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:13 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
190
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [],
"user_mentions": []
},
"favorite_count": 82,
"favorited": false,
"full_text": "On 3/29 we announced that deprecations across our legacy Twitter API access tiers would be completed by 4/29. Unwinding Premium is one step closer to full deprecation of those legacy tiers.",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191520250245121",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"quote_count": 37,
"reply_count": 8,
"retweet_count": 51,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191521323995138"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABCCADg4UKAAIAAAAAGACgCAAAAAA="
}
}
}
}
},
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191522485817345",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191522485817345",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"unified_card": {
"card_fetch_state": "NoCard"
},
"edit_control": {
"edit_tweet_ids": [
"1649191522485817345"
],
"editable_until_msecs": "1682034674000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "134439",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 7,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:14 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
256
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [
{
"display_url": "shorturl.at/uwAKZ",
"expanded_url": "https://shorturl.at/uwAKZ",
"url": "https://t.co/A2FQP2DWuy",
"indices": [
233,
256
]
}
],
"user_mentions": []
},
"favorite_count": 135,
"favorited": false,
"full_text": "If you are a Twitter developer with a commercial use case, we recommend that you apply for our Enterprise tier.\nEnterprise Search and Account Activity API continue to be available to developers on that Enterprise tier. \n\nApply here: https://t.co/A2FQP2DWuy",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191521323995138",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 14,
"reply_count": 29,
"retweet_count": 41,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191522485817345"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABECADg4EKAAIAAAAAGACgCAAAAAA="
}
}
}
}
},
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191523697979392",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191523697979392",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"unified_card": {
"card_fetch_state": "NoCard"
},
"edit_control": {
"edit_tweet_ids": [
"1649191523697979392"
],
"editable_until_msecs": "1682034674000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "104514",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 8,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:14 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
193
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/iGTdPXTp2D",
"indices": [
170,
193
]
}
],
"user_mentions": []
},
"favorite_count": 130,
"favorited": false,
"full_text": "Thank you to those who have already migrated to our new API offerings. We hope you can continue to seamlessly use the Twitter API via our new access tiers. \n\nLearn more: https://t.co/iGTdPXTp2D",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191522485817345",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 3,
"reply_count": 37,
"retweet_count": 41,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191523697979392"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABICADg4EKAAIAAAAAGACgCAAAAAA="
}
}
}
}
}
],
"displayType": "VerticalConversation",
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
}
}
}
}
},
{
"entryId": "cursor-showmorethreads-2944676126953923946",
"sortIndex": "7574180516604530675",
"content": {
"entryType": "TimelineTimelineItem",
"__typename": "TimelineTimelineItem",
"itemContent": {
"itemType": "TimelineTimelineCursor",
"__typename": "TimelineTimelineCursor",
"value": "EwAAAPAEHBkWhMDSse28jeMtJQISFQ4AAA",
"cursorType": "ShowMoreThreads",
"displayTreatment": {
"actionText": "Show more replies"
}
}
}
}
]
},
{
"type": "TimelineTerminateTimeline",
"direction": "Top"
}
],
"responseObjects": {
"immediateReactions": []
},
"metadata": {
"reader_mode_config": {
"is_reader_mode_available": true
}
}
}
}
}

Просмотреть файл

@ -49,7 +49,7 @@
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607490,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
@ -57,7 +57,7 @@
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607490,
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],

Просмотреть файл

@ -49,7 +49,7 @@
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607490,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
@ -57,7 +57,7 @@
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607490,
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -10,12 +10,16 @@ class MockedError(Exception):
GQL_GEN = [
"followers",
"following",
"search",
"tweet_replies",
"retweeters",
"favoriters",
"followers",
"following",
"user_tweets",
"user_tweets_and_replies",
"list_timeline",
"liked_tweets",
]

Просмотреть файл

@ -213,6 +213,19 @@ async def test_tweet_details():
assert doc.user is not None, "tweet.user should not be None"
async def test_tweet_replies():
api = API()
mock_rep(api.tweet_replies_raw, "raw_tweet_replies", as_generator=True)
twid = 1649191520250245121
tweets = await gather(api.tweet_replies(twid, limit=20))
assert len(tweets) > 0
for doc in tweets:
check_tweet(doc)
assert doc.inReplyToTweetId == twid
async def test_followers():
api = API()
mock_rep(api.followers_raw, "raw_followers", as_generator=True)

Просмотреть файл

@ -74,14 +74,16 @@ class API:
return rep if is_res else None, new_total, is_cur and not is_lim
def _get_cursor(self, obj: dict):
if cur := find_obj(obj, lambda x: x.get("cursorType") == "Bottom"):
def _get_cursor(self, obj: dict, cursor_type="Bottom"):
if cur := find_obj(obj, lambda x: x.get("cursorType") == cursor_type):
return cur.get("value")
return None
# gql helpers
async def _gql_items(self, op: str, kv: dict, ft: dict | None = None, limit=-1):
async def _gql_items(
self, op: str, kv: dict, ft: dict | None = None, limit=-1, cursor_type="Bottom"
):
queue, cur, cnt, active = op.split("/")[-1], None, 0, True
kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})}
@ -100,7 +102,7 @@ class API:
obj = rep.json()
els = get_by_path(obj, "entries") or []
els = [x for x in els if not x["entryId"].startswith("cursor-")]
cur = self._get_cursor(obj)
cur = self._get_cursor(obj, cursor_type)
rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit)
if rep is None:
@ -178,19 +180,13 @@ class API:
op = OP_TweetDetail
kv = {
"focalTweetId": str(twid),
"referrer": "tweet", # tweet, profile
"with_rux_injections": False,
"with_rux_injections": True,
"includePromotedContent": True,
"withCommunity": True,
"withQuickPromoteEligibilityTweetFields": True,
"withBirdwatchNotes": True,
"withVoice": True,
"withV2Timeline": True,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": False,
"withSuperFollowsUserFields": False,
**(kv or {}),
}
return await self._gql_item(op, kv)
@ -199,6 +195,32 @@ class API:
rep = await self.tweet_details_raw(twid, kv=kv)
return parse_tweet(rep, twid) if rep else None
# tweet_replies
# note: uses same op as tweet_details, see: https://github.com/vladkens/twscrape/issues/104
async def tweet_replies_raw(self, twid: int, limit=-1, kv=None):
op = OP_TweetDetail
kv = {
"focalTweetId": str(twid),
"referrer": "tweet",
"with_rux_injections": True,
"includePromotedContent": True,
"withCommunity": True,
"withQuickPromoteEligibilityTweetFields": True,
"withBirdwatchNotes": True,
"withVoice": True,
"withV2Timeline": True,
**(kv or {}),
}
async for x in self._gql_items(op, kv, limit=limit, cursor_type="ShowMoreThreads"):
yield x
async def tweet_replies(self, twid: int, limit=-1, kv=None):
async for rep in self.tweet_replies_raw(twid, limit=limit, kv=kv):
for x in parse_tweets(rep.json(), limit):
if x.inReplyToTweetId == twid:
yield x
# followers
async def followers_raw(self, uid: int, limit=-1, kv=None):

Просмотреть файл

@ -183,6 +183,7 @@ def run():
c_lim("search", "Search for tweets", "query", "Search query")
c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int)
c_lim("tweet_replies", "Get replies of a tweet", "tweet_id", "Tweet ID", int)
c_lim("retweeters", "Get retweeters of a tweet", "tweet_id", "Tweet ID", int)
c_lim("favoriters", "Get favoriters of a tweet", "tweet_id", "Tweet ID", int)
c_one("user_by_id", "Get user data by ID", "user_id", "User ID", int)