locsec/twscrape
1
Форкнуть 0
зеркало из https://github.com/viginum-datalab/twscrape.git synced 2025-10-29 13:06:13 +02:00
Код Задачи Пакеты Проекты Релизы Вики Активность

add tweet_replies method #104

Просмотр исходного кода
Этот коммит содержится в:
Vlad Pronsky 2024-02-10 21:51:24 +02:00
родитель d579c5fe9e
Коммит 6114ebed39
20 изменённых файлов: 62393 добавлений и 57397 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
.github/workflows/stale-issues.yml поставляемый
Просмотреть файл

@ -1,4 +1,5 @@
name: 'Close stale issues and PRs' name: close stale issues
on: on:
workflow_dispatch: workflow_dispatch:
schedule: schedule:

3
Makefile
Просмотреть файл

@ -62,10 +62,11 @@ update-mocks:
twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_followers.json twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_followers.json
twscrape following --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_following.json twscrape following --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_following.json
twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_details.json twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_details.json
twscrape tweet_replies --limit 1 --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_replies.json
twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_retweeters.json twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_retweeters.json
twscrape favoriters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_favoriters.json twscrape favoriters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_favoriters.json
twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets.json twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets.json
twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets_and_replies.json twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets_and_replies.json
twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/raw_search.json twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/raw_search.json
twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/raw_list_timeline.json twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/raw_list_timeline.json
twscrape likes --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_likes.json twscrape liked_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_likes.json

4
readme.md
Просмотреть файл

@ -78,6 +78,9 @@ async def main():
await gather(api.retweeters(tweet_id, limit=20)) # list[User] await gather(api.retweeters(tweet_id, limit=20)) # list[User]
await gather(api.favoriters(tweet_id, limit=20)) # list[User] await gather(api.favoriters(tweet_id, limit=20)) # list[User]
# Note: this method have small pagination from X side, like 5 tweets per query
await gather(api.tweet_replies(tweet_id, limit=20)) # list[Tweet]
# get user by login # get user by login
user_login = "xdevelopers" user_login = "xdevelopers"
await api.user_by_login(user_login) # User await api.user_by_login(user_login) # User
@ -234,6 +237,7 @@ twscrape --db test-accounts.db <command>
```sh ```sh
twscrape search "QUERY" --limit=20 twscrape search "QUERY" --limit=20
twscrape tweet_details TWEET_ID twscrape tweet_details TWEET_ID
twscrape tweet_replies TWEET_ID --limit=20
twscrape retweeters TWEET_ID --limit=20 twscrape retweeters TWEET_ID --limit=20
twscrape favoriters TWEET_ID --limit=20 twscrape favoriters TWEET_ID --limit=20
twscrape user_by_id USER_ID twscrape user_by_id USER_ID

36
tests/mocked-data/raw_favoriters.json
Просмотреть файл

@ -108,8 +108,8 @@
} }
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 6002, "favourites_count": 6001,
"followers_count": 897, "followers_count": 898,
"friends_count": 2634, "friends_count": 2634,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
@ -117,7 +117,7 @@
"location": "India", "location": "India",
"media_count": 380, "media_count": 380,
"name": "Tarun Gupta #ngIndia", "name": "Tarun Gupta #ngIndia",
"normal_followers_count": 897, "normal_followers_count": 898,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1754277413373292641" "1754277413373292641"
], ],
@ -126,7 +126,7 @@
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1736141291011158016/d5syLOws_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1736141291011158016/d5syLOws_normal.jpg",
"profile_interstitial_type": "", "profile_interstitial_type": "",
"screen_name": "TarunDevops", "screen_name": "TarunDevops",
"statuses_count": 2322, "statuses_count": 2320,
"translator_type": "none", "translator_type": "none",
"url": "https://t.co/H5vY4SfIeh", "url": "https://t.co/H5vY4SfIeh",
"verified": false, "verified": false,
@ -421,7 +421,7 @@
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 8, "favourites_count": 8,
"followers_count": 1, "followers_count": 1,
"friends_count": 50, "friends_count": 51,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
"listed_count": 0, "listed_count": 0,
@ -479,7 +479,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 6392, "favourites_count": 6392,
"followers_count": 737, "followers_count": 736,
"friends_count": 4980, "friends_count": 4980,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
@ -487,7 +487,7 @@
"location": "Other Universe.", "location": "Other Universe.",
"media_count": 74, "media_count": 74,
"name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54", "name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54",
"normal_followers_count": 737, "normal_followers_count": 736,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1746638470255157399" "1746638470255157399"
], ],
@ -885,23 +885,23 @@
} }
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 98727, "favourites_count": 98732,
"followers_count": 17859, "followers_count": 17863,
"friends_count": 19631, "friends_count": 19636,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
"listed_count": 2, "listed_count": 2,
"location": "Aracaju Sergipe Brasil", "location": "Aracaju Sergipe Brasil",
"media_count": 365, "media_count": 365,
"name": "Pedro Vasconcelos🎶", "name": "Pedro Vasconcelos🎶",
"normal_followers_count": 17859, "normal_followers_count": 17863,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/81217433/1672402393", "profile_banner_url": "https://pbs.twimg.com/profile_banners/81217433/1672402393",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1684590879594491905/0DyAIFwY_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/1684590879594491905/0DyAIFwY_normal.jpg",
"profile_interstitial_type": "", "profile_interstitial_type": "",
"screen_name": "psvas5", "screen_name": "psvas5",
"statuses_count": 40314, "statuses_count": 40317,
"translator_type": "none", "translator_type": "none",
"url": "https://t.co/JDYYzUaorM", "url": "https://t.co/JDYYzUaorM",
"verified": false, "verified": false,
@ -1017,15 +1017,15 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 270, "favourites_count": 270,
"followers_count": 130, "followers_count": 131,
"friends_count": 573, "friends_count": 572,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
"listed_count": 0, "listed_count": 0,
"location": "New Jersey.USA", "location": "New Jersey.USA",
"media_count": 1, "media_count": 1,
"name": "Elena", "name": "Elena",
"normal_followers_count": 130, "normal_followers_count": 131,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706", "profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706",
@ -1076,7 +1076,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 476, "favourites_count": 476,
"followers_count": 90, "followers_count": 89,
"friends_count": 341, "friends_count": 341,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
@ -1084,7 +1084,7 @@
"location": "๊USA", "location": "๊USA",
"media_count": 1, "media_count": 1,
"name": "rajibelena", "name": "rajibelena",
"normal_followers_count": 90, "normal_followers_count": 89,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988", "profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988",
@ -1207,7 +1207,7 @@
} }
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 1198, "favourites_count": 1197,
"followers_count": 19, "followers_count": 19,
"friends_count": 116, "friends_count": 116,
"has_custom_timelines": false, "has_custom_timelines": false,

6438
tests/mocked-data/raw_followers.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

326
tests/mocked-data/raw_following.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

804
tests/mocked-data/raw_likes.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

110168
tests/mocked-data/raw_list_timeline.json
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

46
tests/mocked-data/raw_retweeters.json
Просмотреть файл

@ -99,7 +99,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 15, "favourites_count": 15,
"followers_count": 22, "followers_count": 21,
"friends_count": 32, "friends_count": 32,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
@ -107,7 +107,7 @@
"location": "", "location": "",
"media_count": 0, "media_count": 0,
"name": "Christopher Cameron", "name": "Christopher Cameron",
"normal_followers_count": 22, "normal_followers_count": 21,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706929807328706560/1705679588", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1706929807328706560/1705679588",
@ -158,7 +158,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 22, "favourites_count": 22,
"followers_count": 5, "followers_count": 4,
"friends_count": 49, "friends_count": 49,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
@ -166,7 +166,7 @@
"location": "", "location": "",
"media_count": 0, "media_count": 0,
"name": "Elijah Montgomery", "name": "Elijah Montgomery",
"normal_followers_count": 5, "normal_followers_count": 4,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706876370268422144/1705678757", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1706876370268422144/1705678757",
@ -217,7 +217,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 15, "favourites_count": 15,
"followers_count": 8, "followers_count": 7,
"friends_count": 29, "friends_count": 29,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
@ -225,7 +225,7 @@
"location": "", "location": "",
"media_count": 0, "media_count": 0,
"name": "Joseph Griffin", "name": "Joseph Griffin",
"normal_followers_count": 8, "normal_followers_count": 7,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1706571709682503680/1705673806", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1706571709682503680/1705673806",
@ -452,7 +452,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 6392, "favourites_count": 6392,
"followers_count": 737, "followers_count": 736,
"friends_count": 4980, "friends_count": 4980,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
@ -460,7 +460,7 @@
"location": "Other Universe.", "location": "Other Universe.",
"media_count": 74, "media_count": 74,
"name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54", "name": "98𝕰𝖑𝖎𝖙𝖊𝕸𝕲𝖍𝖔𝖘𝖙~59~𝕺𝖛𝖊𝖗𝕷𝖔𝖗𝖉54",
"normal_followers_count": 737, "normal_followers_count": 736,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1746638470255157399" "1746638470255157399"
], ],
@ -513,15 +513,15 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 270, "favourites_count": 270,
"followers_count": 130, "followers_count": 131,
"friends_count": 573, "friends_count": 572,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
"listed_count": 0, "listed_count": 0,
"location": "New Jersey.USA", "location": "New Jersey.USA",
"media_count": 1, "media_count": 1,
"name": "Elena", "name": "Elena",
"normal_followers_count": 130, "normal_followers_count": 131,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706", "profile_banner_url": "https://pbs.twimg.com/profile_banners/2534108054/1704819706",
@ -572,7 +572,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 476, "favourites_count": 476,
"followers_count": 90, "followers_count": 89,
"friends_count": 341, "friends_count": 341,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
@ -580,7 +580,7 @@
"location": "๊USA", "location": "๊USA",
"media_count": 1, "media_count": 1,
"name": "rajibelena", "name": "rajibelena",
"normal_followers_count": 90, "normal_followers_count": 89,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988", "profile_banner_url": "https://pbs.twimg.com/profile_banners/2843756880/1705349988",
@ -631,15 +631,15 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 258, "favourites_count": 258,
"followers_count": 151, "followers_count": 152,
"friends_count": 1627, "friends_count": 1659,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
"listed_count": 0, "listed_count": 0,
"location": "", "location": "",
"media_count": 18, "media_count": 18,
"name": "deepak kumar", "name": "deepak kumar",
"normal_followers_count": 151, "normal_followers_count": 152,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1712957054028005406" "1712957054028005406"
], ],
@ -952,7 +952,7 @@
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 3448, "favourites_count": 3448,
"followers_count": 232, "followers_count": 232,
"friends_count": 1221, "friends_count": 1222,
"has_custom_timelines": false, "has_custom_timelines": false,
"is_translator": false, "is_translator": false,
"listed_count": 8, "listed_count": 8,
@ -1109,23 +1109,23 @@
} }
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 6792, "favourites_count": 6825,
"followers_count": 523, "followers_count": 526,
"friends_count": 4998, "friends_count": 5001,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
"listed_count": 4, "listed_count": 4,
"location": "California", "location": "California",
"media_count": 812, "media_count": 817,
"name": "Yogi Vedd", "name": "Yogi Vedd",
"normal_followers_count": 523, "normal_followers_count": 526,
"pinned_tweet_ids_str": [], "pinned_tweet_ids_str": [],
"possibly_sensitive": false, "possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/961612928532275200/1518108233", "profile_banner_url": "https://pbs.twimg.com/profile_banners/961612928532275200/1518108233",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/961618203209228288/c_H53G3j_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/961618203209228288/c_H53G3j_normal.jpg",
"profile_interstitial_type": "", "profile_interstitial_type": "",
"screen_name": "Yogi39Yogi", "screen_name": "Yogi39Yogi",
"statuses_count": 8704, "statuses_count": 8733,
"translator_type": "none", "translator_type": "none",
"url": "https://t.co/FkxeNl3zwQ", "url": "https://t.co/FkxeNl3zwQ",
"verified": false, "verified": false,

3809
tests/mocked-data/raw_search.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1296
tests/mocked-data/raw_tweet_details.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

741
tests/mocked-data/raw_tweet_replies.json Обычный файл
Просмотреть файл

@ -0,0 +1,741 @@
{
"data": {
"threaded_conversation_with_injections_v2": {
"instructions": [
{
"type": "TimelineAddEntries",
"entries": [
{
"entryId": "tweet-1649191520250245121",
"sortIndex": "7574180516604530686",
"content": {
"entryType": "TimelineTimelineItem",
"__typename": "TimelineTimelineItem",
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191520250245121",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"edit_control": {
"edit_tweet_ids": [
"1649191520250245121"
],
"editable_until_msecs": "1682034673000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "677243",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 69,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:13 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
98
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [],
"user_mentions": []
},
"favorite_count": 434,
"favorited": false,
"full_text": "Today, we are deprecating our Premium v1.1 API, including Premium Search and Account Activity API.",
"is_quote_status": false,
"lang": "en",
"quote_count": 114,
"reply_count": 101,
"retweet_count": 221,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191520250245121"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread",
"hasModeratedReplies": false
}
}
},
{
"entryId": "conversationthread-1649191521323995138",
"sortIndex": "7574180516604530676",
"content": {
"entryType": "TimelineTimelineModule",
"__typename": "TimelineTimelineModule",
"items": [
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191521323995138",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191521323995138",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"edit_control": {
"edit_tweet_ids": [
"1649191521323995138"
],
"editable_until_msecs": "1682034673000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "189234",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 13,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:13 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
190
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [],
"user_mentions": []
},
"favorite_count": 82,
"favorited": false,
"full_text": "On 3/29 we announced that deprecations across our legacy Twitter API access tiers would be completed by 4/29. Unwinding Premium is one step closer to full deprecation of those legacy tiers.",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191520250245121",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"quote_count": 37,
"reply_count": 8,
"retweet_count": 51,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191521323995138"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABCCADg4UKAAIAAAAAGACgCAAAAAA="
}
}
}
}
},
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191522485817345",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191522485817345",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"unified_card": {
"card_fetch_state": "NoCard"
},
"edit_control": {
"edit_tweet_ids": [
"1649191522485817345"
],
"editable_until_msecs": "1682034674000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "134439",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 7,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:14 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
256
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [
{
"display_url": "shorturl.at/uwAKZ",
"expanded_url": "https://shorturl.at/uwAKZ",
"url": "https://t.co/A2FQP2DWuy",
"indices": [
233,
256
]
}
],
"user_mentions": []
},
"favorite_count": 135,
"favorited": false,
"full_text": "If you are a Twitter developer with a commercial use case, we recommend that you apply for our Enterprise tier.\nEnterprise Search and Account Activity API continue to be available to developers on that Enterprise tier. \n\nApply here: https://t.co/A2FQP2DWuy",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191521323995138",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 14,
"reply_count": 29,
"retweet_count": 41,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191522485817345"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABECADg4EKAAIAAAAAGACgCAAAAAA="
}
}
}
}
},
{
"entryId": "conversationthread-1649191521323995138-tweet-1649191523697979392",
"item": {
"itemContent": {
"itemType": "TimelineTweet",
"__typename": "TimelineTweet",
"tweet_results": {
"result": {
"__typename": "Tweet",
"rest_id": "1649191523697979392",
"has_birdwatch_notes": false,
"core": {
"user_results": {
"result": {
"__typename": "User",
"id": "VXNlcjoyMjQ0OTk0OTQ1",
"rest_id": "2244994945",
"affiliates_highlighted_label": {
"label": {
"url": {
"url": "https://twitter.com/X",
"urlType": "DeepLink"
},
"badge": {
"url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
},
"description": "X",
"userLabelType": "BusinessLabel",
"userLabelDisplayType": "Badge"
}
},
"has_graduated_access": true,
"is_blue_verified": true,
"profile_image_shape": "Square",
"legacy": {
"can_dm": false,
"can_media_tag": true,
"created_at": "Sat Dec 14 04:35:55 +0000 2013",
"default_profile": false,
"default_profile_image": false,
"description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
"entities": {
"description": {
"urls": []
},
"url": {
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/RUXWsqdGk8",
"indices": [
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 2073,
"followers_count": 607579,
"friends_count": 1773,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 2572,
"location": "127.0.0.1",
"media_count": 815,
"name": "Developers",
"normal_followers_count": 607579,
"pinned_tweet_ids_str": [
"1661790253886177280"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "XDevelopers",
"statuses_count": 4041,
"translator_type": "regular",
"url": "https://t.co/RUXWsqdGk8",
"verified": false,
"verified_type": "Business",
"want_retweets": false,
"withheld_in_countries": []
},
"professional": {
"rest_id": "1516891231749517312",
"professional_type": "Business",
"category": [
{
"id": 1009,
"name": "Community",
"icon_name": "IconBriefcaseStroke"
}
]
}
}
}
},
"unmention_data": {},
"unified_card": {
"card_fetch_state": "NoCard"
},
"edit_control": {
"edit_tweet_ids": [
"1649191523697979392"
],
"editable_until_msecs": "1682034674000",
"is_edit_eligible": false,
"edits_remaining": "5"
},
"is_translatable": false,
"views": {
"count": "104514",
"state": "EnabledWithCount"
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"legacy": {
"bookmark_count": 8,
"bookmarked": false,
"created_at": "Thu Apr 20 23:21:14 +0000 2023",
"conversation_id_str": "1649191520250245121",
"display_text_range": [
0,
193
],
"entities": {
"hashtags": [],
"symbols": [],
"timestamps": [],
"urls": [
{
"display_url": "developer.twitter.com",
"expanded_url": "https://developer.twitter.com/",
"url": "https://t.co/iGTdPXTp2D",
"indices": [
170,
193
]
}
],
"user_mentions": []
},
"favorite_count": 130,
"favorited": false,
"full_text": "Thank you to those who have already migrated to our new API offerings. We hope you can continue to seamlessly use the Twitter API via our new access tiers. \n\nLearn more: https://t.co/iGTdPXTp2D",
"in_reply_to_screen_name": "XDevelopers",
"in_reply_to_status_id_str": "1649191522485817345",
"in_reply_to_user_id_str": "2244994945",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 3,
"reply_count": 37,
"retweet_count": 41,
"retweeted": false,
"user_id_str": "2244994945",
"id_str": "1649191523697979392"
},
"quick_promote_eligibility": {
"eligibility": "IneligibleNotProfessional"
}
}
},
"tweetDisplayType": "SelfThread"
},
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
},
"timelinesDetails": {
"controllerData": "DAACDAAEDAABCgABFSABICADg4EKAAIAAAAAGACgCAAAAAA="
}
}
}
}
}
],
"displayType": "VerticalConversation",
"clientEventInfo": {
"details": {
"conversationDetails": {
"conversationSection": "HighQuality"
}
}
}
}
},
{
"entryId": "cursor-showmorethreads-2944676126953923946",
"sortIndex": "7574180516604530675",
"content": {
"entryType": "TimelineTimelineItem",
"__typename": "TimelineTimelineItem",
"itemContent": {
"itemType": "TimelineTimelineCursor",
"__typename": "TimelineTimelineCursor",
"value": "EwAAAPAEHBkWhMDSse28jeMtJQISFQ4AAA",
"cursorType": "ShowMoreThreads",
"displayTreatment": {
"actionText": "Show more replies"
}
}
}
}
]
},
{
"type": "TimelineTerminateTimeline",
"direction": "Top"
}
],
"responseObjects": {
"immediateReactions": []
},
"metadata": {
"reader_mode_config": {
"is_reader_mode_available": true
}
}
}
}
}

4
tests/mocked-data/raw_user_by_id.json
Просмотреть файл

@ -49,7 +49,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 2073, "favourites_count": 2073,
"followers_count": 607490, "followers_count": 607579,
"friends_count": 1773, "friends_count": 1773,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
@ -57,7 +57,7 @@
"location": "127.0.0.1", "location": "127.0.0.1",
"media_count": 815, "media_count": 815,
"name": "Developers", "name": "Developers",
"normal_followers_count": 607490, "normal_followers_count": 607579,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1661790253886177280" "1661790253886177280"
], ],

4
tests/mocked-data/raw_user_by_login.json
Просмотреть файл

@ -49,7 +49,7 @@
}, },
"fast_followers_count": 0, "fast_followers_count": 0,
"favourites_count": 2073, "favourites_count": 2073,
"followers_count": 607490, "followers_count": 607579,
"friends_count": 1773, "friends_count": 1773,
"has_custom_timelines": true, "has_custom_timelines": true,
"is_translator": false, "is_translator": false,
@ -57,7 +57,7 @@
"location": "127.0.0.1", "location": "127.0.0.1",
"media_count": 815, "media_count": 815,
"name": "Developers", "name": "Developers",
"normal_followers_count": 607490, "normal_followers_count": 607579,
"pinned_tweet_ids_str": [ "pinned_tweet_ids_str": [
"1661790253886177280" "1661790253886177280"
], ],

441
tests/mocked-data/raw_user_tweets.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

539
tests/mocked-data/raw_user_tweets_and_replies.json
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

8
tests/test_api.py
Просмотреть файл

@ -10,12 +10,16 @@ class MockedError(Exception):
GQL_GEN = [ GQL_GEN = [
"followers", "search",
"following", "tweet_replies",
"retweeters", "retweeters",
"favoriters", "favoriters",
"followers",
"following",
"user_tweets", "user_tweets",
"user_tweets_and_replies", "user_tweets_and_replies",
"list_timeline",
"liked_tweets",
] ]

13
tests/test_parser.py
Просмотреть файл

@ -213,6 +213,19 @@ async def test_tweet_details():
assert doc.user is not None, "tweet.user should not be None" assert doc.user is not None, "tweet.user should not be None"
async def test_tweet_replies():
api = API()
mock_rep(api.tweet_replies_raw, "raw_tweet_replies", as_generator=True)
twid = 1649191520250245121
tweets = await gather(api.tweet_replies(twid, limit=20))
assert len(tweets) > 0
for doc in tweets:
check_tweet(doc)
assert doc.inReplyToTweetId == twid
async def test_followers(): async def test_followers():
api = API() api = API()
mock_rep(api.followers_raw, "raw_followers", as_generator=True) mock_rep(api.followers_raw, "raw_followers", as_generator=True)

44
twscrape/api.py
Просмотреть файл

@ -74,14 +74,16 @@ class API:
return rep if is_res else None, new_total, is_cur and not is_lim return rep if is_res else None, new_total, is_cur and not is_lim
def _get_cursor(self, obj: dict): def _get_cursor(self, obj: dict, cursor_type="Bottom"):
if cur := find_obj(obj, lambda x: x.get("cursorType") == "Bottom"): if cur := find_obj(obj, lambda x: x.get("cursorType") == cursor_type):
return cur.get("value") return cur.get("value")
return None return None
# gql helpers # gql helpers
async def _gql_items(self, op: str, kv: dict, ft: dict | None = None, limit=-1): async def _gql_items(
self, op: str, kv: dict, ft: dict | None = None, limit=-1, cursor_type="Bottom"
):
queue, cur, cnt, active = op.split("/")[-1], None, 0, True queue, cur, cnt, active = op.split("/")[-1], None, 0, True
kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})} kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})}
@ -100,7 +102,7 @@ class API:
obj = rep.json() obj = rep.json()
els = get_by_path(obj, "entries") or [] els = get_by_path(obj, "entries") or []
els = [x for x in els if not x["entryId"].startswith("cursor-")] els = [x for x in els if not x["entryId"].startswith("cursor-")]
cur = self._get_cursor(obj) cur = self._get_cursor(obj, cursor_type)
rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit) rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit)
if rep is None: if rep is None:
@ -178,19 +180,13 @@ class API:
op = OP_TweetDetail op = OP_TweetDetail
kv = { kv = {
"focalTweetId": str(twid), "focalTweetId": str(twid),
"referrer": "tweet", # tweet, profile "with_rux_injections": True,
"with_rux_injections": False,
"includePromotedContent": True, "includePromotedContent": True,
"withCommunity": True, "withCommunity": True,
"withQuickPromoteEligibilityTweetFields": True, "withQuickPromoteEligibilityTweetFields": True,
"withBirdwatchNotes": True, "withBirdwatchNotes": True,
"withVoice": True, "withVoice": True,
"withV2Timeline": True, "withV2Timeline": True,
"withDownvotePerspective": False,
"withReactionsMetadata": False,
"withReactionsPerspective": False,
"withSuperFollowsTweetFields": False,
"withSuperFollowsUserFields": False,
**(kv or {}), **(kv or {}),
} }
return await self._gql_item(op, kv) return await self._gql_item(op, kv)
@ -199,6 +195,32 @@ class API:
rep = await self.tweet_details_raw(twid, kv=kv) rep = await self.tweet_details_raw(twid, kv=kv)
return parse_tweet(rep, twid) if rep else None return parse_tweet(rep, twid) if rep else None
# tweet_replies
# note: uses same op as tweet_details, see: https://github.com/vladkens/twscrape/issues/104
async def tweet_replies_raw(self, twid: int, limit=-1, kv=None):
op = OP_TweetDetail
kv = {
"focalTweetId": str(twid),
"referrer": "tweet",
"with_rux_injections": True,
"includePromotedContent": True,
"withCommunity": True,
"withQuickPromoteEligibilityTweetFields": True,
"withBirdwatchNotes": True,
"withVoice": True,
"withV2Timeline": True,
**(kv or {}),
}
async for x in self._gql_items(op, kv, limit=limit, cursor_type="ShowMoreThreads"):
yield x
async def tweet_replies(self, twid: int, limit=-1, kv=None):
async for rep in self.tweet_replies_raw(twid, limit=limit, kv=kv):
for x in parse_tweets(rep.json(), limit):
if x.inReplyToTweetId == twid:
yield x
# followers # followers
async def followers_raw(self, uid: int, limit=-1, kv=None): async def followers_raw(self, uid: int, limit=-1, kv=None):

1
twscrape/cli.py
Просмотреть файл

@ -183,6 +183,7 @@ def run():
c_lim("search", "Search for tweets", "query", "Search query") c_lim("search", "Search for tweets", "query", "Search query")
c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int) c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int)
c_lim("tweet_replies", "Get replies of a tweet", "tweet_id", "Tweet ID", int)
c_lim("retweeters", "Get retweeters of a tweet", "tweet_id", "Tweet ID", int) c_lim("retweeters", "Get retweeters of a tweet", "tweet_id", "Tweet ID", int)
c_lim("favoriters", "Get favoriters of a tweet", "tweet_id", "Tweet ID", int) c_lim("favoriters", "Get favoriters of a tweet", "tweet_id", "Tweet ID", int)
c_one("user_by_id", "Get user data by ID", "user_id", "User ID", int) c_one("user_by_id", "Get user data by ID", "user_id", "User ID", int)