From 4b67be62bb5051ef8e76c0b92003d062444abc16 Mon Sep 17 00:00:00 2001 From: Vlad Pronsky Date: Fri, 5 Jan 2024 04:50:56 +0200 Subject: [PATCH] replace black/isort/pylint with ruff --- .vscode/settings.json | 16 ++++++------- Makefile | 21 ++++------------- pyproject.toml | 42 ++++++++++++---------------------- tests/test_utils.py | 10 +++++++- twscrape/account.py | 3 ++- twscrape/accounts_pool.py | 1 - twscrape/api.py | 41 ++++++++++++++++++++++++++++++--- twscrape/constants.py | 48 --------------------------------------- twscrape/login.py | 3 ++- 9 files changed, 77 insertions(+), 108 deletions(-) delete mode 100644 twscrape/constants.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 0948849..238a8df 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,14 +1,14 @@ { - "files.exclude": { - ".ruff_cache": true, - ".pytest_cache": true, - "*.egg-info": true, - "build": true, - ".coverage": true, - }, "[python]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": ["source.organizeImports"], - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "charliermarsh.ruff" + }, + "files.exclude": { + ".ruff_cache": true, + ".pytest_cache": true, + ".coverage": true, + "htmlcov": true, + "dist": true }, } diff --git a/Makefile b/Makefile index 2ae7ae8..57b98bf 100644 --- a/Makefile +++ b/Makefile @@ -7,24 +7,11 @@ install: build: @python -m build -ci: - @make format - @make lint - @make test - -format: - @black . - lint: - @ruff check twscrape - @ruff check tests - -lint-fix: - @ruff check --fix twscrape - @ruff check --fix tests - -pylint: - @pylint --errors-only twscrape + # https://docs.astral.sh/ruff/settings/#sorting-imports + @ruff check --select I --fix . + @ruff format . + @ruff check . test: @pytest -s --cov=twscrape tests/ diff --git a/pyproject.toml b/pyproject.toml index 00130cb..2c52ba1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,9 +2,6 @@ requires = ["hatchling>=1.9.1"] build-backend = "hatchling.build" -[tool.hatch.build.targets.wheel] -packages = ["twscrape"] - [project] name = "twscrape" version = "0.9.0" @@ -30,12 +27,11 @@ dependencies = [ [project.optional-dependencies] dev = [ - "pylint>=2.17.3", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "pytest-httpx>=0.22.0", - "pytest>=7.4.0", - "ruff" + "pytest-asyncio>=0.23.3", + "pytest-cov>=4.1.0", + "pytest-httpx>=0.28.0", + "pytest>=7.4.4", + "ruff>=0.1.11" ] [project.urls] @@ -47,30 +43,20 @@ twscrape = "twscrape.cli:run" [tool.setuptools] packages = ['twscrape'] -[tool.pylint] -max-line-length = 99 -disable = [ - "C0103", # invalid-name - "C0114", # missing-module-docstring - "C0115", # missing-class-docstring - "C0116", # missing-function-docstring - "R0903", # too-few-public-methods - "R0913", # too-many-arguments - "W0105", # pointless-string-statement -] +[tool.hatch.build.targets.wheel] +packages = ["twscrape"] + +[tool.hatch.metadata] +allow-direct-references = true [tool.pytest.ini_options] pythonpath = ["."] asyncio_mode = "auto" -[tool.isort] -profile = "black" - -[tool.black] -line-length = 99 - [tool.ruff] line-length = 99 -[tool.hatch.metadata] -allow-direct-references = true +[tool.ruff.lint] +ignore = ["E501"] + +[tool.ruff.format] \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 0d57340..cf2374b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ -# ruff: noqa: E501 +import pytest + from twscrape.utils import parse_cookies @@ -17,3 +18,10 @@ def test_cookies_parse(): val = "W3sibmFtZSI6ICJhYmMiLCAidmFsdWUiOiAiMTIzIn0sIHsibmFtZSI6ICJkZWYiLCAidmFsdWUiOiAiNDU2In0sIHsibmFtZSI6ICJnaGkiLCAidmFsdWUiOiAiNzg5In1d" assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"} + + val = '{"cookies": {"abc": "123", "def": "456", "ghi": "789"}}' + assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"} + + with pytest.raises(ValueError, match=r"Invalid cookie value: .+"): + val = "{invalid}" + assert parse_cookies(val) == {} diff --git a/twscrape/account.py b/twscrape/account.py index d567d7c..2b6bf4f 100644 --- a/twscrape/account.py +++ b/twscrape/account.py @@ -5,10 +5,11 @@ from datetime import datetime from httpx import AsyncClient, AsyncHTTPTransport -from .constants import TOKEN from .models import JSONTrait from .utils import utc +TOKEN = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" + @dataclass class Account(JSONTrait): diff --git a/twscrape/accounts_pool.py b/twscrape/accounts_pool.py index 17d861e..8433c29 100644 --- a/twscrape/accounts_pool.py +++ b/twscrape/accounts_pool.py @@ -1,4 +1,3 @@ -# ruff: noqa: E501 import asyncio import sqlite3 import uuid diff --git a/twscrape/api.py b/twscrape/api.py index 02d55f7..5e94041 100644 --- a/twscrape/api.py +++ b/twscrape/api.py @@ -1,17 +1,52 @@ -# ruff: noqa: F405 from httpx import Response from .accounts_pool import AccountsPool -from .constants import * # noqa: F403 from .logger import set_log_level from .models import Tweet, User, parse_tweet, parse_tweets, parse_user, parse_users from .queue_client import QueueClient from .utils import encode_params, find_obj, get_by_path -# Note: kv is variables, ft is features from original GQL request +OP_SearchTimeline = "Aj1nGkALq99Xg3XI0OZBtw/SearchTimeline" +OP_UserByRestId = "CO4_gU4G_MRREoqfiTh6Hg/UserByRestId" +OP_UserByScreenName = "NimuplG1OB7Fd2btCLdBOw/UserByScreenName" +OP_TweetDetail = "-H4B_lJDEA-O_7_qWaRiyg/TweetDetail" +OP_Followers = "3_7xfjmh897x8h_n6QBqTA/Followers" +OP_Following = "0yD6Eiv23DKXRDU9VxlG2A/Following" +OP_Retweeters = "sOBhVzDeJl4XGepvi5pHlg/Retweeters" +OP_Favoriters = "E-ZTxvWWIkmOKwYdNTEefg/Favoriters" +OP_UserTweets = "V1ze5q3ijDS1VeLwLY0m7g/UserTweets" +OP_UserTweetsAndReplies = "16nOjYqEdV04vN6-rgg8KA/UserTweetsAndReplies" +OP_ListLatestTweetsTimeline = "whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline" + + +GQL_URL = "https://twitter.com/i/api/graphql" +GQL_FEATURES = { # search values here (view source) https://twitter.com/ + "responsive_web_graphql_exclude_directive_enabled": True, + "verified_phone_label_enabled": False, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False, + "responsive_web_graphql_timeline_navigation_enabled": True, + "tweetypie_unmention_optimization_enabled": True, + "responsive_web_edit_tweet_api_enabled": True, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True, + "view_counts_everywhere_api_enabled": True, + "longform_notetweets_consumption_enabled": True, + "tweet_awards_web_tipping_enabled": False, + "freedom_of_speech_not_reach_fetch_enabled": True, + "standardized_nudges_misinfo": True, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True, + "longform_notetweets_rich_text_read_enabled": True, + "responsive_web_enhance_cards_enabled": False, + "creator_subscriptions_tweet_preview_api_enabled": True, + "longform_notetweets_inline_media_enabled": True, + "responsive_web_media_download_video_enabled": False, + "responsive_web_twitter_article_tweet_consumption_enabled": False, + "c9s_tweet_anatomy_moderator_badge_enabled": True, + "rweb_video_timestamps_enabled": True, +} class API: + # Note: kv is variables, ft is features from original GQL request pool: AccountsPool def __init__(self, pool: AccountsPool | str | None = None, debug=False): diff --git a/twscrape/constants.py b/twscrape/constants.py deleted file mode 100644 index 9d212b5..0000000 --- a/twscrape/constants.py +++ /dev/null @@ -1,48 +0,0 @@ -TOKEN = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" # noqa: E501 - -GQL_URL = "https://twitter.com/i/api/graphql" -LOGIN_URL = "https://api.twitter.com/1.1/onboarding/task.json" - - -OP_SearchTimeline = "Aj1nGkALq99Xg3XI0OZBtw/SearchTimeline" -OP_UserByRestId = "CO4_gU4G_MRREoqfiTh6Hg/UserByRestId" -OP_UserByScreenName = "NimuplG1OB7Fd2btCLdBOw/UserByScreenName" -OP_TweetDetail = "-H4B_lJDEA-O_7_qWaRiyg/TweetDetail" -OP_Followers = "3_7xfjmh897x8h_n6QBqTA/Followers" -OP_Following = "0yD6Eiv23DKXRDU9VxlG2A/Following" -OP_Retweeters = "sOBhVzDeJl4XGepvi5pHlg/Retweeters" -OP_Favoriters = "E-ZTxvWWIkmOKwYdNTEefg/Favoriters" -OP_UserTweets = "V1ze5q3ijDS1VeLwLY0m7g/UserTweets" -OP_UserTweetsAndReplies = "16nOjYqEdV04vN6-rgg8KA/UserTweetsAndReplies" -OP_ListLatestTweetsTimeline = "whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline" - -# search values here (view source) https://twitter.com/ -GQL_FEATURES = { - # "blue_business_profile_image_shape_enabled": True, - "responsive_web_graphql_exclude_directive_enabled": True, - "verified_phone_label_enabled": False, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False, - "responsive_web_graphql_timeline_navigation_enabled": True, - "tweetypie_unmention_optimization_enabled": True, - # "vibe_api_enabled": True, - "responsive_web_edit_tweet_api_enabled": True, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True, - "view_counts_everywhere_api_enabled": True, - "longform_notetweets_consumption_enabled": True, - "tweet_awards_web_tipping_enabled": False, - "freedom_of_speech_not_reach_fetch_enabled": True, - "standardized_nudges_misinfo": True, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True, - # "interactive_text_enabled": True, - # "responsive_web_text_conversations_enabled": False, - "longform_notetweets_rich_text_read_enabled": True, - "responsive_web_enhance_cards_enabled": False, - "creator_subscriptions_tweet_preview_api_enabled": True, - "longform_notetweets_inline_media_enabled": True, - "responsive_web_media_download_video_enabled": False, - # "rweb_lists_timeline_redesign_enabled": True, - "responsive_web_twitter_article_tweet_consumption_enabled": False, - # "responsive_web_home_pinned_timelines_enabled": True, - "c9s_tweet_anatomy_moderator_badge_enabled": True, - "rweb_video_timestamps_enabled": True, -} diff --git a/twscrape/login.py b/twscrape/login.py index 4a37fc9..679feb5 100644 --- a/twscrape/login.py +++ b/twscrape/login.py @@ -3,11 +3,12 @@ from datetime import timedelta from httpx import AsyncClient, HTTPStatusError, Response from .account import Account -from .constants import LOGIN_URL from .imap import imap_get_email_code, imap_login from .logger import logger from .utils import raise_for_status, utc +LOGIN_URL = "https://api.twitter.com/1.1/onboarding/task.json" + async def get_guest_token(client: AsyncClient): rep = await client.post("https://api.twitter.com/1.1/guest/activate.json")