replace black/isort/pylint with ruff

.vscode/settings.json

@@ -1,14 +1,14 @@
 {
-  "files.exclude": {
-    ".ruff_cache": true,
-    ".pytest_cache": true,
-    "*.egg-info": true,
-    "build": true,
-    ".coverage": true,
-  },
   "[python]": {
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": ["source.organizeImports"],
-    "editor.defaultFormatter": "ms-python.black-formatter"
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "files.exclude": {
+    ".ruff_cache": true,
+    ".pytest_cache": true,
+    ".coverage": true,
+    "htmlcov": true,
+    "dist": true
   },
 }

Makefile

@@ -7,24 +7,11 @@ install:
 build:
 	@python -m build
 
-ci:
-	@make format
-	@make lint
-	@make test
-
-format:
-	@black .
-
 lint:
-	@ruff check twscrape
+  # https://docs.astral.sh/ruff/settings/#sorting-imports
-	@ruff check tests
+	@ruff check --select I --fix .
-
+	@ruff format .
-lint-fix:
+	@ruff check .
-	@ruff check --fix twscrape
-	@ruff check --fix tests
-
-pylint:
-	@pylint --errors-only twscrape
 
 test:
 	@pytest -s --cov=twscrape tests/

pyproject.toml

@@ -2,9 +2,6 @@
 requires = ["hatchling>=1.9.1"]
 build-backend = "hatchling.build"
 
-[tool.hatch.build.targets.wheel]
-packages = ["twscrape"]
-
 [project]
 name = "twscrape"
 version = "0.9.0"
@@ -30,12 +27,11 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-  "pylint>=2.17.3",
+  "pytest-asyncio>=0.23.3",
-  "pytest-asyncio>=0.21.0",
+  "pytest-cov>=4.1.0",
-  "pytest-cov>=4.0.0",
+  "pytest-httpx>=0.28.0",
-  "pytest-httpx>=0.22.0",
+  "pytest>=7.4.4",
-  "pytest>=7.4.0",
+  "ruff>=0.1.11"
-  "ruff"
 ]
 
 [project.urls]
@@ -47,30 +43,20 @@ twscrape = "twscrape.cli:run"
 [tool.setuptools]
 packages = ['twscrape']
 
-[tool.pylint]
+[tool.hatch.build.targets.wheel]
-max-line-length = 99
+packages = ["twscrape"]
-disable = [
+
-  "C0103", # invalid-name
+[tool.hatch.metadata]
-  "C0114", # missing-module-docstring
+allow-direct-references = true
-  "C0115", # missing-class-docstring
-  "C0116", # missing-function-docstring
-  "R0903", # too-few-public-methods
-  "R0913", # too-many-arguments
-  "W0105", # pointless-string-statement
-]
 
 [tool.pytest.ini_options]
 pythonpath = ["."]
 asyncio_mode = "auto"
 
-[tool.isort]
-profile = "black"
-
-[tool.black]
-line-length = 99
-
 [tool.ruff]
 line-length = 99
 
-[tool.hatch.metadata]
+[tool.ruff.lint]
-allow-direct-references = true
+ignore = ["E501"]
+
+[tool.ruff.format]

tests/test_utils.py

@@ -1,4 +1,5 @@
-# ruff: noqa: E501
+import pytest
+
 from twscrape.utils import parse_cookies
 
 
@@ -17,3 +18,10 @@ def test_cookies_parse():
 
     val = "W3sibmFtZSI6ICJhYmMiLCAidmFsdWUiOiAiMTIzIn0sIHsibmFtZSI6ICJkZWYiLCAidmFsdWUiOiAiNDU2In0sIHsibmFtZSI6ICJnaGkiLCAidmFsdWUiOiAiNzg5In1d"
     assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = '{"cookies": {"abc": "123", "def": "456", "ghi": "789"}}'
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    with pytest.raises(ValueError, match=r"Invalid cookie value: .+"):
+        val = "{invalid}"
+        assert parse_cookies(val) == {}

twscrape/account.py

@@ -5,10 +5,11 @@ from datetime import datetime
 
 from httpx import AsyncClient, AsyncHTTPTransport
 
-from .constants import TOKEN
 from .models import JSONTrait
 from .utils import utc
 
+TOKEN = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+
 
 @dataclass
 class Account(JSONTrait):

twscrape/accounts_pool.py

@@ -1,4 +1,3 @@
-# ruff: noqa: E501
 import asyncio
 import sqlite3
 import uuid

twscrape/api.py

@@ -1,17 +1,52 @@
-# ruff: noqa: F405
 from httpx import Response
 
 from .accounts_pool import AccountsPool
-from .constants import *  # noqa: F403
 from .logger import set_log_level
 from .models import Tweet, User, parse_tweet, parse_tweets, parse_user, parse_users
 from .queue_client import QueueClient
 from .utils import encode_params, find_obj, get_by_path
 
-# Note: kv is variables, ft is features from original GQL request
+OP_SearchTimeline = "Aj1nGkALq99Xg3XI0OZBtw/SearchTimeline"
+OP_UserByRestId = "CO4_gU4G_MRREoqfiTh6Hg/UserByRestId"
+OP_UserByScreenName = "NimuplG1OB7Fd2btCLdBOw/UserByScreenName"
+OP_TweetDetail = "-H4B_lJDEA-O_7_qWaRiyg/TweetDetail"
+OP_Followers = "3_7xfjmh897x8h_n6QBqTA/Followers"
+OP_Following = "0yD6Eiv23DKXRDU9VxlG2A/Following"
+OP_Retweeters = "sOBhVzDeJl4XGepvi5pHlg/Retweeters"
+OP_Favoriters = "E-ZTxvWWIkmOKwYdNTEefg/Favoriters"
+OP_UserTweets = "V1ze5q3ijDS1VeLwLY0m7g/UserTweets"
+OP_UserTweetsAndReplies = "16nOjYqEdV04vN6-rgg8KA/UserTweetsAndReplies"
+OP_ListLatestTweetsTimeline = "whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline"
+
+
+GQL_URL = "https://twitter.com/i/api/graphql"
+GQL_FEATURES = {  # search values here (view source) https://twitter.com/
+    "responsive_web_graphql_exclude_directive_enabled": True,
+    "verified_phone_label_enabled": False,
+    "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
+    "responsive_web_graphql_timeline_navigation_enabled": True,
+    "tweetypie_unmention_optimization_enabled": True,
+    "responsive_web_edit_tweet_api_enabled": True,
+    "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
+    "view_counts_everywhere_api_enabled": True,
+    "longform_notetweets_consumption_enabled": True,
+    "tweet_awards_web_tipping_enabled": False,
+    "freedom_of_speech_not_reach_fetch_enabled": True,
+    "standardized_nudges_misinfo": True,
+    "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True,
+    "longform_notetweets_rich_text_read_enabled": True,
+    "responsive_web_enhance_cards_enabled": False,
+    "creator_subscriptions_tweet_preview_api_enabled": True,
+    "longform_notetweets_inline_media_enabled": True,
+    "responsive_web_media_download_video_enabled": False,
+    "responsive_web_twitter_article_tweet_consumption_enabled": False,
+    "c9s_tweet_anatomy_moderator_badge_enabled": True,
+    "rweb_video_timestamps_enabled": True,
+}
 
 
 class API:
+    # Note: kv is variables, ft is features from original GQL request
     pool: AccountsPool
 
     def __init__(self, pool: AccountsPool | str | None = None, debug=False):

twscrape/constants.py

@@ -1,48 +0,0 @@
-TOKEN = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"  # noqa: E501
-
-GQL_URL = "https://twitter.com/i/api/graphql"
-LOGIN_URL = "https://api.twitter.com/1.1/onboarding/task.json"
-
-
-OP_SearchTimeline = "Aj1nGkALq99Xg3XI0OZBtw/SearchTimeline"
-OP_UserByRestId = "CO4_gU4G_MRREoqfiTh6Hg/UserByRestId"
-OP_UserByScreenName = "NimuplG1OB7Fd2btCLdBOw/UserByScreenName"
-OP_TweetDetail = "-H4B_lJDEA-O_7_qWaRiyg/TweetDetail"
-OP_Followers = "3_7xfjmh897x8h_n6QBqTA/Followers"
-OP_Following = "0yD6Eiv23DKXRDU9VxlG2A/Following"
-OP_Retweeters = "sOBhVzDeJl4XGepvi5pHlg/Retweeters"
-OP_Favoriters = "E-ZTxvWWIkmOKwYdNTEefg/Favoriters"
-OP_UserTweets = "V1ze5q3ijDS1VeLwLY0m7g/UserTweets"
-OP_UserTweetsAndReplies = "16nOjYqEdV04vN6-rgg8KA/UserTweetsAndReplies"
-OP_ListLatestTweetsTimeline = "whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline"
-
-# search values here (view source) https://twitter.com/
-GQL_FEATURES = {
-    # "blue_business_profile_image_shape_enabled": True,
-    "responsive_web_graphql_exclude_directive_enabled": True,
-    "verified_phone_label_enabled": False,
-    "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
-    "responsive_web_graphql_timeline_navigation_enabled": True,
-    "tweetypie_unmention_optimization_enabled": True,
-    # "vibe_api_enabled": True,
-    "responsive_web_edit_tweet_api_enabled": True,
-    "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
-    "view_counts_everywhere_api_enabled": True,
-    "longform_notetweets_consumption_enabled": True,
-    "tweet_awards_web_tipping_enabled": False,
-    "freedom_of_speech_not_reach_fetch_enabled": True,
-    "standardized_nudges_misinfo": True,
-    "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True,
-    # "interactive_text_enabled": True,
-    # "responsive_web_text_conversations_enabled": False,
-    "longform_notetweets_rich_text_read_enabled": True,
-    "responsive_web_enhance_cards_enabled": False,
-    "creator_subscriptions_tweet_preview_api_enabled": True,
-    "longform_notetweets_inline_media_enabled": True,
-    "responsive_web_media_download_video_enabled": False,
-    # "rweb_lists_timeline_redesign_enabled": True,
-    "responsive_web_twitter_article_tweet_consumption_enabled": False,
-    # "responsive_web_home_pinned_timelines_enabled": True,
-    "c9s_tweet_anatomy_moderator_badge_enabled": True,
-    "rweb_video_timestamps_enabled": True,
-}

twscrape/login.py

@@ -3,11 +3,12 @@ from datetime import timedelta
 from httpx import AsyncClient, HTTPStatusError, Response
 
 from .account import Account
-from .constants import LOGIN_URL
 from .imap import imap_get_email_code, imap_login
 from .logger import logger
 from .utils import raise_for_status, utc
 
+LOGIN_URL = "https://api.twitter.com/1.1/onboarding/task.json"
+
 
 async def get_guest_token(client: AsyncClient):
     rep = await client.post("https://api.twitter.com/1.1/guest/activate.json")