diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 1569bf5..0000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -python 3.12.0 diff --git a/examples/parallel_search.py b/examples/parallel_search.py index 5b24d1e..e3d4ae5 100644 --- a/examples/parallel_search.py +++ b/examples/parallel_search.py @@ -2,6 +2,7 @@ This example shows how to use twscrape to complete some queries in parallel. To limit the number of concurrent requests, see examples/parallel_search_with_limit.py """ + import asyncio import twscrape diff --git a/examples/parallel_search_with_limit.py b/examples/parallel_search_with_limit.py index 64fefc7..d2a855f 100644 --- a/examples/parallel_search_with_limit.py +++ b/examples/parallel_search_with_limit.py @@ -1,6 +1,7 @@ """ This example shows how to use twscrape in parallel with concurrency limit. """ + import asyncio import time diff --git a/twscrape/accounts_pool.py b/twscrape/accounts_pool.py index 86a7089..0ec0ac7 100644 --- a/twscrape/accounts_pool.py +++ b/twscrape/accounts_pool.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from typing import TypedDict from fake_useragent import UserAgent +from httpx import HTTPStatusError from .account import Account from .db import execute, fetchall, fetchone @@ -144,8 +145,12 @@ class AccountsPool: await login(account, cfg=self._login_config) logger.info(f"Logged in to {account.username} successfully") return True + except HTTPStatusError as e: + rep = e.response + logger.error(f"Failed to login '{account.username}': {rep.status_code} - {rep.text}") + return False except Exception as e: - logger.error(f"Failed to login to {account.username}: {e}") + logger.error(f"Failed to login '{account.username}': {e}") return False finally: await self.save(account) diff --git a/twscrape/login.py b/twscrape/login.py index 7d8b019..592946f 100644 --- a/twscrape/login.py +++ b/twscrape/login.py @@ -3,12 +3,12 @@ from dataclasses import dataclass from datetime import timedelta from typing import Any -from httpx import AsyncClient, HTTPStatusError, Response +from httpx import AsyncClient, Response from .account import Account from .imap import imap_get_email_code, imap_login from .logger import logger -from .utils import raise_for_status, utc +from .utils import utc LOGIN_URL = "https://api.twitter.com/1.1/onboarding/task.json" @@ -30,7 +30,7 @@ class TaskCtx: async def get_guest_token(client: AsyncClient): rep = await client.post("https://api.twitter.com/1.1/guest/activate.json") - raise_for_status(rep, "guest_token (most likely ip ban)") + rep.raise_for_status() return rep.json()["guest_token"] @@ -43,7 +43,7 @@ async def login_initiate(client: AsyncClient) -> Response: } rep = await client.post(LOGIN_URL, params={"flow_name": "login"}, json=payload) - raise_for_status(rep, "login_initiate") + rep.raise_for_status() return rep @@ -59,7 +59,7 @@ async def login_alternate_identifier(ctx: TaskCtx, *, username: str) -> Response } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_alternate_identifier") + rep.raise_for_status() return rep @@ -75,7 +75,7 @@ async def login_instrumentation(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_instrumentation") + rep.raise_for_status() return rep @@ -99,7 +99,7 @@ async def login_enter_username(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_username") + rep.raise_for_status() return rep @@ -115,7 +115,7 @@ async def login_enter_password(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_password") + rep.raise_for_status() return rep @@ -131,7 +131,7 @@ async def login_duplication_check(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_duplication_check") + rep.raise_for_status() return rep @@ -147,7 +147,7 @@ async def login_confirm_email(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_confirm_email") + rep.raise_for_status() return rep @@ -174,7 +174,7 @@ async def login_confirm_email_code(ctx: TaskCtx): } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_confirm_email") + rep.raise_for_status() return rep @@ -185,7 +185,7 @@ async def login_success(ctx: TaskCtx) -> Response: } rep = await ctx.client.post(LOGIN_URL, json=payload) - raise_for_status(rep, "login_success") + rep.raise_for_status() return rep @@ -245,12 +245,7 @@ async def login(acc: Account, cfg: LoginConfig | None = None) -> Account: if not rep: break - try: - rep = await next_login_task(ctx, rep) - except HTTPStatusError as e: - if e.response.status_code == 403: - logger.error(f"403 error {log_id}") - return acc + rep = await next_login_task(ctx, rep) client.headers["x-csrf-token"] = client.cookies["ct0"] client.headers["x-twitter-auth-type"] = "OAuth2Session" diff --git a/twscrape/utils.py b/twscrape/utils.py index 51eaa1a..7f0c57e 100644 --- a/twscrape/utils.py +++ b/twscrape/utils.py @@ -4,10 +4,6 @@ from collections import defaultdict from datetime import datetime, timezone from typing import Any, AsyncGenerator, Callable, TypeVar -from httpx import HTTPStatusError, Response - -from .logger import logger - T = TypeVar("T") @@ -32,14 +28,6 @@ async def gather(gen: AsyncGenerator[T, None]) -> list[T]: return items -def raise_for_status(rep: Response, label: str): - try: - rep.raise_for_status() - except HTTPStatusError as e: - logger.warning(f"{label} - {rep.status_code} - {rep.text}") - raise e - - def encode_params(obj: dict): res = {} for k, v in obj.items():