From 32f83ab1cd555be6a5ebd1ca0b2b6ed8fa9dc73e Mon Sep 17 00:00:00 2001 From: Vlad Pronsky Date: Mon, 8 Jan 2024 22:16:10 +0200 Subject: [PATCH] fix #107, v0.10.1 --- pyproject.toml | 2 +- twscrape/login.py | 40 ++++++++++++++++++++-------------------- twscrape/queue_client.py | 18 +++++++++--------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bbff0c2..befc8d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "twscrape" -version = "0.10.0" +version = "0.10.1" authors = [{name = "vladkens", email = "v.pronsky@gmail.com"}] description = "Twitter GraphQL and Search API implementation with SNScrape data models" readme = "readme.md" diff --git a/twscrape/login.py b/twscrape/login.py index 4f18838..bebf4c5 100644 --- a/twscrape/login.py +++ b/twscrape/login.py @@ -217,27 +217,27 @@ async def login(acc: Account, cfg: LoginConfig | None = None) -> Account: if cfg.email_first and not cfg.manual: imap = await imap_login(acc.email, acc.email_password) - client = acc.make_client() - guest_token = await get_guest_token(client) - client.headers["x-guest-token"] = guest_token + async with acc.make_client() as client: + guest_token = await get_guest_token(client) + client.headers["x-guest-token"] = guest_token - rep = await login_initiate(client) - ctx = TaskCtx(client, acc, cfg, None, imap) - while True: - if not rep: - break + rep = await login_initiate(client) + ctx = TaskCtx(client, acc, cfg, None, imap) + while True: + if not rep: + break - try: - rep = await next_login_task(ctx, rep) - except HTTPStatusError as e: - if e.response.status_code == 403: - logger.error(f"403 error {log_id}") - return acc + try: + rep = await next_login_task(ctx, rep) + except HTTPStatusError as e: + if e.response.status_code == 403: + logger.error(f"403 error {log_id}") + return acc - client.headers["x-csrf-token"] = client.cookies["ct0"] - client.headers["x-twitter-auth-type"] = "OAuth2Session" + client.headers["x-csrf-token"] = client.cookies["ct0"] + client.headers["x-twitter-auth-type"] = "OAuth2Session" - acc.active = True - acc.headers = {k: v for k, v in client.headers.items()} - acc.cookies = {k: v for k, v in client.cookies.items()} - return acc + acc.active = True + acc.headers = {k: v for k, v in client.headers.items()} + acc.cookies = {k: v for k, v in client.cookies.items()} + return acc diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index c1c1a5b..751e125 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -2,7 +2,7 @@ import json import os from typing import Any -import httpx +from httpx import AsyncClient, HTTPStatusError, ProxyError, ReadTimeout, Response from .accounts_pool import Account, AccountsPool from .logger import logger @@ -13,7 +13,7 @@ TMP_TS = utc.now().isoformat().split(".")[0].replace("T", "_").replace(":", "-") class Ctx: - def __init__(self, acc: Account, clt: httpx.AsyncClient): + def __init__(self, acc: Account, clt: AsyncClient): self.acc = acc self.clt = clt self.req_count = 0 @@ -27,7 +27,7 @@ class AbortReqError(Exception): pass -def req_id(rep: httpx.Response): +def req_id(rep: Response): lr = str(rep.headers.get("x-rate-limit-remaining", -1)) ll = str(rep.headers.get("x-rate-limit-limit", -1)) sz = max(len(lr), len(ll)) @@ -37,7 +37,7 @@ def req_id(rep: httpx.Response): return f"{lr}/{ll} - {username}" -def dump_rep(rep: httpx.Response): +def dump_rep(rep: Response): count = getattr(dump_rep, "__count", -1) + 1 setattr(dump_rep, "__count", count) @@ -108,7 +108,7 @@ class QueueClient: self.ctx = Ctx(acc, clt) return self.ctx - async def _check_rep(self, rep: httpx.Response) -> None: + async def _check_rep(self, rep: Response) -> None: """ This function can raise Exception and request will be retried or aborted Or if None is returned, response will passed to api parser as is @@ -186,7 +186,7 @@ class QueueClient: try: rep.raise_for_status() - except httpx.HTTPStatusError: + except HTTPStatusError: logger.error(f"Unhandled API response code: {log_msg}") await self._close_ctx(utc.ts() + 60 * 15) # 15 minutes raise HandledError() @@ -194,10 +194,10 @@ class QueueClient: async def get(self, url: str, params: ReqParams = None): return await self.req("GET", url, params=params) - async def req(self, method: str, url: str, params: ReqParams = None) -> httpx.Response | None: + async def req(self, method: str, url: str, params: ReqParams = None) -> Response | None: retry_count = 0 while True: - ctx = await self._get_ctx() + ctx = await self._get_ctx() # not need to close client, class implements __aexit__ if ctx is None: return None @@ -215,7 +215,7 @@ class QueueClient: except HandledError: # retry with new account continue - except (httpx.ReadTimeout, httpx.ProxyError): + except (ReadTimeout, ProxyError): # http transport failed, just retry with same account continue except Exception as e: