fix #107, v0.10.1

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "twscrape"
-version = "0.10.0"
+version = "0.10.1"
 authors = [{name = "vladkens", email = "v.pronsky@gmail.com"}]
 description = "Twitter GraphQL and Search API implementation with SNScrape data models"
 readme = "readme.md"

twscrape/login.py

@@ -217,27 +217,27 @@ async def login(acc: Account, cfg: LoginConfig | None = None) -> Account:
     if cfg.email_first and not cfg.manual:
         imap = await imap_login(acc.email, acc.email_password)
 
-    client = acc.make_client()
+    async with acc.make_client() as client:
-    guest_token = await get_guest_token(client)
+        guest_token = await get_guest_token(client)
-    client.headers["x-guest-token"] = guest_token
+        client.headers["x-guest-token"] = guest_token
 
-    rep = await login_initiate(client)
+        rep = await login_initiate(client)
-    ctx = TaskCtx(client, acc, cfg, None, imap)
+        ctx = TaskCtx(client, acc, cfg, None, imap)
-    while True:
+        while True:
-        if not rep:
+            if not rep:
-            break
+                break
 
-        try:
+            try:
-            rep = await next_login_task(ctx, rep)
+                rep = await next_login_task(ctx, rep)
-        except HTTPStatusError as e:
+            except HTTPStatusError as e:
-            if e.response.status_code == 403:
+                if e.response.status_code == 403:
-                logger.error(f"403 error {log_id}")
+                    logger.error(f"403 error {log_id}")
-                return acc
+                    return acc
 
-    client.headers["x-csrf-token"] = client.cookies["ct0"]
+        client.headers["x-csrf-token"] = client.cookies["ct0"]
-    client.headers["x-twitter-auth-type"] = "OAuth2Session"
+        client.headers["x-twitter-auth-type"] = "OAuth2Session"
 
-    acc.active = True
+        acc.active = True
-    acc.headers = {k: v for k, v in client.headers.items()}
+        acc.headers = {k: v for k, v in client.headers.items()}
-    acc.cookies = {k: v for k, v in client.cookies.items()}
+        acc.cookies = {k: v for k, v in client.cookies.items()}
-    return acc
+        return acc

twscrape/queue_client.py

@@ -2,7 +2,7 @@ import json
 import os
 from typing import Any
 
-import httpx
+from httpx import AsyncClient, HTTPStatusError, ProxyError, ReadTimeout, Response
 
 from .accounts_pool import Account, AccountsPool
 from .logger import logger
@@ -13,7 +13,7 @@ TMP_TS = utc.now().isoformat().split(".")[0].replace("T", "_").replace(":", "-")
 
 
 class Ctx:
-    def __init__(self, acc: Account, clt: httpx.AsyncClient):
+    def __init__(self, acc: Account, clt: AsyncClient):
         self.acc = acc
         self.clt = clt
         self.req_count = 0
@@ -27,7 +27,7 @@ class AbortReqError(Exception):
     pass
 
 
-def req_id(rep: httpx.Response):
+def req_id(rep: Response):
     lr = str(rep.headers.get("x-rate-limit-remaining", -1))
     ll = str(rep.headers.get("x-rate-limit-limit", -1))
     sz = max(len(lr), len(ll))
@@ -37,7 +37,7 @@ def req_id(rep: httpx.Response):
     return f"{lr}/{ll} - {username}"
 
 
-def dump_rep(rep: httpx.Response):
+def dump_rep(rep: Response):
     count = getattr(dump_rep, "__count", -1) + 1
     setattr(dump_rep, "__count", count)
 
@@ -108,7 +108,7 @@ class QueueClient:
         self.ctx = Ctx(acc, clt)
         return self.ctx
 
-    async def _check_rep(self, rep: httpx.Response) -> None:
+    async def _check_rep(self, rep: Response) -> None:
         """
         This function can raise Exception and request will be retried or aborted
         Or if None is returned, response will passed to api parser as is
@@ -186,7 +186,7 @@ class QueueClient:
 
         try:
             rep.raise_for_status()
-        except httpx.HTTPStatusError:
+        except HTTPStatusError:
             logger.error(f"Unhandled API response code: {log_msg}")
             await self._close_ctx(utc.ts() + 60 * 15)  # 15 minutes
             raise HandledError()
@@ -194,10 +194,10 @@ class QueueClient:
     async def get(self, url: str, params: ReqParams = None):
         return await self.req("GET", url, params=params)
 
-    async def req(self, method: str, url: str, params: ReqParams = None) -> httpx.Response | None:
+    async def req(self, method: str, url: str, params: ReqParams = None) -> Response | None:
         retry_count = 0
         while True:
-            ctx = await self._get_ctx()
+            ctx = await self._get_ctx()  # not need to close client, class implements __aexit__
             if ctx is None:
                 return None
 
@@ -215,7 +215,7 @@ class QueueClient:
             except HandledError:
                 # retry with new account
                 continue
-            except (httpx.ReadTimeout, httpx.ProxyError):
+            except (ReadTimeout, ProxyError):
                 # http transport failed, just retry with same account
                 continue
             except Exception as e: