diff --git a/Makefile b/Makefile index 3000e2f..f30a9c2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,3 @@ -.PHONY: all build - all: @echo "hi" diff --git a/readme.md b/readme.md index b254532..b302f06 100644 --- a/readme.md +++ b/readme.md @@ -1,21 +1,13 @@ # twscrape -
- - version - - - python versions - - - test status - - - downloads - - - license - +
+ +[version](https://pypi.org/project/twscrape) +[py versions](https://pypi.org/project/twscrape) +[test status](https://github.com/vladkens/twscrape/actions) +[downloads](https://pypi.org/project/twscrape) +[license](https://github.com/vladkens/twscrape/blob/main/LICENSE) +
Twitter GraphQL API implementation with [SNScrape](https://github.com/JustAnotherArchivist/snscrape) data models. @@ -239,6 +231,10 @@ By default, parsed data is returned. The original tweet responses can be retriev twscrape search "elon mask lang:es" --limit=20 --raw ``` +### Environment variables + +`LOGIN_CODE_TIMEOUT` - how long to wait for email code confirmation in seconds (default `40`) + ## Limitations After 1 July 2023 Twitter [introduced new limits](https://twitter.com/elonmusk/status/1675187969420828672) and still continue to update it periodically. diff --git a/twscrape/accounts_pool.py b/twscrape/accounts_pool.py index 0f6e189..17d861e 100644 --- a/twscrape/accounts_pool.py +++ b/twscrape/accounts_pool.py @@ -138,7 +138,7 @@ class AccountsPool: logger.info(f"Logged in to {account.username} successfully") return True except Exception as e: - logger.error(f"Error logging in to {account.username}: {e}") + logger.error(f"Failed to login to {account.username}: {e}") return False finally: await self.save(account) diff --git a/twscrape/imap.py b/twscrape/imap.py index 9f934ef..e124363 100644 --- a/twscrape/imap.py +++ b/twscrape/imap.py @@ -1,12 +1,16 @@ import asyncio import email as emaillib import imaplib +import os import time from datetime import datetime from .logger import logger +from .utils import int_or -MAX_WAIT_SEC = 30 +_env = dict(os.environ) + +LOGIN_CODE_TIMEOUT = int_or(_env, "LOGIN_CODE_TIMEOUT") or 40 class EmailLoginError(Exception): @@ -46,10 +50,11 @@ def _wait_email_code(imap: imaplib.IMAP4_SSL, count: int, min_t: datetime | None for x in rep: if isinstance(x, tuple): msg = emaillib.message_from_bytes(x[1]) - try: - msg_time = datetime.strptime(msg.get("Date", "").split(' (')[0], "%a, %d %b %Y %H:%M:%S %z") - except ValueError: - msg_time = msg.get("Date", "") + + # https://www.ietf.org/rfc/rfc9051.html#section-6.3.12-13 + msg_time = msg.get("Date", "").split("(")[0].strip() + msg_time = datetime.strptime(msg_time, "%a, %d %b %Y %H:%M:%S %z") + msg_from = str(msg.get("From", "")).lower() msg_subj = str(msg.get("Subject", "")).lower() logger.info(f"({i} of {count}) {msg_from} - {msg_time} - {msg_subj}") @@ -68,24 +73,22 @@ async def imap_get_email_code( imap: imaplib.IMAP4_SSL, email: str, min_t: datetime | None = None ) -> str: try: - start_time, was_count = time.time(), 0 + logger.info(f"Waiting for confirmation code for {email}...") + start_time = time.time() while True: _, rep = imap.select("INBOX") - now_count = int(rep[0].decode("utf-8")) if len(rep) > 0 and rep[0] is not None else 0 - if now_count > was_count: - code = _wait_email_code(imap, now_count, min_t) - if code is not None: - return code + msg_count = int(rep[0].decode("utf-8")) if len(rep) > 0 and rep[0] is not None else 0 + code = _wait_email_code(imap, msg_count, min_t) + if code is not None: + return code + + if LOGIN_CODE_TIMEOUT < time.time() - start_time: + raise EmailCodeTimeoutError(f"Email code timeout ({LOGIN_CODE_TIMEOUT} sec)") - logger.info(f"Waiting for confirmation code for {email}, msg_count: {now_count}") - if MAX_WAIT_SEC < time.time() - start_time: - logger.info(f"Timeout waiting for confirmation code for {email}") - raise EmailCodeTimeoutError() await asyncio.sleep(5) except Exception as e: imap.select("INBOX") imap.close() - logger.error(f"Error getting confirmation code for {email}: {e}") raise e diff --git a/twscrape/login.py b/twscrape/login.py index 4da3c87..a83142d 100644 --- a/twscrape/login.py +++ b/twscrape/login.py @@ -178,7 +178,6 @@ async def next_login_task(client: AsyncClient, acc: Account, rep: Response, imap return await login_instrumentation(client, acc, prev) except Exception as e: acc.error_msg = f"login_step={task_id} err={e}" - logger.error(f"Error in {task_id}: {e}") raise e return None