diff --git a/twscrape/accounts_pool.py b/twscrape/accounts_pool.py index 10710fc..2f59a45 100644 --- a/twscrape/accounts_pool.py +++ b/twscrape/accounts_pool.py @@ -72,11 +72,9 @@ class AccountsPool: qs = "SELECT * FROM accounts WHERE username = :username" rs = await fetchone(self._db_file, qs, {"username": username}) if rs: - logger.debug(f"Account {username} already exists") + logger.warning(f"Account {username} already exists") return - logger.debug(f"Adding account {username}") - account = Account( username=username, password=password, @@ -251,14 +249,35 @@ class AccountsPool: account = await self.get_for_queue(queue) if not account: if not msg_show: - logger.info(f"No accounts available for queue '{queue}' (sleeping for 5 sec)") + nat = await self.next_available_at(queue) + msg = f'No account available for queue "{queue}". Next available at {nat}' + logger.info(msg) msg_show = True await asyncio.sleep(5) continue - logger.debug(f"Using account {account.username} for queue '{queue}'") return account + async def next_available_at(self, queue: str): + qs = f""" + SELECT json_extract(locks, '$."{queue}"') as lock_until + FROM accounts + WHERE active = true AND json_extract(locks, '$."{queue}"') IS NOT NULL + ORDER BY lock_until ASC + LIMIT 1 + """ + rs = await fetchone(self._db_file, qs) + if rs: + now = datetime.utcnow().replace(tzinfo=timezone.utc) + trg = datetime.fromisoformat(rs[0]).replace(tzinfo=timezone.utc) + if trg < now: + return "now" + + at_local = datetime.now() + (trg - now) + return at_local.strftime("%H:%M:%S") + + return "none" + async def stats(self): def locks_count(queue: str): return f""" diff --git a/twscrape/api.py b/twscrape/api.py index ed7844b..14d304e 100644 --- a/twscrape/api.py +++ b/twscrape/api.py @@ -2,6 +2,7 @@ from httpx import Response from .accounts_pool import AccountsPool from .constants import * +from .logger import set_log_level from .models import Tweet, User from .queue_client import QueueClient from .utils import encode_params, find_obj, get_by_path, to_old_obj, to_old_rep @@ -23,6 +24,8 @@ class API: self.pool = AccountsPool() self.debug = debug + if self.debug: + set_log_level("DEBUG") # general helpers diff --git a/twscrape/cli.py b/twscrape/cli.py index 44ce45c..fd27e5c 100644 --- a/twscrape/cli.py +++ b/twscrape/cli.py @@ -49,7 +49,6 @@ async def main(args): print(f"SQLite runtime: {sqlite3.sqlite_version} ({await get_sqlite_version()})") return - logger.debug(f"Using database: {args.db}") pool = AccountsPool(args.db) api = API(pool, debug=args.debug) diff --git a/twscrape/db.py b/twscrape/db.py index 5d7da3c..e7fb6f4 100644 --- a/twscrape/db.py +++ b/twscrape/db.py @@ -79,9 +79,8 @@ async def migrate(db: aiosqlite.Connection): } logger.debug(f"Current migration v{uv} (latest v{len(migrations)})") - for i in range(uv + 1, len(migrations) + 1): - logger.debug(f"Running migration to v{i}") + logger.info(f"Running migration to v{i}") try: await migrations[i]() except sqlite3.OperationalError as e: diff --git a/twscrape/login.py b/twscrape/login.py index e6324b2..9b84410 100644 --- a/twscrape/login.py +++ b/twscrape/login.py @@ -154,7 +154,6 @@ async def next_login_task(client: AsyncClient, acc: Account, rep: Response, imap prev = rep.json() assert "flow_token" in prev, f"flow_token not in {rep.text}" - logger.debug(f"login tasks: {[x['subtask_id'] for x in prev['subtasks']]}") for x in prev["subtasks"]: task_id = x["subtask_id"] diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index 9af36cf..5035aac 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -118,7 +118,7 @@ class QueueClient: msg = "; ".join([f'({x.get("code", -1)}) {x["message"]}' for x in res["errors"]]) if self.debug: - fn = logger.info if rep.status_code == 200 else logger.warning + fn = logger.debug if rep.status_code == 200 else logger.warning fn(f"{rep.status_code:3d} - {req_id(rep)} - {msg}") if msg.startswith("The following features cannot be null"): diff --git a/twscrape/utils.py b/twscrape/utils.py index d1d32e4..a347058 100644 --- a/twscrape/utils.py +++ b/twscrape/utils.py @@ -22,7 +22,7 @@ def raise_for_status(rep: Response, label: str): try: rep.raise_for_status() except HTTPStatusError as e: - logger.info(f"{label} - {rep.status_code} - {rep.text}") + logger.warning(f"{label} - {rep.status_code} - {rep.text}") raise e