update logs; print next available account time for queue

twscrape/accounts_pool.py

@@ -72,11 +72,9 @@ class AccountsPool:
         qs = "SELECT * FROM accounts WHERE username = :username"
         rs = await fetchone(self._db_file, qs, {"username": username})
         if rs:
-            logger.debug(f"Account {username} already exists")
+            logger.warning(f"Account {username} already exists")
             return
 
-        logger.debug(f"Adding account {username}")
-
         account = Account(
             username=username,
             password=password,
@@ -251,14 +249,35 @@ class AccountsPool:
             account = await self.get_for_queue(queue)
             if not account:
                 if not msg_show:
-                    logger.info(f"No accounts available for queue '{queue}' (sleeping for 5 sec)")
+                    nat = await self.next_available_at(queue)
+                    msg = f'No account available for queue "{queue}". Next available at {nat}'
+                    logger.info(msg)
                     msg_show = True
                 await asyncio.sleep(5)
                 continue
 
-            logger.debug(f"Using account {account.username} for queue '{queue}'")
             return account
 
+    async def next_available_at(self, queue: str):
+        qs = f"""
+        SELECT json_extract(locks, '$."{queue}"') as lock_until
+        FROM accounts
+        WHERE active = true AND json_extract(locks, '$."{queue}"') IS NOT NULL
+        ORDER BY lock_until ASC
+        LIMIT 1
+        """
+        rs = await fetchone(self._db_file, qs)
+        if rs:
+            now = datetime.utcnow().replace(tzinfo=timezone.utc)
+            trg = datetime.fromisoformat(rs[0]).replace(tzinfo=timezone.utc)
+            if trg < now:
+                return "now"
+
+            at_local = datetime.now() + (trg - now)
+            return at_local.strftime("%H:%M:%S")
+
+        return "none"
+
     async def stats(self):
         def locks_count(queue: str):
             return f"""

twscrape/api.py

@@ -2,6 +2,7 @@ from httpx import Response
 
 from .accounts_pool import AccountsPool
 from .constants import *
+from .logger import set_log_level
 from .models import Tweet, User
 from .queue_client import QueueClient
 from .utils import encode_params, find_obj, get_by_path, to_old_obj, to_old_rep
@@ -23,6 +24,8 @@ class API:
             self.pool = AccountsPool()
 
         self.debug = debug
+        if self.debug:
+            set_log_level("DEBUG")
 
     # general helpers
 

twscrape/cli.py

@@ -49,7 +49,6 @@ async def main(args):
         print(f"SQLite runtime: {sqlite3.sqlite_version} ({await get_sqlite_version()})")
         return
 
-    logger.debug(f"Using database: {args.db}")
     pool = AccountsPool(args.db)
     api = API(pool, debug=args.debug)
 

twscrape/db.py

@@ -79,9 +79,8 @@ async def migrate(db: aiosqlite.Connection):
     }
 
     logger.debug(f"Current migration v{uv} (latest v{len(migrations)})")
-
     for i in range(uv + 1, len(migrations) + 1):
-        logger.debug(f"Running migration to v{i}")
+        logger.info(f"Running migration to v{i}")
         try:
             await migrations[i]()
         except sqlite3.OperationalError as e:

twscrape/login.py

@@ -154,7 +154,6 @@ async def next_login_task(client: AsyncClient, acc: Account, rep: Response, imap
 
     prev = rep.json()
     assert "flow_token" in prev, f"flow_token not in {rep.text}"
-    logger.debug(f"login tasks: {[x['subtask_id'] for x in prev['subtasks']]}")
 
     for x in prev["subtasks"]:
         task_id = x["subtask_id"]

twscrape/queue_client.py

@@ -118,7 +118,7 @@ class QueueClient:
             msg = "; ".join([f'({x.get("code", -1)}) {x["message"]}' for x in res["errors"]])
 
         if self.debug:
-            fn = logger.info if rep.status_code == 200 else logger.warning
+            fn = logger.debug if rep.status_code == 200 else logger.warning
             fn(f"{rep.status_code:3d} - {req_id(rep)} - {msg}")
 
         if msg.startswith("The following features cannot be null"):

twscrape/utils.py

@@ -22,7 +22,7 @@ def raise_for_status(rep: Response, label: str):
     try:
         rep.raise_for_status()
     except HTTPStatusError as e:
-        logger.info(f"{label} - {rep.status_code} - {rep.text}")
+        logger.warning(f"{label} - {rep.status_code} - {rep.text}")
         raise e