feat: current usage overview in cli

twscrape/accounts_pool.py

@@ -236,11 +236,9 @@ class AccountsPool:
                 AND json_extract(locks, '$.{queue}') > datetime('now')
             """
 
-        gql_ops = """
-        SearchTimeline UserByRestId UserByScreenName TweetDetail Followers Following
-        Retweeters Favoriters UserTweets UserTweetsAndReplies
-        """
-        gql_ops = [x.strip() for x in gql_ops.split(" ") if x.strip()]
+        qs = "SELECT DISTINCT(f.key) as k from accounts, json_each(stats) f"
+        rs = await fetchall(self._db_file, qs)
+        gql_ops = [x["k"] for x in rs]
 
         config = [
             ("total", "SELECT COUNT(*) FROM accounts"),

twscrape/cli.py

@@ -58,7 +58,18 @@ async def main(args):
         return
 
     if args.command == "stats":
-        print(await pool.stats())
+        rep = await pool.stats()
+        total, active, inactive = rep["total"], rep["active"], rep["inactive"]
+
+        res = []
+        for k, v in rep.items():
+            if not k.startswith("locked") or v == 0:
+                continue
+            res.append({"queue": k, "locked": v, "available": max(active - v, 0)})
+
+        res = sorted(res, key=lambda x: x["locked"], reverse=True)
+        print_table(res, hr_after=True)
+        print(f"Total: {total} - Active: {active} - Inactive: {inactive}")
         return
 
     if args.command == "add_accounts":
@@ -141,6 +152,7 @@ def run():
     relogin = subparsers.add_parser("relogin", help="Re-login selected accounts")
     relogin.add_argument("usernames", nargs="+", default=[], help="Usernames to re-login")
     subparsers.add_parser("relogin_failed", help="Retry login for failed accounts")
+    subparsers.add_parser("stats", help="Get current usage stats")
 
     c_lim("search", "Search for tweets", "query", "Search query")
     c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int)

twscrape/queue_client.py

@@ -127,7 +127,7 @@ class QueueClient:
                 logger.error(f"[{rep.status_code}] {e.request.url}\n{rep.text}")
                 raise e
             except Exception as e:
-                logger.warning(f"Unknown error, retrying. Err: {e}")
+                logger.warning(f"Unknown error, retrying. Err ({type(e)}): {str(e)}")
 
     async def get(self, url: str, params: ReqParams = None):
         try:

twscrape/utils.py

@@ -144,7 +144,7 @@ def from_utciso(iso: str) -> datetime:
     return datetime.fromisoformat(iso).replace(tzinfo=timezone.utc)
 
 
-def print_table(rows: list[dict]):
+def print_table(rows: list[dict], hr_after=False):
     if not rows:
         return
 
@@ -169,7 +169,9 @@ def print_table(rows: list[dict]):
         line = [f"{row[k]:<{colw[i]}}" for i, k in enumerate(keys)]
         lines.append(" ".join(line))
 
-    # max_len = max(len(x) for x in lines)
+    max_len = max(len(x) for x in lines)
     # lines.insert(1, "─" * max_len)
     # lines.insert(0, "─" * max_len)
     print("\n".join(lines))
+    if hr_after:
+        print("-" * max_len)