diff --git a/twscrape/accounts_pool.py b/twscrape/accounts_pool.py index 1e05687..06877a1 100644 --- a/twscrape/accounts_pool.py +++ b/twscrape/accounts_pool.py @@ -236,11 +236,9 @@ class AccountsPool: AND json_extract(locks, '$.{queue}') > datetime('now') """ - gql_ops = """ - SearchTimeline UserByRestId UserByScreenName TweetDetail Followers Following - Retweeters Favoriters UserTweets UserTweetsAndReplies - """ - gql_ops = [x.strip() for x in gql_ops.split(" ") if x.strip()] + qs = "SELECT DISTINCT(f.key) as k from accounts, json_each(stats) f" + rs = await fetchall(self._db_file, qs) + gql_ops = [x["k"] for x in rs] config = [ ("total", "SELECT COUNT(*) FROM accounts"), diff --git a/twscrape/cli.py b/twscrape/cli.py index 1443833..867774f 100644 --- a/twscrape/cli.py +++ b/twscrape/cli.py @@ -58,7 +58,18 @@ async def main(args): return if args.command == "stats": - print(await pool.stats()) + rep = await pool.stats() + total, active, inactive = rep["total"], rep["active"], rep["inactive"] + + res = [] + for k, v in rep.items(): + if not k.startswith("locked") or v == 0: + continue + res.append({"queue": k, "locked": v, "available": max(active - v, 0)}) + + res = sorted(res, key=lambda x: x["locked"], reverse=True) + print_table(res, hr_after=True) + print(f"Total: {total} - Active: {active} - Inactive: {inactive}") return if args.command == "add_accounts": @@ -141,6 +152,7 @@ def run(): relogin = subparsers.add_parser("relogin", help="Re-login selected accounts") relogin.add_argument("usernames", nargs="+", default=[], help="Usernames to re-login") subparsers.add_parser("relogin_failed", help="Retry login for failed accounts") + subparsers.add_parser("stats", help="Get current usage stats") c_lim("search", "Search for tweets", "query", "Search query") c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int) diff --git a/twscrape/queue_client.py b/twscrape/queue_client.py index f39d411..fbaac24 100644 --- a/twscrape/queue_client.py +++ b/twscrape/queue_client.py @@ -127,7 +127,7 @@ class QueueClient: logger.error(f"[{rep.status_code}] {e.request.url}\n{rep.text}") raise e except Exception as e: - logger.warning(f"Unknown error, retrying. Err: {e}") + logger.warning(f"Unknown error, retrying. Err ({type(e)}): {str(e)}") async def get(self, url: str, params: ReqParams = None): try: diff --git a/twscrape/utils.py b/twscrape/utils.py index 23be6ea..4ea72ff 100644 --- a/twscrape/utils.py +++ b/twscrape/utils.py @@ -144,7 +144,7 @@ def from_utciso(iso: str) -> datetime: return datetime.fromisoformat(iso).replace(tzinfo=timezone.utc) -def print_table(rows: list[dict]): +def print_table(rows: list[dict], hr_after=False): if not rows: return @@ -169,7 +169,9 @@ def print_table(rows: list[dict]): line = [f"{row[k]:<{colw[i]}}" for i, k in enumerate(keys)] lines.append(" ".join(line)) - # max_len = max(len(x) for x in lines) + max_len = max(len(x) for x in lines) # lines.insert(1, "─" * max_len) # lines.insert(0, "─" * max_len) print("\n".join(lines)) + if hr_after: + print("-" * max_len)