rename project; add retries

Makefile

@@ -1,17 +1,22 @@
+.PHONY: all build
+
 all:
 	@echo "hi"
 
+build:
+	python -m build
+
 lint:
-	ruff check twapi
+	ruff check twscrape
 
 lint-fix:
-	ruff check --fix twapi
+	ruff check --fix twscrape
 
 pylint:
-	pylint --errors-only twapi
+	pylint --errors-only twscrape
 
 test:
-	pytest --cov=twapi tests/
+	pytest --cov=twscrape tests/
 
 act:
 	act --container-architecture linux/amd64

pyproject.toml

@@ -1,15 +1,15 @@
 [build-system]
-requires = ['setuptools>=61', 'setuptools_scm>=6.2']
+requires = ["hatchling"]
-build-backend = "setuptools.build_meta"
+build-backend = "hatchling.build"
 
 [project]
-name = "tw-api"
+name = "twscrape"
 version = "0.1.0"
-authors = [{name = "vladkens"}]
+authors = [{name = "vladkens", email = "v.pronsky@gmail.com"}]
 description = "Twitter GraphQL and Search API implementation with SNScrape data models"
 readme = "readme.md"
 requires-python = ">=3.10"
-keywords = ["twitter", "api", "scrape", "snscrape", "tw-api", "twapi"]
+keywords = ["twitter", "scrape", "scrapper", "api", "snscrape"]
 license = {text = "MIT"}
 classifiers = [
   'Development Status :: 4 - Beta',
@@ -34,10 +34,10 @@ dev = [
 ]
 
 [project.urls]
-repository = "https://github.com/vladkens/tw-api"
+repository = "https://github.com/vladkens/twscrape"
 
 [tool.setuptools]
-packages = ['twapi']
+packages = ['twscrape']
 
 [tool.pylint]
 max-line-length = 99

readme.md

@@ -18,8 +18,8 @@ pip install https://github.com/vladkens/tw-api
 
 ```python
 import asyncio
-from twapi import AccountsPool, API, gather
+from twscrape import AccountsPool, API, gather
-from twapi.logger import set_log_level
+from twscrape.logger import set_log_level
 
 async def main():
     pool = AccountsPool()  # or pool = AccountsPool("path-to.db") - default is `accounts.db` 

tests/test_parser.py

@@ -2,8 +2,8 @@ import asyncio
 import json
 import os
 
-from twapi import API, AccountsPool, gather
+from twscrape import API, AccountsPool, gather
-from twapi.logger import set_log_level
+from twscrape.logger import set_log_level
 
 BASE_DIR = os.path.dirname(__file__)
 DATA_DIR = os.path.join(BASE_DIR, "mocked-data")
@@ -247,10 +247,9 @@ async def test_user_tweets_and_replies():
 async def main():
     # prepare mock files from real twitter replies
     # you need to have some account to perform this
+    FRESH = False
 
     pool = AccountsPool()
-    pool.restore()
-
     api = API(pool)
 
     jobs = [
@@ -269,7 +268,7 @@ async def main():
     for filename, fn in jobs:
         filename = os.path.join(DATA_DIR, f"{filename}")
         print("-" * 20)
-        if os.path.exists(filename):
+        if os.path.exists(filename) and FRESH is False:
             print(f"File {filename} already exists")
             continue
 

tests/test_pool.py

@@ -1,7 +1,7 @@
 import os
 
-from twapi.accounts_pool import AccountsPool
+from twscrape.accounts_pool import AccountsPool
-from twapi.db import DB
+from twscrape.db import DB
 
 DB_FILE = "/tmp/twapi_test.db"
 

twscrape/account.py

@@ -3,7 +3,7 @@ import sqlite3
 from dataclasses import asdict, dataclass
 from datetime import datetime
 
-from httpx import AsyncClient
+from httpx import AsyncClient, AsyncHTTPTransport
 
 from .constants import TOKEN
 
@@ -57,7 +57,8 @@ class Account:
         return rs
 
     def make_client(self) -> AsyncClient:
-        client = AsyncClient(proxies=self.proxy)
+        transport = AsyncHTTPTransport(retries=2)
+        client = AsyncClient(proxies=self.proxy, follow_redirects=True, transport=transport)
 
         # saved from previous usage
         client.cookies.update(self.cookies)