From ae6eabe2715abb5048c2a44f2a0501837fdeaaeb Mon Sep 17 00:00:00 2001 From: Vlad Pronsky Date: Sat, 6 May 2023 15:48:03 +0300 Subject: [PATCH] rename project; add retries --- Makefile | 13 +++++++++---- pyproject.toml | 22 +++++++++++----------- readme.md | 4 ++-- tests/test_parser.py | 9 ++++----- tests/test_pool.py | 4 ++-- {twapi => twscrape}/__init__.py | 0 {twapi => twscrape}/account.py | 5 +++-- {twapi => twscrape}/accounts_pool.py | 0 {twapi => twscrape}/api.py | 0 {twapi => twscrape}/constants.py | 0 {twapi => twscrape}/db.py | 0 {twapi => twscrape}/imap.py | 0 {twapi => twscrape}/logger.py | 0 {twapi => twscrape}/login.py | 0 {twapi => twscrape}/models.py | 0 {twapi => twscrape}/utils.py | 0 16 files changed, 31 insertions(+), 26 deletions(-) rename {twapi => twscrape}/__init__.py (100%) rename {twapi => twscrape}/account.py (91%) rename {twapi => twscrape}/accounts_pool.py (100%) rename {twapi => twscrape}/api.py (100%) rename {twapi => twscrape}/constants.py (100%) rename {twapi => twscrape}/db.py (100%) rename {twapi => twscrape}/imap.py (100%) rename {twapi => twscrape}/logger.py (100%) rename {twapi => twscrape}/login.py (100%) rename {twapi => twscrape}/models.py (100%) rename {twapi => twscrape}/utils.py (100%) diff --git a/Makefile b/Makefile index 3164b18..ca49076 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,22 @@ +.PHONY: all build + all: @echo "hi" +build: + python -m build + lint: - ruff check twapi + ruff check twscrape lint-fix: - ruff check --fix twapi + ruff check --fix twscrape pylint: - pylint --errors-only twapi + pylint --errors-only twscrape test: - pytest --cov=twapi tests/ + pytest --cov=twscrape tests/ act: act --container-architecture linux/amd64 diff --git a/pyproject.toml b/pyproject.toml index 181a463..2690bf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,21 @@ [build-system] -requires = ['setuptools>=61', 'setuptools_scm>=6.2'] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] -name = "tw-api" +name = "twscrape" version = "0.1.0" -authors = [{name = "vladkens"}] +authors = [{name = "vladkens", email = "v.pronsky@gmail.com"}] description = "Twitter GraphQL and Search API implementation with SNScrape data models" readme = "readme.md" requires-python = ">=3.10" -keywords = ["twitter", "api", "scrape", "snscrape", "tw-api", "twapi"] +keywords = ["twitter", "scrape", "scrapper", "api", "snscrape"] license = {text = "MIT"} classifiers = [ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ] dependencies = [ "aiosqlite==0.17.0", @@ -34,10 +34,10 @@ dev = [ ] [project.urls] -repository = "https://github.com/vladkens/tw-api" +repository = "https://github.com/vladkens/twscrape" [tool.setuptools] -packages = ['twapi'] +packages = ['twscrape'] [tool.pylint] max-line-length = 99 diff --git a/readme.md b/readme.md index 14840a7..0845a00 100644 --- a/readme.md +++ b/readme.md @@ -18,8 +18,8 @@ pip install https://github.com/vladkens/tw-api ```python import asyncio -from twapi import AccountsPool, API, gather -from twapi.logger import set_log_level +from twscrape import AccountsPool, API, gather +from twscrape.logger import set_log_level async def main(): pool = AccountsPool() # or pool = AccountsPool("path-to.db") - default is `accounts.db` diff --git a/tests/test_parser.py b/tests/test_parser.py index b75b092..4e10f6c 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,8 +2,8 @@ import asyncio import json import os -from twapi import API, AccountsPool, gather -from twapi.logger import set_log_level +from twscrape import API, AccountsPool, gather +from twscrape.logger import set_log_level BASE_DIR = os.path.dirname(__file__) DATA_DIR = os.path.join(BASE_DIR, "mocked-data") @@ -247,10 +247,9 @@ async def test_user_tweets_and_replies(): async def main(): # prepare mock files from real twitter replies # you need to have some account to perform this + FRESH = False pool = AccountsPool() - pool.restore() - api = API(pool) jobs = [ @@ -269,7 +268,7 @@ async def main(): for filename, fn in jobs: filename = os.path.join(DATA_DIR, f"{filename}") print("-" * 20) - if os.path.exists(filename): + if os.path.exists(filename) and FRESH is False: print(f"File {filename} already exists") continue diff --git a/tests/test_pool.py b/tests/test_pool.py index 1af0c1c..142d568 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -1,7 +1,7 @@ import os -from twapi.accounts_pool import AccountsPool -from twapi.db import DB +from twscrape.accounts_pool import AccountsPool +from twscrape.db import DB DB_FILE = "/tmp/twapi_test.db" diff --git a/twapi/__init__.py b/twscrape/__init__.py similarity index 100% rename from twapi/__init__.py rename to twscrape/__init__.py diff --git a/twapi/account.py b/twscrape/account.py similarity index 91% rename from twapi/account.py rename to twscrape/account.py index 4a450b1..8cebaa4 100644 --- a/twapi/account.py +++ b/twscrape/account.py @@ -3,7 +3,7 @@ import sqlite3 from dataclasses import asdict, dataclass from datetime import datetime -from httpx import AsyncClient +from httpx import AsyncClient, AsyncHTTPTransport from .constants import TOKEN @@ -57,7 +57,8 @@ class Account: return rs def make_client(self) -> AsyncClient: - client = AsyncClient(proxies=self.proxy) + transport = AsyncHTTPTransport(retries=2) + client = AsyncClient(proxies=self.proxy, follow_redirects=True, transport=transport) # saved from previous usage client.cookies.update(self.cookies) diff --git a/twapi/accounts_pool.py b/twscrape/accounts_pool.py similarity index 100% rename from twapi/accounts_pool.py rename to twscrape/accounts_pool.py diff --git a/twapi/api.py b/twscrape/api.py similarity index 100% rename from twapi/api.py rename to twscrape/api.py diff --git a/twapi/constants.py b/twscrape/constants.py similarity index 100% rename from twapi/constants.py rename to twscrape/constants.py diff --git a/twapi/db.py b/twscrape/db.py similarity index 100% rename from twapi/db.py rename to twscrape/db.py diff --git a/twapi/imap.py b/twscrape/imap.py similarity index 100% rename from twapi/imap.py rename to twscrape/imap.py diff --git a/twapi/logger.py b/twscrape/logger.py similarity index 100% rename from twapi/logger.py rename to twscrape/logger.py diff --git a/twapi/login.py b/twscrape/login.py similarity index 100% rename from twapi/login.py rename to twscrape/login.py diff --git a/twapi/models.py b/twscrape/models.py similarity index 100% rename from twapi/models.py rename to twscrape/models.py diff --git a/twapi/utils.py b/twscrape/utils.py similarity index 100% rename from twapi/utils.py rename to twscrape/utils.py