зеркало из
https://github.com/viginum-datalab/twscrape.git
synced 2025-10-29 13:06:13 +02:00
update readme; fix ci
Этот коммит содержится в:
родитель
9509378441
Коммит
ab3ffda420
5
.github/workflows/ci.yml
поставляемый
5
.github/workflows/ci.yml
поставляемый
@ -18,10 +18,7 @@ jobs:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install --quiet -r requirements.txt
|
||||
pip install --quiet -r requirements-dev.txt
|
||||
run: pip install -e .[dev]
|
||||
|
||||
- name: lint
|
||||
run: make lint
|
||||
|
||||
@ -6,6 +6,14 @@ Twitter GraphQL and Search API implementation with [SNScrape](https://github.com
|
||||
pip install https://github.com/vladkens/tw-api
|
||||
```
|
||||
|
||||
## Features
|
||||
- Support both Search & GraphQL Twitter API
|
||||
- Async / Await functions (can run multiple scrappers in parallel same time)
|
||||
- Login flow (with receiving verification code from email)
|
||||
- Saving / restore accounts sessions
|
||||
- Raw Twitter API responses & SNScrape models
|
||||
- Automatic account switching to smooth Twitter API rate limits
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
@ -60,7 +68,6 @@ async def main():
|
||||
doc.json() # -> json string
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
@ -2,5 +2,5 @@
|
||||
from .account import Account
|
||||
from .accounts_pool import AccountsPool
|
||||
from .api import API
|
||||
from .models import *
|
||||
from .models import * # noqa: F403
|
||||
from .utils import gather
|
||||
|
||||
@ -1,12 +1,10 @@
|
||||
import email.utils
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from snscrape.modules import twitter
|
||||
|
||||
from .logger import logger
|
||||
from .utils import find_item, get_or, int_or_none
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ def get_typed_object(obj: dict, res: defaultdict[str, list]):
|
||||
if obj_type is not None:
|
||||
res[obj_type].append(obj)
|
||||
|
||||
for k, v in obj.items():
|
||||
for _, v in obj.items():
|
||||
if isinstance(v, dict):
|
||||
get_typed_object(v, res)
|
||||
elif isinstance(v, list):
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user