зеркало из
https://github.com/viginum-datalab/twscrape.git
synced 2025-10-29 21:16:25 +02:00
update readme; fix ci
Этот коммит содержится в:
родитель
9509378441
Коммит
ab3ffda420
5
.github/workflows/ci.yml
поставляемый
5
.github/workflows/ci.yml
поставляемый
@ -18,10 +18,7 @@ jobs:
|
|||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
- name: install dependencies
|
- name: install dependencies
|
||||||
run: |
|
run: pip install -e .[dev]
|
||||||
python -m pip install --upgrade pip
|
|
||||||
pip install --quiet -r requirements.txt
|
|
||||||
pip install --quiet -r requirements-dev.txt
|
|
||||||
|
|
||||||
- name: lint
|
- name: lint
|
||||||
run: make lint
|
run: make lint
|
||||||
|
|||||||
@ -6,6 +6,14 @@ Twitter GraphQL and Search API implementation with [SNScrape](https://github.com
|
|||||||
pip install https://github.com/vladkens/tw-api
|
pip install https://github.com/vladkens/tw-api
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
- Support both Search & GraphQL Twitter API
|
||||||
|
- Async / Await functions (can run multiple scrappers in parallel same time)
|
||||||
|
- Login flow (with receiving verification code from email)
|
||||||
|
- Saving / restore accounts sessions
|
||||||
|
- Raw Twitter API responses & SNScrape models
|
||||||
|
- Automatic account switching to smooth Twitter API rate limits
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@ -60,7 +68,6 @@ async def main():
|
|||||||
doc.json() # -> json string
|
doc.json() # -> json string
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
```
|
```
|
||||||
|
|||||||
@ -2,5 +2,5 @@
|
|||||||
from .account import Account
|
from .account import Account
|
||||||
from .accounts_pool import AccountsPool
|
from .accounts_pool import AccountsPool
|
||||||
from .api import API
|
from .api import API
|
||||||
from .models import *
|
from .models import * # noqa: F403
|
||||||
from .utils import gather
|
from .utils import gather
|
||||||
|
|||||||
@ -1,12 +1,10 @@
|
|||||||
import email.utils
|
import email.utils
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from snscrape.modules import twitter
|
|
||||||
|
|
||||||
from .logger import logger
|
from .logger import logger
|
||||||
from .utils import find_item, get_or, int_or_none
|
from .utils import find_item, get_or, int_or_none
|
||||||
|
|
||||||
|
|||||||
@ -83,7 +83,7 @@ def get_typed_object(obj: dict, res: defaultdict[str, list]):
|
|||||||
if obj_type is not None:
|
if obj_type is not None:
|
||||||
res[obj_type].append(obj)
|
res[obj_type].append(obj)
|
||||||
|
|
||||||
for k, v in obj.items():
|
for _, v in obj.items():
|
||||||
if isinstance(v, dict):
|
if isinstance(v, dict):
|
||||||
get_typed_object(v, res)
|
get_typed_object(v, res)
|
||||||
elif isinstance(v, list):
|
elif isinstance(v, list):
|
||||||
|
|||||||
Загрузка…
x
Ссылка в новой задаче
Block a user