improve sqlite version check, reduce minimal version #58

Dockerfile-test

@@ -1,26 +0,0 @@
-FROM python:3.10-alpine
-ARG SQLITE_Y=2021
-ARG SQLITE_V=3340100
-
-RUN pip install --upgrade pip
-RUN python -c "import sqlite3;print(sqlite3.sqlite_version)"
-
-# https://www.sqlite.org/chronology.html
-RUN apk add build-base
-RUN wget https://sqlite.org/${SQLITE_Y}/sqlite-autoconf-${SQLITE_V}.tar.gz -O sqlite.tar.gz \
-    && tar xvfz sqlite.tar.gz \
-    && cd sqlite-autoconf-${SQLITE_V} \
-    && ./configure --prefix=/usr/local --build=aarch64-unknown-linux-gnu \
-    && make \
-    && make install \
-    && cd .. \
-    && rm -rf sqlite*
-
-RUN sqlite3 --version
-RUN python -c "import sqlite3;print(sqlite3.sqlite_version)"
-
-WORKDIR /app
-COPY pyproject.toml readme.md /app/
-RUN pip install -e .[dev]
-COPY . /app
-RUN pytest tests/

Makefile

@@ -1,5 +1,7 @@
 .PHONY: all build
 
+SQTEST = docker -l warning build -f sqlite.dockerfile
+
 all:
 	@echo "hi"
 
@@ -44,7 +46,25 @@ changelog:
 	@git log $(shell git describe --tags --abbrev=0 HEAD)^..HEAD --pretty=format:'- %s'
 
 test34:
-	docker build -f Dockerfile-test .
+	@# https://www.sqlite.org/chronology.html
+	# @$(SQTEST) --build-arg SQLY=2017 --build-arg SQLV=3210000 -t twscrape_sq21 .
+	# @$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3220000 -t twscrape_sq22 .
+	@$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3230000 -t twscrape_sq23 .
+	@$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3240000 -t twscrape_sq24 .
+	@$(SQTEST) --build-arg SQLY=2019 --build-arg SQLV=3270200 -t twscrape_sq27 .
+	@$(SQTEST) --build-arg SQLY=2019 --build-arg SQLV=3300100 -t twscrape_sq30 .
+	@$(SQTEST) --build-arg SQLY=2020 --build-arg SQLV=3330000 -t twscrape_sq33 .
+	@$(SQTEST) --build-arg SQLY=2021 --build-arg SQLV=3340100 -t twscrape_sq34 .
+	@$(SQTEST) --build-arg SQLY=2023 --build-arg SQLV=3430000 -t twscrape_sq43 .
+	# @docker run twscrape_sq21
+	# @docker run twscrape_sq22
+	# @docker run twscrape_sq23
+	@docker run twscrape_sq24
+	@docker run twscrape_sq27
+	@docker run twscrape_sq30
+	@docker run twscrape_sq33
+	@docker run twscrape_sq34
+	@docker run twscrape_sq43
 
 update-mocks:
 	twscrape user_by_id --raw 2244994945 | jq > ./tests/mocked-data/user_by_id_raw.json

sqlite.dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10-alpine as base
+RUN apk add build-base
+
+FROM base
+ARG SQLY=2021
+ARG SQLV=3340100
+
+RUN wget https://sqlite.org/${SQLY}/sqlite-autoconf-${SQLV}.tar.gz -O sqlite.tar.gz \
+    && tar xvfz sqlite.tar.gz \
+    && cd sqlite-autoconf-${SQLV} \
+    && ./configure --prefix=/usr/local --build=aarch64-unknown-linux-gnu \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -rf sqlite*
+
+WORKDIR /app
+COPY pyproject.toml readme.md /app/
+RUN pip install -e .[dev]
+COPY . /app
+
+CMD sqlite3 --version; python -c "import sqlite3;print(sqlite3.sqlite_version)"; pytest tests/

twscrape/db.py

@@ -6,7 +6,7 @@ import aiosqlite
 
 from .logger import logger
 
-MIN_SQLITE_VERSION = "3.34"
+MIN_SQLITE_VERSION = "3.24"
 
 
 def lock_retry(max_retries=5, delay=1):
@@ -38,9 +38,12 @@ async def check_version():
     ver = await get_sqlite_version()
     ver = ".".join(ver.split(".")[:2])
 
-    if ver < MIN_SQLITE_VERSION:
+    try:
         msg = f"SQLite version '{ver}' is too old, please upgrade to {MIN_SQLITE_VERSION}+"
-        raise SystemError(msg)
+        if float(ver) < float(MIN_SQLITE_VERSION):
+            raise SystemError(msg)
+    except ValueError:
+        pass
 
 
 async def migrate(db: aiosqlite.Connection):