diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index dc75001..cdd9c87 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,18 +11,13 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build setuptools wheel twine - - name: Build and publish + python-version: "3.13" + - name: Build package + run: uv build + - name: Publish to PyPI env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python -m build - twine upload dist/* + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASSWORD }} + run: uv publish diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index db26bc2..2001c59 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -17,7 +17,7 @@ jobs: with: python-version: "3.13" - name: Install dependencies - run: uv sync --extra dev + run: uv sync --dev - name: lint with ruff run: | uv run ruff format tdclient --diff --exit-non-zero-on-fix diff --git a/README.rst b/README.rst index e903036..77fdb6c 100644 --- a/README.rst +++ b/README.rst @@ -319,7 +319,7 @@ Install the development extras and invoke ``ruff`` and ``pyright`` using .. code-block:: sh - $ uv sync --extra dev + $ uv sync --dev $ uv run ruff format tdclient --diff --exit-non-zero-on-fix $ uv run ruff check tdclient $ uv run pyright tdclient @@ -341,7 +341,7 @@ Install the development extras (which include ``tox``) with ``uv``. .. code-block:: sh - $ uv sync --extra dev + $ uv sync --dev Then, run ``tox`` via ``uv``. diff --git a/pyproject.toml b/pyproject.toml index 35fdb7f..a037258 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,11 +34,6 @@ dependencies = [ ] [project.optional-dependencies] -dev = [ - "ruff", - "pyright", - "tox>=4", -] docs = [ "sphinx", "sphinx_rtd_theme", @@ -78,15 +73,23 @@ known-third-party = ["dateutil","msgpack","pkg_resources","pytest","setuptools", [tool.pyright] include = ["tdclient"] exclude = ["**/__pycache__", "tdclient/test", "docs"] -typeCheckingMode = "basic" +typeCheckingMode = "strict" pythonVersion = "3.10" pythonPlatform = "All" -reportMissingTypeStubs = false -reportUnknownMemberType = false -reportUnknownArgumentType = false -reportUnknownVariableType = false -reportMissingImports = "warning" +reportMissingTypeStubs = "warning" +reportUnknownMemberType = "error" +reportUnknownArgumentType = "error" +reportMissingImports = "error" # Pre-commit venv configuration venvPath = "." venv = ".venv" + +[dependency-groups] +dev = [ + "ruff", + "pyright", + "tox>=4", + "msgpack-types>=0.5.0", + "types-certifi", +] diff --git a/tdclient/api.py b/tdclient/api.py index c308dd4..612c0f4 100644 --- a/tdclient/api.py +++ b/tdclient/api.py @@ -14,11 +14,11 @@ import time import urllib.parse as urlparse from array import array +from collections.abc import Iterator from typing import IO, Any, cast import msgpack import urllib3 -import urllib3.util from tdclient import errors, version from tdclient.bulk_import_api import BulkImportAPI @@ -31,7 +31,7 @@ from tdclient.schedule_api import ScheduleAPI from tdclient.server_status_api import ServerStatusAPI from tdclient.table_api import TableAPI -from tdclient.types import BytesOrStream, StreamBody +from tdclient.types import BytesOrStream, DataFormat, FileLike, StreamBody from tdclient.user_api import UserAPI from tdclient.util import ( csv_dict_record_reader, @@ -42,7 +42,7 @@ ) try: - import certifi + import certifi # type: ignore[reportMissingImports] except ImportError: certifi = None @@ -576,7 +576,9 @@ def close(self) -> None: # all connections in pool will be closed eventually during gc. self.http.clear() - def _prepare_file(self, file_like, fmt, **kwargs): + def _prepare_file( + self, file_like: FileLike, fmt: DataFormat, **kwargs: Any + ) -> IO[bytes]: fp = tempfile.TemporaryFile() with contextlib.closing(gzip.GzipFile(mode="wb", fileobj=fp)) as gz: packer = msgpack.Packer() @@ -591,34 +593,41 @@ def _prepare_file(self, file_like, fmt, **kwargs): fp.seek(0) return fp - def _read_file(self, file_like, fmt, **kwargs): + def _read_file(self, file_like: FileLike, fmt: DataFormat, **kwargs: Any) -> Any: compressed = fmt.endswith(".gz") + fmt_str = str(fmt) if compressed: - fmt = fmt[0 : len(fmt) - len(".gz")] - reader_name = f"_read_{fmt}_file" + fmt_str = fmt_str[0 : len(fmt_str) - len(".gz")] + reader_name = f"_read_{fmt_str}_file" if hasattr(self, reader_name): reader = getattr(self, reader_name) else: raise TypeError(f"unknown format: {fmt}") if hasattr(file_like, "read"): if compressed: - file_like = gzip.GzipFile(fileobj=file_like) + file_like = gzip.GzipFile(fileobj=file_like) # type: ignore[arg-type] return reader(file_like, **kwargs) else: + # At this point, file_like must be str or bytes (not IO[bytes]) + file_path = cast("str | bytes", file_like) if compressed: - file_like = gzip.GzipFile(fileobj=open(file_like, "rb")) + file_like = gzip.GzipFile(fileobj=open(file_path, "rb")) # type: ignore[arg-type] else: - file_like = open(file_like, "rb") + file_like = open(file_path, "rb") return reader(file_like, **kwargs) - def _read_msgpack_file(self, file_like, **kwargs): + def _read_msgpack_file( + self, file_like: IO[bytes], **kwargs: Any + ) -> Iterator[dict[str, Any]]: # current impl doesn't tolerate any unpack error - unpacker = msgpack.Unpacker(file_like, raw=False) + unpacker = msgpack.Unpacker(file_like, raw=False) # type: ignore[arg-type] for record in unpacker: validate_record(record) yield record - def _read_json_file(self, file_like, **kwargs): + def _read_json_file( + self, file_like: IO[bytes], **kwargs: Any + ) -> Iterator[dict[str, Any]]: # current impl doesn't tolerate any JSON parse error for s in file_like: record = json.loads(s.decode("utf-8")) @@ -627,20 +636,22 @@ def _read_json_file(self, file_like, **kwargs): def _read_csv_file( self, - file_like, - dialect=csv.excel, - columns=None, - encoding="utf-8", - dtypes=None, - converters=None, - **kwargs, - ): + file_like: IO[bytes], + dialect: type[csv.Dialect] = csv.excel, + columns: list[str] | None = None, + encoding: str = "utf-8", + dtypes: dict[str, Any] | None = None, + converters: dict[str, Any] | None = None, + **kwargs: Any, + ) -> Iterator[dict[str, Any]]: if columns is None: - reader = csv_dict_record_reader(file_like, encoding, dialect) + reader = csv_dict_record_reader(file_like, encoding, dialect) # type: ignore[arg-type] else: - reader = csv_text_record_reader(file_like, encoding, dialect, columns) + reader = csv_text_record_reader(file_like, encoding, dialect, columns) # type: ignore[arg-type] return read_csv_records(reader, dtypes, converters, **kwargs) - def _read_tsv_file(self, file_like, **kwargs): + def _read_tsv_file( + self, file_like: IO[bytes], **kwargs: Any + ) -> Iterator[dict[str, Any]]: return self._read_csv_file(file_like, dialect=csv.excel_tab, **kwargs) diff --git a/tdclient/bulk_import_api.py b/tdclient/bulk_import_api.py index 5a6c7ae..40ec900 100644 --- a/tdclient/bulk_import_api.py +++ b/tdclient/bulk_import_api.py @@ -26,7 +26,7 @@ class BulkImportAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -50,7 +50,7 @@ def raise_error( ) -> None: ... def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: ... def _prepare_file( - self, file_like: FileLike, fmt: str, **kwargs: Any + self, file_like: FileLike, fmt: DataFormat, **kwargs: Any ) -> IO[bytes]: ... def create_bulk_import( @@ -165,7 +165,7 @@ def validate_part_name(part_name: str) -> None: part_name (str): The part name the user is trying to use """ # Check for duplicate periods - d = collections.defaultdict(int) + d: collections.defaultdict[str, int] = collections.defaultdict(int) for char in part_name: d[char] += 1 @@ -378,5 +378,5 @@ def bulk_import_error_records( body = io.BytesIO(res.read()) decompressor = gzip.GzipFile(fileobj=body) - unpacker = msgpack.Unpacker(decompressor, raw=False) + unpacker = msgpack.Unpacker(decompressor, raw=False) # type: ignore[arg-type] yield from unpacker diff --git a/tdclient/bulk_import_model.py b/tdclient/bulk_import_model.py index 8ea061d..14a4782 100644 --- a/tdclient/bulk_import_model.py +++ b/tdclient/bulk_import_model.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from tdclient.model import Model -from tdclient.types import FileLike +from tdclient.types import BytesOrStream, DataFormat, FileLike if TYPE_CHECKING: from tdclient.client import Client @@ -112,7 +112,7 @@ def perform( self, wait: bool = False, wait_interval: int = 5, - wait_callback: Callable[[], None] | None = None, + wait_callback: Callable[["Job"], None] | None = None, timeout: float | None = None, ) -> "Job": """Perform bulk import @@ -162,7 +162,9 @@ def error_record_items(self) -> Iterator[dict[str, Any]]: """ yield from self._client.bulk_import_error_records(self.name) - def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> bool: + def upload_part( + self, part_name: str, bytes_or_stream: BytesOrStream, size: int + ) -> None: """Upload a part to bulk import session Args: @@ -177,8 +179,8 @@ def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> b return response def upload_file( - self, part_name: str, fmt: str, file_like: FileLike, **kwargs: Any - ) -> float: + self, part_name: str, fmt: DataFormat, file_like: FileLike, **kwargs: Any + ) -> None: """Upload a part to Bulk Import session, from an existing file on filesystem. Args: diff --git a/tdclient/client.py b/tdclient/client.py index b40048e..93c13f9 100644 --- a/tdclient/client.py +++ b/tdclient/client.py @@ -790,12 +790,12 @@ def history( """ result = self.api.history(name, _from or 0, to) - def scheduled_job(m): + def scheduled_job(m: tuple[Any, ...]) -> models.ScheduledJob: ( scheduled_at, job_id, type, - status, + _status, query, start_at, end_at, @@ -824,7 +824,9 @@ def scheduled_job(m): return [scheduled_job(m) for m in result] - def run_schedule(self, name: str, time: int, num: int) -> list[models.ScheduledJob]: + def run_schedule( + self, name: str, time: int, num: int | None = None + ) -> list[models.ScheduledJob]: """Execute the specified query. Args: @@ -837,8 +839,11 @@ def run_schedule(self, name: str, time: int, num: int) -> list[models.ScheduledJ """ results = self.api.run_schedule(name, time, num) - def scheduled_job(m): + def scheduled_job( + m: tuple[Any, str, datetime.datetime | None], + ) -> models.ScheduledJob: job_id, type, scheduled_at = m + assert scheduled_at is not None return models.ScheduledJob(self, scheduled_at, job_id, type, None) return [scheduled_job(m) for m in results] @@ -904,7 +909,7 @@ def results(self) -> list[models.Result]: """ results = self.api.list_result() - def result(m): + def result(m: tuple[str, str, None]) -> models.Result: name, url, organizations = m return models.Result(self, name, url, organizations) @@ -943,7 +948,7 @@ def users(self): """ results = self.api.list_users() - def user(m): + def user(m: tuple[str, None, None, str]) -> models.User: name, org, roles, email = m return models.User(self, name, org, roles, email) @@ -1011,7 +1016,7 @@ def close(self) -> None: def job_from_dict(client: Client, dd: dict[str, Any], **values: Any) -> models.Job: - d = dict() + d: dict[str, Any] = dict() d.update(dd) d.update(values) return models.Job( diff --git a/tdclient/connection.py b/tdclient/connection.py index 3ff539a..ecc3c11 100644 --- a/tdclient/connection.py +++ b/tdclient/connection.py @@ -23,7 +23,7 @@ def __init__( wait_callback: Callable[["Cursor"], None] | None = None, **kwargs: Any, ) -> None: - cursor_kwargs = dict() + cursor_kwargs: dict[str, Any] = dict() if type is not None: cursor_kwargs["type"] = type if db is not None: diff --git a/tdclient/connector_api.py b/tdclient/connector_api.py index c20db2c..d6b5fee 100644 --- a/tdclient/connector_api.py +++ b/tdclient/connector_api.py @@ -125,7 +125,7 @@ def connector_guess(self, job: dict[str, Any] | bytes) -> dict[str, Any]: self.raise_error("DataConnector configuration guess failed", res, body) return self.checked_json(body, []) - def connector_preview(self, job): + def connector_preview(self, job: dict[str, Any]) -> dict[str, Any]: """Show the preview of the Data Connector job. Args: @@ -135,14 +135,14 @@ def connector_preview(self, job): :class:`dict` """ headers = {"content-type": "application/json; charset=utf-8"} - payload = json.dumps(job).encode("utf-8") if isinstance(job, dict) else job + payload = json.dumps(job).encode("utf-8") with self.post("/v3/bulk_loads/preview", payload, headers=headers) as res: code, body = res.status, res.read() if code != 200: self.raise_error("DataConnector job preview failed", res, body) return self.checked_json(body, []) - def connector_issue(self, db, table, job): + def connector_issue(self, db: str, table: str, job: dict[str, Any]) -> str: """Create a Data Connector job. Args: @@ -167,7 +167,7 @@ def connector_issue(self, db, table, job): js = self.checked_json(body, ["job_id"]) return str(js["job_id"]) - def connector_list(self): + def connector_list(self) -> list[dict[str, Any]]: """Show the list of available Data Connector sessions. Returns: @@ -180,7 +180,14 @@ def connector_list(self): # cannot use `checked_json` since `GET /v3/bulk_loads` returns an array return json.loads(body.decode("utf-8")) - def connector_create(self, name, database, table, job, params=None): + def connector_create( + self, + name: str, + database: str, + table: str, + job: dict[str, Any], + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: """Create a Data Connector session. Args: @@ -230,7 +237,7 @@ def connector_create(self, name, database, table, job, params=None): ) return self.checked_json(body, []) - def connector_show(self, name): + def connector_show(self, name: str) -> dict[str, Any]: """Show a specific Data Connector session information. Args: @@ -247,7 +254,7 @@ def connector_show(self, name): ) return self.checked_json(body, []) - def connector_update(self, name, job): + def connector_update(self, name: str, job: dict[str, Any]) -> dict[str, Any]: """Update a specific Data Connector session. Args: @@ -273,7 +280,7 @@ def connector_update(self, name, job): ) return self.checked_json(body, []) - def connector_delete(self, name): + def connector_delete(self, name: str) -> dict[str, Any]: """Delete a Data Connector session. Args: @@ -290,7 +297,7 @@ def connector_delete(self, name): ) return self.checked_json(body, []) - def connector_history(self, name): + def connector_history(self, name: str) -> list[dict[str, Any]]: """Show the list of the executed jobs information for the Data Connector job. Args: @@ -309,7 +316,7 @@ def connector_history(self, name): ) return json.loads(body.decode("utf-8")) - def connector_run(self, name, **kwargs): + def connector_run(self, name: str, **kwargs: Any) -> dict[str, Any]: """Create a job to execute Data Connector session. Args: diff --git a/tdclient/cursor.py b/tdclient/cursor.py index d334f6b..03792ec 100644 --- a/tdclient/cursor.py +++ b/tdclient/cursor.py @@ -48,10 +48,11 @@ def close(self) -> None: def execute(self, query: str, args: dict[str, Any] | None = None) -> str | None: if args is not None: - if isinstance(args, dict): - query = query.format(**args) - else: - raise errors.NotSupportedError + if not isinstance(args, dict): # type: ignore[reportUnnecessaryIsInstance] + raise errors.NotSupportedError( + "args must be a dict for named placeholders" + ) + query = query.format(**args) self._executed = self._api.query(query, **self._query_kwargs) self._rows = None self._rownumber = 0 diff --git a/tdclient/database_api.py b/tdclient/database_api.py index 3e1964e..fd18e72 100644 --- a/tdclient/database_api.py +++ b/tdclient/database_api.py @@ -18,7 +18,7 @@ class DatabaseAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -45,7 +45,7 @@ def list_databases(self) -> dict[str, Any]: if code != 200: self.raise_error("List databases failed", res, body) js = self.checked_json(body, ["databases"]) - result = {} + result: dict[str, dict[str, Any]] = {} for m in js["databases"]: name = m.get("name") m = dict(m) diff --git a/tdclient/database_model.py b/tdclient/database_model.py index 64b3ac1..3bc463e 100644 --- a/tdclient/database_model.py +++ b/tdclient/database_model.py @@ -65,7 +65,7 @@ def tables(self) -> list["Table"]: assert self._tables is not None return self._tables - def create_log_table(self, name: str) -> "Table": + def create_log_table(self, name: str) -> bool: """ Args: name (str): name of new log table diff --git a/tdclient/import_api.py b/tdclient/import_api.py index 942525e..e9eaf86 100644 --- a/tdclient/import_api.py +++ b/tdclient/import_api.py @@ -31,7 +31,7 @@ def raise_error( ) -> None: ... def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: ... def _prepare_file( - self, file_like: FileLike, fmt: str, **kwargs: Any + self, file_like: FileLike, fmt: DataFormat, **kwargs: Any ) -> IO[bytes]: ... def import_data( @@ -74,7 +74,7 @@ def import_data( format=format, ) - kwargs = {} + kwargs: dict[str, Any] = {} with self.put(path, bytes_or_stream, size, **kwargs) as res: code, body = res.status, res.read() if code / 100 != 2: diff --git a/tdclient/job_api.py b/tdclient/job_api.py index 1f1643c..37dc658 100644 --- a/tdclient/job_api.py +++ b/tdclient/job_api.py @@ -29,8 +29,8 @@ class JobAPI: # Methods from API class def get( self, - url: str, - params: dict[str, Any] | bytes | None = None, + path: str, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... def post( @@ -78,9 +78,8 @@ def list_jobs( Returns: a list of :class:`dict` which represents a job """ - params = {} - if _from is not None: - params["from"] = str(_from) + params: dict[str, Any] = {} + params["from"] = str(_from) if to is not None: params["to"] = str(to) if status is not None: @@ -92,7 +91,7 @@ def list_jobs( if code != 200: self.raise_error("List jobs failed", res, body) js = self.checked_json(body, ["jobs"]) - jobs = [] + jobs: list[dict[str, Any]] = [] for m in js["jobs"]: if m.get("result") is not None and 0 < len(str(m["result"])): result = m["result"] @@ -221,7 +220,7 @@ def job_result(self, job_id: str) -> list[dict[str, Any]]: Returns: Job result in :class:`list` """ - result = [] + result: list[dict[str, Any]] = [] for row in self.job_result_format_each(job_id, "msgpack"): result.append(row) return result @@ -252,7 +251,7 @@ def job_result_format( Returns: The query result of the specified job in. """ - result = [] + result: list[dict[str, Any]] = [] for row in self.job_result_format_each(job_id, format, header): result.append(row) return result @@ -299,7 +298,9 @@ def job_result_format_each( self.download_job_result(job_id, path, num_threads) with gzip.GzipFile(path, "rb") as f: unpacker = msgpack.Unpacker( - f, raw=False, max_buffer_size=1000 * 1024**2 + f, # type: ignore[arg-type] + raw=False, + max_buffer_size=1000 * 1024**2, # type: ignore[arg-type] ) for row in unpacker: yield row @@ -345,13 +346,17 @@ def download_job_result(self, job_id: str, path: str, num_threads: int = 4) -> b format="msgpack.gz", ) - def get_chunk(url, start, end): + def get_chunk( + url: str, start: int, end: int + ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: chunk_headers = {"Range": f"bytes={start}-{end}"} response = self.get(url, headers=chunk_headers) return response - def download_chunk(url, start, end, index, file_name): + def download_chunk( + url: str, start: int, end: int, index: int, file_name: str + ) -> bool: with get_chunk(url, start, end) as response: if response.status == 206: # Partial content (range supported) with open(f"{file_name}.part{index}", "wb") as f: @@ -364,7 +369,7 @@ def download_chunk(url, start, end, index, file_name): ) return False - def combine_chunks(file_name, total_parts): + def combine_chunks(file_name: str, total_parts: int) -> None: with open(file_name, "wb") as final_file: for i in range(total_parts): with open(f"{file_name}.part{i}", "rb") as part_file: @@ -372,8 +377,12 @@ def combine_chunks(file_name, total_parts): os.remove(f"{file_name}.part{i}") def download_file_multithreaded( - url, file_name, file_size, num_threads=4, chunk_size=100 * 1024**2 - ): + url: str, + file_name: str, + file_size: int, + num_threads: int = 4, + chunk_size: int = 100 * 1024**2, + ) -> None: start = 0 part_index = 0 diff --git a/tdclient/job_model.py b/tdclient/job_model.py index 8c21042..8bd4a20 100644 --- a/tdclient/job_model.py +++ b/tdclient/job_model.py @@ -9,6 +9,7 @@ if TYPE_CHECKING: from tdclient.client import Client + from tdclient.types import ResultFormat class Schema: @@ -223,7 +224,7 @@ def wait( raise RuntimeError("timeout") # TODO: throw proper error self.update() - def kill(self) -> str: + def kill(self) -> str | None: """Kill the job Returns: @@ -269,7 +270,7 @@ def result(self) -> Iterator[dict[str, Any]]: yield row # type: ignore[misc] def result_format( - self, fmt: str, store_tmpfile: bool = False, num_threads: int = 4 + self, fmt: "ResultFormat", store_tmpfile: bool = False, num_threads: int = 4 ) -> Iterator[dict[str, Any]]: """ Args: diff --git a/tdclient/model.py b/tdclient/model.py index e208e22..41ccb01 100644 --- a/tdclient/model.py +++ b/tdclient/model.py @@ -1,12 +1,17 @@ #!/usr/bin/env python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from tdclient.client import Client + class Model: - def __init__(self, client): + def __init__(self, client: "Client") -> None: self._client = client @property - def client(self): + def client(self) -> "Client": """ Returns: a :class:`tdclient.client.Client` instance """ diff --git a/tdclient/result_api.py b/tdclient/result_api.py index 51614af..6dc328c 100644 --- a/tdclient/result_api.py +++ b/tdclient/result_api.py @@ -19,7 +19,7 @@ class ResultAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/result_model.py b/tdclient/result_model.py index 34f2c50..a56c03f 100644 --- a/tdclient/result_model.py +++ b/tdclient/result_model.py @@ -11,7 +11,9 @@ class Result(Model): """Result on Treasure Data Service""" - def __init__(self, client: "Client", name: str, url: str, org_name: str) -> None: + def __init__( + self, client: "Client", name: str, url: str, org_name: str | None + ) -> None: super().__init__(client) self._name = name self._url = url @@ -28,6 +30,6 @@ def url(self) -> str: return self._url @property - def org_name(self) -> str: + def org_name(self) -> str | None: """str: organization name""" return self._org_name diff --git a/tdclient/schedule_api.py b/tdclient/schedule_api.py index 266594c..9407496 100644 --- a/tdclient/schedule_api.py +++ b/tdclient/schedule_api.py @@ -20,7 +20,7 @@ class ScheduleAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -188,9 +188,8 @@ def history( Returns: dict: History of the scheduled query. """ - params = {} - if _from is not None: - params["from"] = str(_from) + params: dict[str, Any] = {} + params["from"] = str(_from) if to is not None: params["to"] = str(to) with self.get( @@ -213,7 +212,6 @@ def run_schedule( time (int): Time in Unix epoch format that would be set as TD_SCHEDULED_TIME num (int, optional): Indicates how many times the query will be executed. Value should be 9 or less. - Default: 1 Returns: list of tuple: [(job_id:int, type:str, scheduled_at:str)] """ @@ -231,14 +229,14 @@ def run_schedule( return [job_to_tuple(m) for m in js["jobs"]] -def job_to_tuple(m): +def job_to_tuple(m: dict[str, Any]) -> tuple[str | None, str, datetime.datetime | None]: job_id = m.get("job_id") scheduled_at = parse_date(get_or_else(m, "scheduled_at", "1970-01-01T00:00:00Z")) t = m.get("type", "?") return job_id, t, scheduled_at -def schedule_to_tuple(m): +def schedule_to_tuple(m: dict[str, Any]) -> dict[str, Any]: m = dict(m) if "timezone" not in m: m["timezone"] = "UTC" @@ -247,7 +245,20 @@ def schedule_to_tuple(m): return m -def history_to_tuple(m): +def history_to_tuple( + m: dict[str, Any], +) -> tuple[ + datetime.datetime | None, + Any, + str, + Any, + Any, + datetime.datetime | None, + datetime.datetime | None, + Any, + Any, + Any, +]: job_id = m.get("job_id") t = m.get("type", "?") database = m.get("database") diff --git a/tdclient/schedule_model.py b/tdclient/schedule_model.py index 4b54ed3..06c70b4 100644 --- a/tdclient/schedule_model.py +++ b/tdclient/schedule_model.py @@ -154,4 +154,5 @@ def run(self, time: int, num: int | None = None) -> list[ScheduledJob]: Returns: [:class:`tdclient.models.ScheduledJob`] """ + assert self.name is not None, "schedule name is required" return self._client.run_schedule(self.name, time, num) diff --git a/tdclient/table_api.py b/tdclient/table_api.py index ab362f3..26d09f9 100644 --- a/tdclient/table_api.py +++ b/tdclient/table_api.py @@ -20,7 +20,7 @@ class TableAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... @@ -71,7 +71,7 @@ def list_tables(self, db: str) -> dict[str, Any]: if code != 200: self.raise_error("List tables failed", res, body) js = self.checked_json(body, ["tables"]) - result = {} + result: dict[str, dict[str, Any]] = {} for m in js["tables"]: m = dict(m) m["type"] = m.get("type", "?") @@ -236,8 +236,8 @@ def tail( if code != 200: self.raise_error("Tail table failed", res, "") - unpacker = msgpack.Unpacker(res, raw=False) - result = [] + unpacker = msgpack.Unpacker(res, raw=False) # type: ignore[arg-type] + result: list[dict[str, Any]] = [] for row in unpacker: result.append(row) diff --git a/tdclient/table_model.py b/tdclient/table_model.py index 2a5248f..eb6ff1c 100644 --- a/tdclient/table_model.py +++ b/tdclient/table_model.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import datetime -from typing import TYPE_CHECKING, Any +import warnings +from typing import TYPE_CHECKING, Any, cast from tdclient.model import Model -from tdclient.types import DataFormat, FileLike +from tdclient.types import BytesOrStream, DataFormat, ExportParams, FileLike if TYPE_CHECKING: from tdclient.database_model import Database @@ -158,12 +159,18 @@ def tail( the contents of the table in reverse order based on the registered time (last data first). """ - return self._client.tail(self._db_name, self._table_name, count, to, _from) + if to is not None or _from is not None: + warnings.warn( + "'to' and '_from' parameters are deprecated and ignored", + DeprecationWarning, + stacklevel=2, + ) + return self._client.tail(self._db_name, self._table_name, count, None, None) def import_data( self, format: DataFormat, - bytes_or_stream: FileLike, + bytes_or_stream: BytesOrStream, size: int, unique_id: str | None = None, ) -> float: @@ -241,7 +248,7 @@ def export_data(self, storage_type: str, **kwargs: Any) -> "Job": :class:`tdclient.models.Job` """ return self._client.export_data( - self._db_name, self._table_name, storage_type, kwargs + self._db_name, self._table_name, storage_type, cast("ExportParams", kwargs) ) @property diff --git a/tdclient/user_api.py b/tdclient/user_api.py index e420a0f..16a3ae9 100644 --- a/tdclient/user_api.py +++ b/tdclient/user_api.py @@ -13,7 +13,7 @@ class UserAPI: def get( self, path: str, - params: dict[str, Any] | bytes | None = None, + params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, **kwargs: Any, ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ... diff --git a/tdclient/user_model.py b/tdclient/user_model.py index b7011f8..143ce88 100644 --- a/tdclient/user_model.py +++ b/tdclient/user_model.py @@ -15,8 +15,8 @@ def __init__( self, client: "Client", name: str, - org_name: str, - role_names: list[str], + org_name: str | None, + role_names: list[str] | None, email: str, **kwargs: Any, ) -> None: @@ -34,14 +34,14 @@ def name(self) -> str: return self._name @property - def org_name(self) -> str: + def org_name(self) -> str | None: """ Returns: organization name """ return self._org_name @property - def role_names(self) -> list[str]: + def role_names(self) -> list[str] | None: """ TODO: add docstring """ diff --git a/tdclient/util.py b/tdclient/util.py index 2504312..abed4e8 100644 --- a/tdclient/util.py +++ b/tdclient/util.py @@ -299,10 +299,10 @@ def normalized_msgpack(value: Any) -> Any: Normalized value """ if isinstance(value, list | tuple): - return [normalized_msgpack(v) for v in value] + return [normalized_msgpack(v) for v in value] # type: ignore[reportUnknownVariableType] elif isinstance(value, dict): return dict( - [(normalized_msgpack(k), normalized_msgpack(v)) for (k, v) in value.items()] + [(normalized_msgpack(k), normalized_msgpack(v)) for (k, v) in value.items()] # type: ignore[reportUnknownVariableType] ) if isinstance(value, int): diff --git a/uv.lock b/uv.lock index 6308a06..1486808 100644 --- a/uv.lock +++ b/uv.lock @@ -497,6 +497,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, ] +[[package]] +name = "msgpack-types" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/26/a15707f2af5681333cd598724bedd1948844ac2af45eafc4175af0671a8d/msgpack_types-0.5.0.tar.gz", hash = "sha256:aebd1b8da23f8f9966d66ebb1a43bd261b95751c6a267bd21a124d2ccac84201", size = 6702, upload-time = "2024-09-21T13:55:05.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/cd9d2b0ef506f6164cd81d4e92e408095041f28523d751b9f7dabdc244eb/msgpack_types-0.5.0-py3-none-any.whl", hash = "sha256:8b633ed75e495a555fa0615843de559a74b1d176828d59bb393d266e51f6bda7", size = 8182, upload-time = "2024-09-21T13:55:04.232Z" }, +] + [[package]] name = "nodeenv" version = "1.9.1" @@ -885,11 +898,6 @@ dependencies = [ ] [package.optional-dependencies] -dev = [ - { name = "pyright" }, - { name = "ruff" }, - { name = "tox" }, -] docs = [ { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -902,22 +910,37 @@ test = [ { name = "pyyaml" }, ] +[package.dev-dependencies] +dev = [ + { name = "msgpack-types" }, + { name = "pyright" }, + { name = "ruff" }, + { name = "tox" }, + { name = "types-certifi" }, +] + [package.metadata] requires-dist = [ { name = "coverage", extras = ["toml"], marker = "extra == 'test'" }, { name = "coveralls", marker = "extra == 'test'", specifier = ">=1.1,<5.0" }, { name = "msgpack", specifier = ">=0.6.2" }, - { name = "pyright", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3" }, { name = "python-dateutil" }, { name = "pyyaml", marker = "extra == 'test'" }, - { name = "ruff", marker = "extra == 'dev'" }, { name = "sphinx", marker = "extra == 'docs'" }, { name = "sphinx-rtd-theme", marker = "extra == 'docs'" }, - { name = "tox", marker = "extra == 'dev'", specifier = ">=4" }, { name = "urllib3" }, ] -provides-extras = ["dev", "docs", "test"] +provides-extras = ["docs", "test"] + +[package.metadata.requires-dev] +dev = [ + { name = "msgpack-types", specifier = ">=0.5.0" }, + { name = "pyright" }, + { name = "ruff" }, + { name = "tox", specifier = ">=4" }, + { name = "types-certifi" }, +] [[package]] name = "tomli" @@ -990,6 +1013,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/cc/e09c0d663a004945f82beecd4f147053567910479314e8d01ba71e5d5dea/tox-4.32.0-py3-none-any.whl", hash = "sha256:451e81dc02ba8d1ed20efd52ee409641ae4b5d5830e008af10fe8823ef1bd551", size = 175905, upload-time = "2025-10-24T18:03:36.337Z" }, ] +[[package]] +name = "types-certifi" +version = "2021.10.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/68/943c3aeaf14624712a0357c4a67814dba5cea36d194f5c764dad7959a00c/types-certifi-2021.10.8.3.tar.gz", hash = "sha256:72cf7798d165bc0b76e1c10dd1ea3097c7063c42c21d664523b928e88b554a4f", size = 2095, upload-time = "2022-06-09T15:19:05.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/63/2463d89481e811f007b0e1cd0a91e52e141b47f9de724d20db7b861dcfec/types_certifi-2021.10.8.3-py3-none-any.whl", hash = "sha256:b2d1e325e69f71f7c78e5943d410e650b4707bb0ef32e4ddf3da37f54176e88a", size = 2136, upload-time = "2022-06-09T15:19:03.127Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"