diff --git a/.mise.toml b/.mise.toml index b31659e..f27f070 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,4 +1,4 @@ [tools] python="3.11" -poetry="2.2" +poetry="2.2.1" java="liberica-1.8.0" diff --git a/.tool-versions b/.tool-versions index b23db8d..3a495f2 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ python 3.11.14 -poetry 2.2.0 +poetry 2.2.1 java liberica-1.8.0 diff --git a/poetry.lock b/poetry.lock index 68fc5b2..7b1987a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "argcomplete" @@ -2533,13 +2533,6 @@ optional = false python-versions = ">=3.8" groups = ["dev", "test"] files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, @@ -2970,18 +2963,18 @@ files = [ [[package]] name = "werkzeug" -version = "3.0.6" +version = "3.1.5" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev", "test"] files = [ - {file = "werkzeug-3.0.6-py3-none-any.whl", hash = "sha256:1bc0c2310d2fbb07b1dd1105eba2f7af72f322e1e455f2f93c993bee8c8a5f17"}, - {file = "werkzeug-3.0.6.tar.gz", hash = "sha256:a8dd59d4de28ca70471a34cba79bed5f7ef2e036a76b3ab0835474246eb41f8d"}, + {file = "werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc"}, + {file = "werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67"}, ] [package.dependencies] -MarkupSafe = ">=2.1.1" +markupsafe = ">=2.1.1" [package.extras] watchdog = ["watchdog (>=2.3)"] @@ -3127,4 +3120,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.12" -content-hash = "93fd40fa4a8924c95ee4aa46f75d693294c32670721b31f25576e3fe60236013" +content-hash = "08ea1eedf25a896fdc21f03d04f4403d47d655fc90eb5eb310ff7cde7e3b7a6d" diff --git a/pyproject.toml b/pyproject.toml index 3b9a736..4497211 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ faker = "18.11.1" behave = "1.3.3" coverage = "7.11.0" moto = {extras = ["s3"], version = "4.0.13"} -Werkzeug = "3.0.6" # Dependency of moto which needs 3.0.6 for security vuln mitigation +Werkzeug = "3.1.5" pytest = "8.4.2" pytest-lazy-fixtures = "1.4.0" # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported xlsx2csv = "0.8.2" diff --git a/src/dve/core_engine/models.py b/src/dve/core_engine/models.py index 75a14ed..09fcbb3 100644 --- a/src/dve/core_engine/models.py +++ b/src/dve/core_engine/models.py @@ -8,7 +8,7 @@ import os import uuid from collections.abc import MutableMapping -from pathlib import Path, PurePath +from pathlib import Path from typing import Any, Optional from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator @@ -64,16 +64,9 @@ class SubmissionInfo(AuditRecord): datetime_received: Optional[dt.datetime] = None # type: ignore """The datetime the file was received.""" - @validator("file_name") - def _ensure_metadata_extension_removed(cls, filename): # pylint: disable=no-self-argument - path = PurePath(filename) - return path.stem - @validator("file_extension") def _ensure_just_file_stem(cls, extension: str): # pylint: disable=no-self-argument - if "." in extension: - return extension.split(".")[-1] - return extension + return extension.rsplit(".", 1)[-1] @property def file_name_with_ext(self): diff --git a/tests/features/steps/steps_pipeline.py b/tests/features/steps/steps_pipeline.py index b047905..fa1e848 100644 --- a/tests/features/steps/steps_pipeline.py +++ b/tests/features/steps/steps_pipeline.py @@ -219,7 +219,7 @@ def submit_file_for_processing(context: Context, dataset: str, file_name: str): sub_info = { "submission_id": uuid4().hex, "dataset_id": dataset, - "file_name": file_name, + "file_name": file_name.rsplit(".", 1)[0], "file_extension": Path(file_name).suffix, "reporting_period_start": "2025-11-01 00:00:00", "reporting_period_end": "2025-11-30 23:59:59" diff --git a/tests/test_core_engine/test_models.py b/tests/test_core_engine/test_models.py index 87c8f9d..e187de1 100644 --- a/tests/test_core_engine/test_models.py +++ b/tests/test_core_engine/test_models.py @@ -18,7 +18,7 @@ "submitted": { "submission_id": CONSTANT_SUBMISSION_ID, "dataset_id": "test0", - "file_name": "my_file.csv", + "file_name": "my_file", "file_extension": "csv", }, "expected": { @@ -35,13 +35,13 @@ "submitted": { "submission_id": CONSTANT_SUBMISSION_ID, "dataset_id": "test1", - "file_name": "my_file.csv.csv", - "file_extension": "csv", + "file_name": "my_file", + "file_extension": ".csv.csv", }, "expected": { "submission_id": CONSTANT_SUBMISSION_ID, "dataset_id": "test1", - "file_name": "my_file.csv", + "file_name": "my_file", "file_extension": "csv", }, }, @@ -52,7 +52,7 @@ "submitted": { "submission_id": CONSTANT_SUBMISSION_ID, "dataset_id": "test2", - "file_name": "my_file.xml", + "file_name": "my_file", "file_extension": "csv.csv.xml", }, "expected": { @@ -63,6 +63,40 @@ }, }, ), + # submission with multiple file extensions + ( + { + "submitted": { + "submission_id": CONSTANT_SUBMISSION_ID, + "dataset_id": "test2", + "file_name": "my.file.perfect", + "file_extension": "csv", + }, + "expected": { + "submission_id": CONSTANT_SUBMISSION_ID, + "dataset_id": "test2", + "file_name": "my.file.perfect", + "file_extension": "csv", + }, + }, + ), + # submission with multiple file extensions + ( + { + "submitted": { + "submission_id": CONSTANT_SUBMISSION_ID, + "dataset_id": "test2", + "file_name": "m_y%fil\ne.perfect", + "file_extension": "csv", + }, + "expected": { + "submission_id": CONSTANT_SUBMISSION_ID, + "dataset_id": "test2", + "file_name": "m_y%fil\ne.perfect", + "file_extension": "csv", + }, + }, + ), ], ) def test_submission_info( # pylint: disable=missing-function-docstring