diff --git a/mergin/cli.py b/mergin/cli.py index c20beb9..ed9ad7c 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -248,16 +248,20 @@ def list_projects(ctx, name, namespace, order_params): @click.argument("project") @click.argument("directory", type=click.Path(), required=False) @click.option("--version", default=None, help="Version of project to download") +@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'") +@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'") @click.pass_context -def download(ctx, project, directory, version): +def download(ctx, project, directory, version, include, exclude): """Download last version of mergin project.""" mc = ctx.obj["client"] if mc is None: return + if include and exclude: + raise click.UsageError("--include and --exclude cannot be used together") directory = directory or os.path.basename(project) click.echo("Downloading into {}".format(directory)) try: - job = download_project_async(mc, project, directory, version) + job = download_project_async(mc, project, directory, version, include=include, exclude=exclude) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 1555051..efe1c8a 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -63,6 +63,7 @@ from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable from .utils import ( DateTimeEncoder, + filter_files, get_versions_with_file_changes, int_version, is_version_acceptable, @@ -902,7 +903,7 @@ def project_versions(self, project_path, since=1, to=None): filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions)) return filtered_versions - def download_project(self, project_path, directory, version=None): + def download_project(self, project_path, directory, version=None, include=None, exclude=None): """ Download project into given directory. If version is not specified, latest version is downloaded @@ -914,8 +915,16 @@ def download_project(self, project_path, directory, version=None): :param version: Project version to download, e.g. v42 :type version: String + + :param include: Optional list of glob patterns (matched against each file's project path, e.g. + "media/*" or "*.gpkg") - only matching files are downloaded. + :type include: List[String] + + :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive + with include. + :type exclude: List[String] """ - job = download_project_async(self, project_path, directory, version) + job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude) download_project_wait(job) download_project_finalize(job) @@ -1158,6 +1167,10 @@ def project_status(self, directory): server_info = self.project_info(mp.project_full_name(), since=mp.version()) pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version")) + # on a sparse checkout, don't report excluded files as pending server changes - + # they were never meant to be pulled in the first place + file_filter = mp.file_filter() + pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()} push_changes = mp.get_push_changes() push_changes_summary = mp.get_list_of_push_changes(push_changes) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..971188d 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, filter_files, is_path_in_scope, save_to_file from typing import List, Optional # status = download_project_async(...) @@ -242,12 +242,18 @@ def _cleanup_failed_download(mergin_project: MerginProject = None): return dest_path -def download_project_async(mc, project_path, directory, project_version=None): +def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None): """ Starts project download in background and returns handle to the pending project download. Using that object it is possible to watch progress or cancel the ongoing work. + + `include`/`exclude` are optional lists of glob patterns (matched against each file's project + path, e.g. "media/*" or "*.gpkg") to only download a subset of the project's files. They are + mutually exclusive. """ + if include and exclude: + raise ClientError("Cannot use both include and exclude filters at the same time") if "/" not in project_path: raise ClientError("Project name needs to be fully qualified, e.g. /") if os.path.exists(directory): @@ -276,6 +282,12 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"got project info. version {version}") + # keep only the files matching the filter (if any) + project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude) + # persisted once since it must never change again for this checkout + if include or exclude: + mp.write_file_filter({"include": include, "exclude": exclude}) + # prepare download update_tasks = [] # stuff to do at the end of download for file in project_info["files"]: @@ -525,6 +537,9 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: mp.log.info("--- pull aborted") raise + file_filter = mp.file_filter() + delta.changes = [c for c in delta.changes if is_path_in_scope(c.path, **file_filter)] + mp.log.info(f"got project versions: local version {local_version} / server version {server_version}") if local_version == server_version: @@ -748,6 +763,9 @@ def pull_project_finalize(job: PullJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply pull actions: " + str(e)) + # keep only in-scope files in the metadata we're about to persist + job.project_info["files"] = filter_files(job.project_info["files"], **job.mp.file_filter()) + job.mp.update_metadata(job.project_info) if job.mp.has_unfinished_pull(): diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..155d270 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, filter_files POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -458,6 +458,9 @@ def push_project_finalize(job: UploadJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise err + # keep only in-scope files in the metadata we're about to persist + job.server_resp["files"] = filter_files(job.server_resp["files"], **job.mp.file_filter()) + job.mp.update_metadata(job.server_resp) try: job.mp.apply_push_changes(asdict(job.changes)) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..a8da37f 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -24,6 +24,8 @@ unique_path_name, conflicted_copy_file_name, edit_conflict_file_name, + filter_files, + is_path_in_scope, ) from .local_changes import FileChange @@ -211,9 +213,28 @@ def version(self) -> str: return self._metadata["version"] def files(self) -> list: - """Returns project's list of files (each file being a dictionary)""" + """Returns project's list of files (each file being a dictionary), scoped to this + project's file_filter() if one is set (sparse checkout).""" self._read_metadata() - return self._metadata["files"] + return filter_files(self._metadata["files"], **self.file_filter()) + + def file_filter(self) -> dict: + """ + Returns the include/exclude file filter this project was downloaded with, as a dict + with "include" and "exclude" keys. Stored in its own file (.mergin/file_filter.json) + """ + filter_file = self.fpath_meta("file_filter.json") + if not os.path.exists(filter_file): + return {"include": None, "exclude": None} + with open(filter_file, "r") as f: + return json.load(f) + + def write_file_filter(self, file_filter: dict) -> None: + """ + Persists the include/exclude file filter this project was downloaded with. + """ + with open(self.fpath_meta("file_filter.json"), "w") as f: + json.dump(file_filter, f, indent=2) @property def metadata(self) -> dict: @@ -304,10 +325,12 @@ def ignore_file(self, file): def inspect_files(self): """ Inspect files in project directory and return metadata. + Only files matching this project's file_filter() are included. :returns: metadata for files in project directory in server required format :rtype: list[dict] """ + file_filter = self.file_filter() files_meta = [] for root, dirs, files in os.walk(self.dir, topdown=True): dirs[:] = [d for d in dirs if d not in [".mergin"]] @@ -318,6 +341,8 @@ def inspect_files(self): abs_path = os.path.abspath(os.path.join(root, file)) rel_path = os.path.relpath(abs_path, start=self.dir) proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path + if not is_path_in_scope(proj_path, **file_filter): + continue files_meta.append( { "path": proj_path, diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index bd987d0..93e9153 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1,4 +1,5 @@ import hashlib +import inspect import json import logging import os @@ -1153,6 +1154,270 @@ def test_download_versions(mc): mc.download_project(project, project_dir_v3, "v3") +def test_download_project_with_filter(mc): + """Test downloading a project with include/exclude filters, and that they're mutually exclusive.""" + test_project = "test_download_project_filter" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + include_dir = os.path.join(TMP_DIR, test_project + "_include") + exclude_dir = os.path.join(TMP_DIR, test_project + "_exclude") + conflict_dir = os.path.join(TMP_DIR, test_project + "_conflict") + + cleanup(mc, project, [project_dir, include_dir, exclude_dir, conflict_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + # include filter: only matching files are fetched + mc.download_project(project, include_dir, include=["*.gpkg"]) + + downloaded_files = set() + for root, _, files in os.walk(include_dir): + if ".mergin" in root.split(os.sep): + continue + for f in files: + rel = os.path.relpath(os.path.join(root, f), include_dir) + downloaded_files.add(rel.replace(os.sep, "/")) + + assert downloaded_files + assert all(f.endswith(".gpkg") for f in downloaded_files) + assert "test.qgs" not in downloaded_files + assert "test.txt" not in downloaded_files + + mp = MerginProject(include_dir) + assert all(f["path"].endswith(".gpkg") for f in mp.files()) + assert mp.file_filter() == {"include": ["*.gpkg"], "exclude": None} + + # exclude filter: matching files are skipped + mc.download_project(project, exclude_dir, exclude=["test_dir/*"]) + + assert os.path.exists(os.path.join(exclude_dir, "base.gpkg")) + assert not os.path.exists(os.path.join(exclude_dir, "test_dir")) + + mp = MerginProject(exclude_dir) + assert not any(f["path"].startswith("test_dir/") for f in mp.files()) + assert mp.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + + # include and exclude cannot be combined + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + mc.download_project(project, conflict_dir, include=["*.gpkg"], exclude=["*.txt"]) + assert not os.path.exists(conflict_dir) + + +def test_sparse_checkout_filter_is_immutable(mc): + """Once a directory has been downloaded with a filter, there is no way to change that + filter in place - the only way to get a different filter is to check out into a fresh + directory. + """ + test_project = "test_sparse_checkout_immutable" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + original_filter = MerginProject(sparse_dir).file_filter() + + # trying to download again into the same directory - with the same or a different filter - + # must fail without touching anything, regardless of what filter (if any) is requested + for kwargs in ({"exclude": ["test_dir/*"]}, {"include": ["*.gpkg"]}, {}): + with pytest.raises(ClientError, match="Project directory already exists"): + mc.download_project(project, sparse_dir, **kwargs) + + assert MerginProject(sparse_dir).file_filter() == original_filter + + # pull_project()/push_project() take no filter arguments at all - there is no API through + # which a different filter could be supplied for an existing checkout + assert "include" not in inspect.signature(mc.pull_project).parameters + assert "include" not in inspect.signature(mc.push_project).parameters + + +def test_pull_on_sparse_checkout(mc): + """A sparse checkout keeps respecting its filter across subsequent pulls: excluded files + stay ignored even as the server moves ahead, and a genuine local-vs-server conflict on an + included file is still detected and resolved correctly. + """ + test_project = "test_pull_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + sparse_version_before = MerginProject(sparse_dir).version() + + # change an excluded file on the server, by pushing from the reference (full) checkout + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change that only affects an excluded file") + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != sparse_version_before + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # now a genuine conflict: edit an included file locally, but don't push it yet + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + + # meanwhile a conflicting edit to the same file gets pushed from the reference (full) checkout + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != mp_sparse.version() + + # pulling the sparse checkout now must detect the conflict (not silently drop the local + # edit, not crash, and not leave the sparse checkout in a broken state) + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) # filter still respected + conflict_files = [f for f in os.listdir(sparse_dir) if "conflicted copy" in f] + assert conflict_files, "expected a conflicted copy of base.gpkg to be created" + + +def test_push_on_sparse_checkout(mc): + """Pushing from a sparse checkout must never touch the files it never downloaded: not a + no-op push (must not delete excluded files it's not tracking), not a real push of an + included file's edit (must not wipe the persisted filter from local metadata. + """ + test_project = "test_push_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # no local edits - must be a no-op, not a deletion of excluded files + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # a real push of an included file's edit must not lose the persisted filter afterwards + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # manually add a file outside the filter's scope - it must not get pushed either + os.makedirs(os.path.join(sparse_dir, "test_dir"), exist_ok=True) + with open(os.path.join(sparse_dir, "test_dir", "new_stray.txt"), "w") as f: + f.write("should not be pushed") + + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/new_stray.txt" not in server_files + + # a delete-only push goes through a different shortcut code path server-side + # it must also keep the filter intact and leave excluded files alone + # old_metadata.json already present on server and not affected by filter + os.remove(os.path.join(sparse_dir, "old_metadata.json")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "old_metadata.json" not in server_files + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + +def test_sparse_checkout_pull_push_v1_api(mc): + """Same filter-persistence guarantees as test_pull_on_sparse_checkout/test_push_on_sparse_checkout, + but forcing the legacy v1 pull/push code paths. + """ + server_features = mc.server_features() + mc._server_features = {"v2_pull_enabled": False, "v2_push_enabled": False} + + test_project = "test_sparse_checkout_v1_api" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change an excluded file on the server - a v1 pull must still skip it + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file, v1 api") + mc.push_project(project_dir) + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == mc.project_info(project)["version"] + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # a real edit to an included file, pushed via v1 - filter must survive it + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + mc._server_features = server_features + + +def test_project_status_on_sparse_checkout(mc): + """`status` must not report excluded files as pending server changes - + they were deliberately never meant to be pulled, so they shouldn't show up as if a pull + were needed to fetch them. + """ + test_project = "test_status_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change both an excluded and an included file on the server + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file") + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + + pull_changes, _, _ = mc.project_status(sparse_dir) + + changed_paths = {f["path"] for files in pull_changes.values() for f in files} + assert "test_dir/test2.txt" not in changed_paths + # a real, included change must still be reported + assert "base.gpkg" in changed_paths + + def test_paginated_project_list(mc): """Test the new endpoint for projects list with pagination, ordering etc.""" test_projects = dict() diff --git a/mergin/test/test_mergin_project.py b/mergin/test/test_mergin_project.py index 97fe554..b6879ea 100644 --- a/mergin/test/test_mergin_project.py +++ b/mergin/test/test_mergin_project.py @@ -186,6 +186,7 @@ def test_get_local_delta(): # Mock files() to return origin info for version lookup mp.files = lambda: [] mp.inspect_files = lambda: [] # Dummy return + mp.file_filter = lambda: {"include": None, "exclude": None} # no sparse checkout filter in this test # check if geopackage is updated (is_open) but missing - geodiff lib error, than updated file is reported mock_changes = { diff --git a/mergin/test/test_utils.py b/mergin/test/test_utils.py new file mode 100644 index 0000000..481fa49 --- /dev/null +++ b/mergin/test/test_utils.py @@ -0,0 +1,80 @@ +import pytest + +from ..utils import is_path_in_scope, filter_files +from ..common import ClientError + + +@pytest.mark.parametrize( + "path, include, exclude, expected", + [ + # no filter at all -> everything kept + pytest.param("anything/at/all.gpkg", None, None, True, id="no-filter"), + pytest.param("data.gpkg", [], None, True, id="empty-include-list"), + pytest.param("data.gpkg", None, [], True, id="empty-exclude-list"), + # basic include/exclude + pytest.param("data.gpkg", ["*.gpkg"], None, True, id="include-match"), + pytest.param("data.txt", ["*.gpkg"], None, False, id="include-no-match"), + pytest.param("media/photo.jpg", None, ["media/*"], False, id="exclude-match"), + pytest.param("data.gpkg", None, ["media/*"], True, id="exclude-no-match"), + # subfolders: fnmatch's '*' crosses '/', so it reaches arbitrarily deep + pytest.param("media/photo.jpg", None, ["media/*"], False, id="subfolder-direct-child"), + pytest.param("media/sub/deep/photo.jpg", None, ["media/*"], False, id="subfolder-deeply-nested"), + pytest.param("layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-root"), + pytest.param("data/nested/deep/layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-any-depth"), + # patterns still anchor to the *full* path, not just the basename + pytest.param("nested/media/photo.jpg", None, ["media/*"], True, id="not-anchored-to-basename-kept"), + pytest.param("nested/media/photo.jpg", None, ["*/media/*"], False, id="leading-star-catches-nested-media"), + pytest.param("media/photo.jpg", None, ["*/media/*"], True, id="leading-star-misses-root-level-media"), + # case sensitivity: fnmatchcase, not fnmatch - always case-sensitive, any OS + pytest.param("data.GPKG", ["*.gpkg"], None, False, id="case-mismatch-in-path"), + pytest.param("data.gpkg", ["*.GPKG"], None, False, id="case-mismatch-in-pattern"), + pytest.param("Media/photo.jpg", None, ["media/*"], True, id="case-mismatch-in-directory-kept"), + # lists of patterns: a path matches if it matches ANY pattern in the list (OR) + pytest.param("data.gpkg", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-1st-matches"), + pytest.param("map.qgz", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-2nd-matches"), + pytest.param("project.qgs", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-3rd-matches"), + pytest.param("readme.txt", ["*.gpkg", "*.qgz", "project.qgs"], None, False, id="include-list-none-match"), + pytest.param("media/photo.jpg", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-1st-matches"), + pytest.param("scratch.tmp", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-2nd-matches"), + pytest.param("data.gpkg-wal", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-3rd-matches"), + pytest.param("data.gpkg", None, ["media/*", "*.tmp", "*-wal"], True, id="exclude-list-none-match"), + ], +) +def test_is_path_in_scope(path, include, exclude, expected): + assert is_path_in_scope(path, include=include, exclude=exclude) is expected + + +@pytest.mark.parametrize( + "include, exclude, expected_paths", + [ + pytest.param(None, None, {"a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"}, id="no-filter"), + pytest.param(["*.gpkg", "*.qgz"], None, {"a.gpkg", "b.qgz", "media/d.gpkg"}, id="include-list"), + pytest.param(None, ["media/*"], {"a.gpkg", "b.qgz", "c.txt"}, id="exclude-subfolder"), + ], +) +def test_filter_files(include, exclude, expected_paths): + files = [{"path": p} for p in ["a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"]] + result = filter_files(files, include=include, exclude=exclude) + assert {f["path"] for f in result} == expected_paths + + +def test_filter_files_keeps_matching_dicts_as_is(): + """filter_files() passes matching dicts through unchanged""" + files = [ + {"path": "project.gpkg", "size": 100}, + {"path": "media/photo.jpg", "size": 999}, + ] + + result = filter_files(files, exclude=["media/*"]) + + assert result == [{"path": "project.gpkg", "size": 100}] + assert result[0] is files[0] + + +def test_filter_files_raises_on_mutually_exclusive_args(): + """Unlike is_path_in_scope, filter_files() is the validating entry point and + enforces that include/exclude are mutually exclusive. + """ + files = [{"path": "a.gpkg"}] + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + filter_files(files, include=["*.gpkg"], exclude=["*.txt"]) diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..85c2cca 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -2,13 +2,14 @@ import io import json import hashlib +import fnmatch import re import sqlite3 from datetime import datetime from pathlib import Path import tempfile from enum import Enum -from typing import Optional, Type, Union, ByteString +from typing import List, Optional, Type, Union, ByteString from .common import ClientError @@ -280,6 +281,33 @@ def is_mergin_config(path: str) -> bool: return filename == "mergin-config.json" +def is_path_in_scope(path: str, include: List[str] = None, exclude: List[str] = None) -> bool: + """ + Returns whether `path` should be kept under a sparse-checkout style include/exclude filter. + + With `include`, only paths matching at least one glob pattern are kept. With `exclude`, + paths matching at least one pattern are dropped. With neither given, every path is kept. + + Assumes include/exclude were already validated as mutually exclusive by the caller. + """ + if include: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in include) + if exclude: + return not any(fnmatch.fnmatchcase(path, pattern) for pattern in exclude) + return True + + +def filter_files(files: List[dict], include: List[str] = None, exclude: List[str] = None) -> List[dict]: + """ + Keep only files (dict with 'path' key) matching a sparse-checkout style filter. + + .. seealso:: is_path_in_scope + """ + if include and exclude: + raise ClientError("Cannot use both include and exclude filters at the same time") + return [f for f in files if is_path_in_scope(f["path"], include=include, exclude=exclude)] + + def bytes_to_human_size(bytes: int): """ Convert bytes to human readable size