Add sparse checkout support: filter downloaded/pushed files by include/exclude pattern

Author: varmar05

mergin/cli.py

@@ -248,16 +248,20 @@ def list_projects(ctx, name, namespace, order_params):
 @click.argument("project")
 @click.argument("directory", type=click.Path(), required=False)
 @click.option("--version", default=None, help="Version of project to download")
+@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'")
+@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'")
 @click.pass_context
-def download(ctx, project, directory, version):
+def download(ctx, project, directory, version, include, exclude):
     """Download last version of mergin project."""
     mc = ctx.obj["client"]
     if mc is None:
         return
+    if include and exclude:
+        raise click.UsageError("--include and --exclude cannot be used together")
     directory = directory or os.path.basename(project)
     click.echo("Downloading into {}".format(directory))
     try:
-        job = download_project_async(mc, project, directory, version)
+        job = download_project_async(mc, project, directory, version, include=include, exclude=exclude)
         with click.progressbar(length=job.total_size) as bar:
             last_transferred_size = 0
             while download_project_is_running(job):

mergin/client.py

@@ -63,6 +63,7 @@
 from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable
 from .utils import (
     DateTimeEncoder,
+    filter_files,
     get_versions_with_file_changes,
     int_version,
     is_version_acceptable,
@@ -902,7 +903,7 @@ def project_versions(self, project_path, since=1, to=None):
         filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions))
         return filtered_versions
 
-    def download_project(self, project_path, directory, version=None):
+    def download_project(self, project_path, directory, version=None, include=None, exclude=None):
         """
         Download project into given directory. If version is not specified, latest version is downloaded
 
@@ -914,8 +915,16 @@ def download_project(self, project_path, directory, version=None):
 
         :param version: Project version to download, e.g. v42
         :type version: String
+
+        :param include: Optional list of glob patterns (matched against each file's project path, e.g.
+            "media/*" or "*.gpkg") - only matching files are downloaded.
+        :type include: List[String]
+
+        :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive
+            with include.
+        :type exclude: List[String]
         """
-        job = download_project_async(self, project_path, directory, version)
+        job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude)
         download_project_wait(job)
         download_project_finalize(job)
 
@@ -1158,6 +1167,10 @@ def project_status(self, directory):
         server_info = self.project_info(mp.project_full_name(), since=mp.version())
 
         pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version"))
+        # on a sparse checkout, don't report excluded files as pending server changes -
+        # they were never meant to be pulled in the first place
+        file_filter = mp.file_filter()
+        pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()}
 
         push_changes = mp.get_push_changes()
         push_changes_summary = mp.get_list_of_push_changes(push_changes)

mergin/client_pull.py

@@ -25,7 +25,7 @@
 from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType
 from .models import ProjectDelta, ProjectDeltaChange, PullAction
 from .merginproject import MerginProject
-from .utils import cleanup_tmp_dir, save_to_file
+from .utils import cleanup_tmp_dir, filter_files, path_matches_filter, save_to_file, validates_file_filter
 from typing import List, Optional
 
 # status = download_project_async(...)
@@ -242,10 +242,15 @@ def _cleanup_failed_download(mergin_project: MerginProject = None):
     return dest_path
 
 
-def download_project_async(mc, project_path, directory, project_version=None):
+@validates_file_filter
+def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None):
     """
     Starts project download in background and returns handle to the pending project download.
     Using that object it is possible to watch progress or cancel the ongoing work.
+
+    `include`/`exclude` are optional lists of glob patterns (matched against each file's project
+    path, e.g. "media/*" or "*.gpkg") to only download a subset of the project's files. They are
+    mutually exclusive.
     """
 
     if "/" not in project_path:
@@ -276,6 +281,11 @@ def download_project_async(mc, project_path, directory, project_version=None):
 
     mp.log.info(f"got project info. version {version}")
 
+    # keep only the files matching the filter (if any)
+    project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude)
+    if include or exclude:
+        project_info["file_filter"] = {"include": include, "exclude": exclude}
+
     # prepare download
     update_tasks = []  # stuff to do at the end of download
     for file in project_info["files"]:
@@ -525,6 +535,9 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
         mp.log.info("--- pull aborted")
         raise
 
+    file_filter = mp.file_filter()
+    delta.changes = [c for c in delta.changes if path_matches_filter(c.path, **file_filter)]
+
     mp.log.info(f"got project versions: local version {local_version} / server version {server_version}")
 
     if local_version == server_version:
@@ -748,6 +761,14 @@ def pull_project_finalize(job: PullJob):
         cleanup_tmp_dir(job.mp, job.tmp_dir)  # delete our temporary dir and all its content
         raise ClientError("Failed to apply pull actions: " + str(e))
 
+    file_filter = job.mp.file_filter()
+    job.project_info["files"] = filter_files(job.project_info["files"], **file_filter)
+    # keep the sparse checkout: re-apply the filter this project was downloaded with,
+    # since job.project_info is a fresh, unfiltered response from the server
+    # and update_metadata() replaces the whole metadata dict rather than merging into it
+    if file_filter["include"] or file_filter["exclude"]:
+        job.project_info["file_filter"] = file_filter
+
     job.mp.update_metadata(job.project_info)
 
     if job.mp.has_unfinished_pull():

mergin/client_push.py

@@ -34,7 +34,7 @@
 )
 from .merginproject import MerginProject, pygeodiff
 from .editor import filter_changes
-from .utils import get_data_checksum, cleanup_tmp_dir
+from .utils import get_data_checksum, cleanup_tmp_dir, filter_files
 
 POST_JSON_HEADERS = {"Content-Type": "application/json"}
 
@@ -458,6 +458,14 @@ def push_project_finalize(job: UploadJob):
             cleanup_tmp_dir(job.mp, job.tmp_dir)  # delete our temporary dir and all its content
             raise err
 
+    # keep the sparse checkout: re-apply the filter this project was
+    # downloaded with, since job.server_resp is a fresh, unfiltered response from the server
+    # and update_metadata() replaces the whole metadata dict rather than merging into it
+    file_filter = job.mp.file_filter()
+    job.server_resp["files"] = filter_files(job.server_resp["files"], **file_filter)
+    if file_filter["include"] or file_filter["exclude"]:
+        job.server_resp["file_filter"] = file_filter
+
     job.mp.update_metadata(job.server_resp)
     try:
         job.mp.apply_push_changes(asdict(job.changes))

mergin/merginproject.py

@@ -24,6 +24,7 @@
     unique_path_name,
     conflicted_copy_file_name,
     edit_conflict_file_name,
+    filter_files,
 )
 from .local_changes import FileChange
 
@@ -215,6 +216,13 @@ def files(self) -> list:
         self._read_metadata()
         return self._metadata["files"]
 
+    def file_filter(self) -> dict:
+        """
+        Returns the include/exclude file filter this project was downloaded with, as a dict with "include" and "exclude" keys.
+        """
+        self._read_metadata()
+        return self._metadata.get("file_filter", {"include": None, "exclude": None})
+
     @property
     def metadata(self) -> dict:
         """Gets raw access to metadata. Kept only for backwards compatibility and will be removed."""
@@ -566,7 +574,8 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]:
         :rtype: List[ProjectDeltaItem]
         """
         result = []
-        changes = self.compare_file_sets(self.files(), self.inspect_files())
+        current_files = filter_files(self.inspect_files(), **self.file_filter())
+        changes = self.compare_file_sets(self.files(), current_files)
         added = changes.get("added", [])
         removed = changes.get("removed", [])
         updated = changes.get("updated", [])
@@ -655,7 +664,8 @@ def get_push_changes(self):
         :returns: changes metadata for files to be pushed to server
         :rtype: dict
         """
-        changes = self.compare_file_sets(self.files(), self.inspect_files())
+        current_files = filter_files(self.inspect_files(), **self.file_filter())
+        changes = self.compare_file_sets(self.files(), current_files)
         # do checkpoint to push changes from wal file to gpkg
         for file in changes["added"] + changes["updated"]:
             size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log)