Add Postgres metadata backend and cleanup tooling (clearing metadata.json)

.gitignore

@@ -1,8 +1,28 @@
-config.yaml
-/**/__pycache__/
-.idea
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+*.egg-info/
+
+# Virtualenvs
+.env
+.venv
+venv/
+
+# Editor
 .vscode/
-CLAUDE.md
-dist/
-coverage.xml
-.coverage
+.idea/
+
+# Docker/Postgres Daten
+pgdata/
+data/
+
+# Logs
+*.log
+
+# Sensitive Dateien / lokale Config
+config.yaml
+deploy/config.yaml
+*.session
+*.db

Dockerfile

@@ -37,6 +37,7 @@ COPY --from=builder /usr/local/bin /usr/local/bin
 COPY tgfs/ ./tgfs/
 COPY asgidav/ ./asgidav/
 COPY main.py ./
+COPY cleanup_deleted.py ./ 
 
 # Create non-root user
 RUN useradd --create-home --shell /bin/bash tgfs
@@ -48,4 +49,4 @@ EXPOSE 1900
 ENV DATA_DIR=/home/tgfs/.tgfs
 
 # Run the application
-CMD ["python", "main.py"]
+CMD ["python", "main.py"]

README.md

@@ -60,3 +60,19 @@ Before committing and pushing, run the following command to install git hooks:
 ```bash
 pre-commit install
 ```
+## Docker + Postgres Example
+
+This repository contains a Postgres-backed TGFS variant.
+
+Example deployment files are under `deploy/`:
+
+- `docker-compose.example.yml` – example Docker setup
+- `config.example.yaml` – example TGFS configuration (no real secrets)
+
+To run locally:
+
+```bash
+cd deploy
+cp config.example.yaml config.yaml   # edit with your own API keys & secrets
+cp docker-compose.example.yml docker-compose.yml
+docker compose up -d

cleanup_deleted.py

@@ -0,0 +1,125 @@
+import asyncio
+import json
+import logging
+import os
+from typing import Any, Dict
+
+import asyncpg
+
+from tgfs.config import get_config, MetadataType
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+def _clean_node(node: Any) -> Any:
+    """
+    Entfernt rekursiv alle Knoten mit {"deleted": true} aus dem
+    TGFS-Metadatenbaum (Dictionary-Struktur).
+    """
+    if isinstance(node, dict):
+        # Wenn dieser Knoten selbst als gelöscht markiert ist -> komplett entfernen
+        if node.get("deleted") is True:
+            return None
+
+        # Wenn der Knoten Kinder hat, diese rekursiv säubern
+        if "children" in node and isinstance(node["children"], list):
+            new_children = []
+            for child in node["children"]:
+                cleaned = _clean_node(child)
+                if cleaned is not None:
+                    new_children.append(cleaned)
+            node["children"] = new_children
+
+        # Evtl. verschachtelte Strukturen weiter säubern
+        for k, v in list(node.items()):
+            if isinstance(v, (dict, list)):
+                cleaned = _clean_node(v)
+                node[k] = cleaned
+
+        return node
+
+    if isinstance(node, list):
+        new_list = []
+        for item in node:
+            cleaned = _clean_node(item)
+            if cleaned is not None:
+                new_list.append(cleaned)
+        return new_list
+
+    # primitive Typen (str/int/...) unverändert lassen
+    return node
+
+
+async def cleanup_one_meta(dsn: str, name: str) -> None:
+    logger.info("Cleaning metadata for name=%s", name)
+
+    conn = await asyncpg.connect(dsn)
+    try:
+        row = await conn.fetchrow(
+            "SELECT data FROM tgfs_metadata WHERE name = $1",
+            name,
+        )
+        if row is None:
+            logger.info("No row found for name=%s, skipping", name)
+            return
+
+        data = row["data"]
+        # asyncpg kann JSONB als dict oder str liefern
+        if isinstance(data, str):
+            data = json.loads(data)
+
+        if not isinstance(data, dict):
+            logger.warning("Unexpected data type for name=%s: %r", name, type(data))
+            return
+
+        # Wir erwarten, dass der Root-Baum unter "dir" hängt
+        if "dir" not in data:
+            logger.warning('No "dir" key in metadata for name=%s, skipping', name)
+            return
+
+        cleaned_dir = _clean_node(data["dir"])
+        data["dir"] = cleaned_dir
+
+        cleaned_json = json.dumps(data)
+        await conn.execute(
+            """
+            UPDATE tgfs_metadata
+            SET data = $2::jsonb,
+                mtime = NOW()
+            WHERE name = $1
+            """,
+            name,
+            cleaned_json,
+        )
+
+        logger.info("Finished cleaning metadata for name=%s", name)
+    finally:
+        await conn.close()
+
+
+async def main() -> None:
+    # TGFS-Konfiguration laden (nutzt TGFS_CONFIG_FILE / TGFS_DATA_DIR)
+    cfg = get_config()
+
+    # Für alle Channels, die postgres als Metadaten-Backend nutzen
+    tasks = []
+    for _, meta_cfg in cfg.tgfs.metadata.items():
+        if meta_cfg.type != MetadataType.POSTGRES:
+            continue
+        if meta_cfg.postgres is None:
+            continue
+
+        dsn = meta_cfg.postgres.dsn
+        name = meta_cfg.name or "default"
+        tasks.append(cleanup_one_meta(dsn, name))
+
+    if not tasks:
+        logger.info("No postgres metadata configurations found. Nothing to do.")
+        return
+
+    await asyncio.gather(*tasks)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

deploy/config.example.yaml

@@ -0,0 +1,32 @@
+telegram:
+  api_id: 123456
+  api_hash: "your_api_hash_here"
+  lib: telethon
+  bot:
+    session_file: bot.session
+    tokens:
+      - "0000000:FAKE-TELEGRAM-BOT-TOKEN"
+  private_file_channel:
+    - "0000000000"
+
+tgfs:
+  users:
+    demo:
+      password: "demo_password"
+  download:
+    chunk_size_kb: 1024
+  jwt:
+    secret: "CHANGE_ME_TO_A_RANDOM_SECRET"
+    algorithm: HS256
+    life: 604800
+
+  metadata:
+    "0000000000":
+      name: default
+      type: postgres
+      postgres:
+        dsn: "postgresql://tgfs:example_password@db:5432/tgfs"
+
+  server:
+    host: 0.0.0.0
+    port: 1900

deploy/docker-compose.example.yml

@@ -0,0 +1,30 @@
+services:
+  db:
+    image: postgres:16
+    container_name: tgfs-postgres-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: tgfs
+      POSTGRES_USER: tgfs
+      POSTGRES_PASSWORD: example_pw
+    volumes:
+      - ./pgdata:/var/lib/postgresql/data
+
+  tgfs:
+    build: .
+    container_name: tgfs-postgres-app
+    restart: unless-stopped
+    depends_on:
+      - db
+    environment:
+      # TGFS liest diese Variablen in tgfs/config.py
+      TGFS_CONFIG_FILE: /app/config.yaml
+      TGFS_DATA_DIR: /data
+    volumes:
+      # deine TGFS-Konfiguration
+      - ./config.yaml:/app/config.yaml:ro
+      # Persistente Daten (Sessions, Logs etc.)
+      - ./data:/data
+    ports:
+      # WebDAV / TGFS HTTP
+      - "1900:1900"

tgfs/config.py

@@ -80,35 +80,50 @@ def from_dict(cls, data: dict) -> Self:
         )
 
 
+@dataclass
+class PostgresMetadataConfig:
+    dsn: str  # z.B. "postgresql://user:pass@host:5432/tgfs"
+
+    @classmethod
+    def from_dict(cls, data: dict) -> Self:
+        return cls(
+            dsn=data["dsn"],
+        )
+
+
 class MetadataType(Enum):
     PINNED_MESSAGE = "pinned_message"
     GITHUB_REPO = "github_repo"
+    POSTGRES = "postgres"
 
 
 class MetadataConfigDict(TypedDict):
     name: str
     type: str
     github_repo: Optional[Dict]
+    postgres: Optional[Dict]
 
 
 @dataclass
 class MetadataConfig:
     name: str
     type: MetadataType
     github_repo: Optional[GithubRepoConfig]
+    postgres: Optional[PostgresMetadataConfig]
 
     @classmethod
     def from_dict(cls, data: MetadataConfigDict) -> Self:
-        if (
-            data.get("type", MetadataType.PINNED_MESSAGE.value)
-            == MetadataType.PINNED_MESSAGE.value
-        ):
+        t = data.get("type", MetadataType.PINNED_MESSAGE.value)
+
+        if t == MetadataType.PINNED_MESSAGE.value:
             return cls(
                 name=data.get("name", "default"),
                 type=MetadataType.PINNED_MESSAGE,
                 github_repo=None,
+                postgres=None,
             )
-        if data["type"] == MetadataType.GITHUB_REPO.value:
+
+        if t == MetadataType.GITHUB_REPO.value:
             if not (gh_repo_config := data.get("github_repo")):
                 raise ValueError(
                     "GitHub repo configuration is required for GITHUB_REPO type"
@@ -117,9 +132,24 @@ def from_dict(cls, data: MetadataConfigDict) -> Self:
                 name=data.get("name", "default"),
                 type=MetadataType.GITHUB_REPO,
                 github_repo=GithubRepoConfig.from_dict(gh_repo_config),
+                postgres=None,
             )
+
+        if t == MetadataType.POSTGRES.value:
+            if not (pg_cfg := data.get("postgres")):
+                raise ValueError(
+                    "postgres configuration is required for POSTGRES metadata type"
+                )
+            return cls(
+                name=data.get("name", "default"),
+                type=MetadataType.POSTGRES,
+                github_repo=None,
+                postgres=PostgresMetadataConfig.from_dict(pg_cfg),
+            )
+
         raise ValueError(
-            f"Unknown metadata type: {data['type']}, available options: {', '.join(e.value for e in MetadataType)}"
+            f"Unknown metadata type: {data['type']}, "
+            f"available options: {', '.join(e.value for e in MetadataType)}"
         )