From 80fd0498d72a12fb7a1aeede61df1f9351e1cf19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 2 Jul 2026 16:59:21 +0800 Subject: [PATCH 01/14] fix --- tests/api/test_upload_download.py | 6 +-- tests/cli/test_upload.py | 10 ++-- ultron/cli/__init__.py | 26 ++++++--- ultron/cli/client.py | 35 ++++--------- ultron/cli/commands.py | 87 +++++++++++++++++++++++-------- 5 files changed, 100 insertions(+), 64 deletions(-) diff --git a/tests/api/test_upload_download.py b/tests/api/test_upload_download.py index 0ace9ba..a12ef6e 100644 --- a/tests/api/test_upload_download.py +++ b/tests/api/test_upload_download.py @@ -34,7 +34,7 @@ from types import SimpleNamespace from ultron.cli.client import ApiError, UltronClient -from ultron.cli.commands import cmd_download, cmd_list, cmd_upload, _repo_name +from ultron.cli.commands import cmd_download, cmd_status, cmd_upload, _repo_name from ultron.cli import config as cli_config from ultron.services.harness.allowlist import ( ALL_AGENT_NAME, @@ -280,11 +280,11 @@ def test_05_upload_dry_run(self): # 06. List: list sub-agents # ----------------------------------------------------------------------- def test_06_upload_list(self): - """cmd_list should enumerate sub-agents on disk and return 0.""" + """cmd_status should enumerate sub-agents on disk and return 0.""" local = self._create_local_workspace(QODER_ALL_FILES) try: args = SimpleNamespace(framework="qoder", local_dir=local) - rc = cmd_list(args) + rc = cmd_status(args) self.assertEqual(rc, 0) finally: self._cleanup_dir(local) diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index 31a7324..2284e90 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -192,7 +192,7 @@ def test_upload_global_only_no_agents_dir(self, *_): self.assertFalse(p.startswith("agents/")) -class TestListCli(unittest.TestCase): +class TestStatusCli(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) @@ -204,12 +204,12 @@ def setUp(self): def tearDown(self): self.tmp.cleanup() - def test_list_shows_agents(self): - rc = _run(["list", "--framework", "qoder", "--local_dir", str(self.root)]) + def test_status_shows_agents(self): + rc = _run(["status", "--framework", "qoder", "--local_dir", str(self.root)]) self.assertEqual(rc, 0) - def test_list_unknown_framework_fails(self): - rc = _run(["list", "--framework", "nope", "--local_dir", str(self.root)]) + def test_status_unknown_framework_fails(self): + rc = _run(["status", "--framework", "nope", "--local_dir", str(self.root)]) self.assertEqual(rc, 1) diff --git a/ultron/cli/__init__.py b/ultron/cli/__init__.py index 89a75e2..3afde21 100644 --- a/ultron/cli/__init__.py +++ b/ultron/cli/__init__.py @@ -10,7 +10,7 @@ import argparse import sys -from .commands import cmd_convert, cmd_download, cmd_list, cmd_login, cmd_recover, cmd_stop, cmd_upload, cmd_watch +from .commands import cmd_convert, cmd_download, cmd_login, cmd_recover, cmd_status, cmd_stop, cmd_upload, cmd_watch def build_parser() -> argparse.ArgumentParser: @@ -120,17 +120,27 @@ def build_parser() -> argparse.ArgumentParser: ) p_cv.set_defaults(func=cmd_convert) - # ---- list ---- - p_list = sub.add_parser( - "list", - help="List discoverable sub-agents for a framework.", + # ---- status ---- + p_status = sub.add_parser( + "status", + help="Show local agent status for a framework.", ) - p_list.add_argument( + p_status.add_argument( "--framework", "-f", required=True, help="Agent framework / bot type.", ) - p_list.add_argument("--local_dir", "-d", help="Override workspace root.") - p_list.set_defaults(func=cmd_list) + p_status.add_argument("--local_dir", "-d", help="Override workspace root.") + p_status.set_defaults(func=cmd_status) + + # ---- backups ---- + p_backups = sub.add_parser( + "backups", + help="List available backups.", + ) + p_backups.add_argument("--framework", "-f", help="Filter by framework.") + p_backups.add_argument("--name", "-n", help="Filter by agent name.") + p_backups.add_argument("--local_dir", "-d", help="Override workspace root.") + p_backups.set_defaults(func=cmd_recover, target=None, list=True) # ---- watch ---- p_watch = sub.add_parser( diff --git a/ultron/cli/client.py b/ultron/cli/client.py index c03a196..558075e 100644 --- a/ultron/cli/client.py +++ b/ultron/cli/client.py @@ -65,7 +65,7 @@ def login(self, token: str) -> str: data = self._openapi.get_current_user() except HubError as exc: raise _wrap(exc) from exc - return data.get("username", data.get("Username", "")) + return data.get("username") or data.get("Username") or "" # ---- repository ---- @@ -96,27 +96,12 @@ def list_agents(self, owner: Optional[str] = None, page_number: int = 1, page_si "GET", "/agents", params=params, require_token=False) except HubError as exc: raise _wrap(exc) from exc - # Normalize response: server may return {Data: [...], Total: N} - # or {items: [...], total_count: N}. - if isinstance(data, dict): - items = None - for key in ("Data", "items", "data"): - if key in data: - items = data[key] - break - if items is None: - items = [] - total = data.get("Total") - if total is None: - total = data.get("total_count") - if total is None: - total = data.get("TotalCount") - if total is None: - total = len(items) - return {"items": items, "total_count": total} - # If response is a list directly if isinstance(data, list): return {"items": data, "total_count": len(data)} + if isinstance(data, dict): + items = data.get("Data") or [] + total = data.get("Total") or data.get("TotalCount") or len(items) + return {"items": items, "total_count": total} return {"items": [], "total_count": 0} def create_repo( @@ -188,7 +173,7 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] raw = [] if isinstance(data, dict): - raw = data.get("trees") or data.get("Trees") or [] + raw = data.get("Trees") or data.get("trees") or [] elif isinstance(data, list): raw = data @@ -196,10 +181,10 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] if not isinstance(item, dict): continue all_entries.append({ - "path": item.get("path") or item.get("Path") or "", - "type": item.get("type") or item.get("Type") or "", - "sha256": item.get("sha256") or item.get("Sha256") or "", - "committed_date": item.get("committed_date") or item.get("Committed_date") or 0, + "path": item.get("Path") or item.get("path") or "", + "type": item.get("Type") or item.get("type") or "", + "sha256": item.get("Sha256") or item.get("sha256") or "", + "committed_date": item.get("Committed_date") or item.get("committed_date") or 0, }) if len(raw) < page_size: diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index c702d55..408c5dd 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -161,7 +161,7 @@ def _convert(resources: dict, source_fw: str, target_fw: str) -> dict: return result.merged_files -def cmd_list(args) -> int: +def cmd_status(args) -> int: """List discoverable sub-agents for a framework.""" framework = args.framework if framework not in ALLOWLIST_REGISTRY: @@ -330,48 +330,88 @@ def cmd_download(args) -> int: return 0 -def cmd_convert(args) -> int: - """Local-only format conversion: read a workspace, convert, write it out.""" - source_fw = args.source - target_fw = args.target - for fw, label in ((source_fw, "--from"), (target_fw, "--to")): - if fw not in ALLOWLIST_REGISTRY: - return _fail(f"unknown framework '{fw}' for {label}. Available: {_frameworks()}") +def convert_workspace( + src_spec, source_fw: str, target_fw: str, dst_spec, dry_run: bool = False +) -> int: + """Shared convert logic: merge → filter defaults → backup → write. - name = args.name or "default" - src_spec = _build_allowlist(source_fw, name, args.local_dir) + Returns 0 on success, 1 on failure. + """ src_root = src_spec.workspace_root resources = src_spec.collect() if not resources: return _fail( - f"no {source_fw} files found under {src_root}. Check the path or pass --local_dir." + f"no {source_fw} files found under {src_root}." ) - converted = _convert(resources, source_fw, target_fw) - - # Destination: --out wins, else the target framework's workspace root. - if args.out: - dst_spec = _build_allowlist(target_fw, name, args.out) + # Convert via merge_resources to get full action details + if source_fw == target_fw: + converted = resources + default_paths = set() else: - dst_spec = _build_allowlist(target_fw, name, None) + result = merge_resources( + incoming=resources, + source_product=source_fw, + target_product=target_fw, + source_defaults=get_defaults(source_fw), + target_defaults=get_defaults(target_fw), + ) + default_paths = { + a.path for a in result.actions if a.action == 'default' + } + converted = result.merged_files + dst_root = dst_spec.workspace_root + # Filter out default-only files: don't create or overwrite with empty templates + effective = {k: v for k, v in converted.items() if k not in default_paths} + skipped_defaults = sorted(default_paths & set(converted.keys())) + print( - f"Convert {source_fw} ({src_root}) -> {target_fw} ({dst_root}): " - f"{len(resources)} in, {len(converted)} out" + f"Convert {source_fw}/{src_spec.agent_name} ({src_root}) -> " + f"{target_fw}/{dst_spec.agent_name} ({dst_root}): " + f"{len(resources)} in, {len(effective)} out" ) - for rel in sorted(converted): + for rel in sorted(effective): print(f" {rel} -> {dst_root / rel}") + if skipped_defaults: + print(f" ({len(skipped_defaults)} default template(s) skipped: " + f"{', '.join(skipped_defaults)})") - if args.dry_run: + if dry_run: print("\n[dry-run] nothing written.") return 0 - written = dst_spec.apply(converted) + if not effective: + print("\nNo effective files to write (all were default templates).") + return 0 + + # Backup existing target files before overwriting + from .sync import backup_local + existing = dst_spec.collect() + if existing: + backup_path = backup_local(dst_spec, f"{target_fw}_{dst_spec.agent_name}") + print(f" Backup: {backup_path}") + + written = dst_spec.apply(effective) print(f"\nWrote {len(written)} file(s) under {dst_root}.") return 0 +def cmd_convert(args) -> int: + """Local-only format conversion: read a workspace, convert, write it out.""" + source_fw = args.source + target_fw = args.target + for fw, label in ((source_fw, "--from"), (target_fw, "--to")): + if fw not in ALLOWLIST_REGISTRY: + return _fail(f"unknown framework '{fw}' for {label}. Available: {_frameworks()}") + + name = args.name or "default" + src_spec = _build_allowlist(source_fw, name, args.local_dir) + dst_spec = _build_allowlist(target_fw, name, args.out) + return convert_workspace(src_spec, source_fw, target_fw, dst_spec, dry_run=args.dry_run) + + def cmd_watch(args) -> int: """Start background bidirectional sync for agent files.""" from .cache import pid_file @@ -428,7 +468,7 @@ def cmd_watch(args) -> int: try: info = client.repo_info(group, repo) if info: - remote_fw = info.get("Framework") or info.get("framework") or "" + remote_fw = info.get("Framework", "") if remote_fw and remote_fw != framework: return _fail( f"framework mismatch: local={framework}, remote={remote_fw}. " @@ -588,3 +628,4 @@ def cmd_recover(args) -> int: print(f"\nRestored {restored} file(s), removed {deleted} extra file(s).") return 0 + From 66725a76d06dad7df5169a992684449644fc494a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 2 Jul 2026 17:10:06 +0800 Subject: [PATCH 02/14] fix --- ultron/cli/commands.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 408c5dd..cef6b59 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -169,16 +169,11 @@ def cmd_status(args) -> int: spec = _build_allowlist(framework, DEFAULT_AGENT_NAME, getattr(args, 'local_dir', None)) agents = spec.list_agents() - print(f"Sub-agents for {framework}:") + print(f"Agents for {framework}:") for a in agents: - # Show file count for each agent. - if a == DEFAULT_AGENT_NAME: - tmp = _build_allowlist(framework, GLOBAL_AGENT_NAME, getattr(args, 'local_dir', None)) - else: - tmp = _build_allowlist(framework, a, getattr(args, 'local_dir', None)) + tmp = _build_allowlist(framework, a, getattr(args, 'local_dir', None)) count = len(tmp.collect_bytes()) - label = " (global/shared files only)" if a == DEFAULT_AGENT_NAME else "" - print(f" {a} — {count} file(s){label}") + print(f" {a} — {count} file(s)") return 0 From 89f5f90d705f674d629e9522754a56cc8ca9fd5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 2 Jul 2026 17:13:54 +0800 Subject: [PATCH 03/14] fix --- ultron/cli/commands.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index cef6b59..719e44d 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -172,8 +172,10 @@ def cmd_status(args) -> int: print(f"Agents for {framework}:") for a in agents: tmp = _build_allowlist(framework, a, getattr(args, 'local_dir', None)) - count = len(tmp.collect_bytes()) - print(f" {a} — {count} file(s)") + files = tmp.collect_bytes() + print(f" {a} — {len(files)} file(s), root: {tmp.workspace_root}") + for rel in sorted(files): + print(f" {rel}") return 0 @@ -525,6 +527,24 @@ def cmd_recover(args) -> int: # --list mode: enumerate backups and exit if args.list: + # Filter by framework/name if provided (filename: {fw}_{name}_{date}_{time}.zip) + fw_filter = getattr(args, 'framework', None) + name_filter = getattr(args, 'name', None) + if fw_filter or name_filter: + filtered = [] + for f in backups: + parts = f.stem.rsplit("_", 2) + if len(parts) >= 3: + prefix = parts[0] # e.g. "qwenpaw_default" + else: + prefix = f.stem + if fw_filter and not prefix.startswith(fw_filter): + continue + if name_filter and f"_{name_filter}_" not in f.stem: + continue + filtered.append(f) + backups = filtered + if not backups: print("No backups found.") return 0 From ebb5c4b3dd86c09cc325d0aa5e6335c9c591cf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 2 Jul 2026 17:22:51 +0800 Subject: [PATCH 04/14] fix --- tests/cli/test_download_convert.py | 4 +- tests/cli/test_upload.py | 95 ++++++++++++++++++++++++++++++ ultron/cli/__init__.py | 2 +- ultron/cli/commands.py | 15 ++++- 4 files changed, 112 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_download_convert.py b/tests/cli/test_download_convert.py index eb4dec8..0b08abf 100644 --- a/tests/cli/test_download_convert.py +++ b/tests/cli/test_download_convert.py @@ -153,7 +153,7 @@ def tearDown(self): def test_convert_local_nanobot_to_hermes(self): rc = _run([ "convert", "--from", "nanobot", "--to", "hermes", - "--local_dir", str(self.src), "--out", str(self.out), + "--local_dir", str(self.src), "--out-dir", str(self.out), ]) self.assertEqual(rc, 0) self.assertTrue((self.out / "SOUL.md").is_file()) @@ -163,7 +163,7 @@ def test_convert_local_nanobot_to_hermes(self): def test_convert_dry_run_writes_nothing(self): rc = _run([ "convert", "--from", "nanobot", "--to", "hermes", - "--local_dir", str(self.src), "--out", str(self.out), "--dry-run", + "--local_dir", str(self.src), "--out-dir", str(self.out), "--dry-run", ]) self.assertEqual(rc, 0) self.assertFalse(self.out.exists()) diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index 2284e90..f89249b 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -213,5 +213,100 @@ def test_status_unknown_framework_fails(self): self.assertEqual(rc, 1) +class TestBackupsFilterCli(unittest.TestCase): + """Test backup list/restore framework and name filtering.""" + + def setUp(self): + import zipfile + self.tmp = tempfile.TemporaryDirectory() + # Create fake backup zips in a temp cache dir. + self.cache_dir = Path(self.tmp.name) + # Simulate backups for different frameworks (real zip files). + for name in [ + "qoder_default_20260624_120000.zip", + "qoder_reviewer_20260624_130000.zip", + "qwenpaw_default_20260702_170208.zip", + "nanobot_mybot_20260703_100000.zip", + ]: + zpath = self.cache_dir / name + with zipfile.ZipFile(zpath, 'w') as zf: + zf.writestr("dummy.txt", "placeholder") + + def tearDown(self): + self.tmp.cleanup() + + @mock.patch("ultron.cli.cache.cache_dir") + def test_backups_list_all(self, mock_cache): + """Without --framework, list all backups.""" + mock_cache.return_value = self.cache_dir + rc = _run(["backups"]) + self.assertEqual(rc, 0) + + @mock.patch("ultron.cli.cache.cache_dir") + def test_backups_list_filter_by_framework(self, mock_cache): + """With --framework qoder, only qoder backups appear.""" + mock_cache.return_value = self.cache_dir + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + rc = _run(["backups", "--framework", "qoder"]) + self.assertEqual(rc, 0) + output = buf.getvalue() + self.assertIn("qoder_default_20260624_120000.zip", output) + self.assertIn("qoder_reviewer_20260624_130000.zip", output) + self.assertNotIn("qwenpaw", output) + self.assertNotIn("nanobot", output) + + @mock.patch("ultron.cli.cache.cache_dir") + def test_backups_list_filter_by_name(self, mock_cache): + """With --name reviewer, only matching backups appear.""" + mock_cache.return_value = self.cache_dir + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + rc = _run(["backups", "--name", "reviewer"]) + self.assertEqual(rc, 0) + output = buf.getvalue() + self.assertIn("qoder_reviewer_20260624_130000.zip", output) + self.assertNotIn("qoder_default", output) + self.assertNotIn("qwenpaw", output) + + @mock.patch("ultron.cli.cache.cache_dir") + def test_backups_list_no_match(self, mock_cache): + """Filter with nonexistent framework returns 'No backups found'.""" + mock_cache.return_value = self.cache_dir + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + rc = _run(["backups", "--framework", "hermes"]) + self.assertEqual(rc, 0) + self.assertIn("No backups found", buf.getvalue()) + + @mock.patch("ultron.cli.cache.cache_dir") + def test_restore_last_filters_by_framework(self, mock_cache): + """'restore last -f qoder' picks the latest qoder backup, not qwenpaw.""" + mock_cache.return_value = self.cache_dir + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + rc = _run(["restore", "last", "--framework", "qoder"]) + # rc=1 because the fake zip is not a real zip, but it should attempt + # the qoder_reviewer (latest qoder) not the qwenpaw one. + # If it picked wrong, it would fail with "no backups found" or use qwenpaw. + # Since there are qoder backups, it should NOT say "no backups found". + self.assertNotIn("no backups found", buf.getvalue().lower()) + + @mock.patch("ultron.cli.cache.cache_dir") + def test_restore_last_no_match_fails(self, mock_cache): + """'restore last -f hermes' with no hermes backups should fail.""" + mock_cache.return_value = self.cache_dir + rc = _run(["restore", "last", "--framework", "hermes"]) + self.assertEqual(rc, 1) + + if __name__ == "__main__": unittest.main() diff --git a/ultron/cli/__init__.py b/ultron/cli/__init__.py index 3afde21..5dce964 100644 --- a/ultron/cli/__init__.py +++ b/ultron/cli/__init__.py @@ -111,7 +111,7 @@ def build_parser() -> argparse.ArgumentParser: help="Source workspace root to read (default: source framework's path).", ) p_cv.add_argument( - "--out", "-o", + "--out-dir", "-o", help="Destination directory to write (default: target framework's path).", ) p_cv.add_argument( diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 719e44d..4b4b1d0 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -405,7 +405,7 @@ def cmd_convert(args) -> int: name = args.name or "default" src_spec = _build_allowlist(source_fw, name, args.local_dir) - dst_spec = _build_allowlist(target_fw, name, args.out) + dst_spec = _build_allowlist(target_fw, name, args.out_dir) return convert_workspace(src_spec, source_fw, target_fw, dst_spec, dry_run=args.dry_run) @@ -562,6 +562,19 @@ def cmd_recover(args) -> int: if not target: return _fail("specify a target: 'last' or a backup filename. Use --list to see available backups.") + # Filter backups by framework/name if provided + fw_filter = getattr(args, 'framework', None) + name_filter = getattr(args, 'name', None) + if fw_filter or name_filter: + filtered = [] + for f in backups: + if fw_filter and not f.stem.startswith(fw_filter): + continue + if name_filter and f"_{name_filter}_" not in f.stem: + continue + filtered.append(f) + backups = filtered + # Resolve target to a zip path if target == "last": if not backups: From ff54354b340bcda204fcf50586a37e3935727990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 11:31:59 +0800 Subject: [PATCH 05/14] wip --- ultron/cli/client.py | 61 +++++++++++++++++++------------------------ ultron/cli/watcher.py | 29 ++++++++++++++------ 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/ultron/cli/client.py b/ultron/cli/client.py index 558075e..ab90680 100644 --- a/ultron/cli/client.py +++ b/ultron/cli/client.py @@ -16,7 +16,6 @@ from typing import Dict, List, Optional from urllib.parse import unquote -import requests from modelscope_hub._openapi import OpenAPIClient from modelscope_hub.config import HubConfig from modelscope_hub.errors import HubError, NotExistError @@ -207,16 +206,13 @@ def download_repo_file(self, path: str, name: str, file_path: str, Returns bytes when *binary=True*, otherwise str. """ url = f"{self.server}/agents/{path}/{name}/resolve/{revision}/{file_path}" - headers = {"Authorization": f"Bearer {self.token}"} if self.token else {} try: - resp = requests.get(url, headers=headers, timeout=self.timeout) - resp.raise_for_status() - except requests.HTTPError as exc: - status = exc.response.status_code if exc.response is not None else 0 - detail = exc.response.text if exc.response is not None else str(exc) - raise ApiError(status, detail) from exc - except requests.RequestException as exc: - raise ApiError(0, str(exc)) from exc + resp = self._openapi.request( + "GET", url=url, unwrap=False, + timeout=float(self.timeout), + ) + except HubError as exc: + raise _wrap(exc) from exc return resp.content if binary else resp.text # ---- upload (two-step OSS) ---- @@ -228,18 +224,15 @@ def _request_upload_urls(self, filenames: List[str]) -> dict: capitalised keys: {"Code": 200, "Data": {...}, "Success": true}. """ url = f"{self.server}/api/v1/agents/repo/files/upload" - headers = {"Authorization": f"Bearer {self.token}", - "Content-Type": "application/json"} try: - resp = requests.post(url, json={"FileNames": filenames}, - headers=headers, timeout=self.timeout) - resp.raise_for_status() - except requests.HTTPError as exc: - status = exc.response.status_code if exc.response is not None else 0 - detail = exc.response.text if exc.response is not None else str(exc) - raise ApiError(status, detail) from exc - except requests.RequestException as exc: - raise ApiError(0, str(exc)) from exc + resp = self._openapi.request( + "POST", url=url, + json_body={"FileNames": filenames}, + unwrap=False, + timeout=float(self.timeout), + ) + except HubError as exc: + raise _wrap(exc) from exc body = resp.json() if not body.get("Success"): raise ApiError(body.get("Code", 0), body.get("Message", "upload credential failed")) @@ -277,19 +270,19 @@ def _upload_to_oss(self, signed_url: str, data: bytes) -> None: """ url = self._normalize_oss_url(signed_url) try: - resp = requests.put(url, data=data, - headers={ - "Content-Type": "application/octet-stream", - "x-oss-meta-author": "aliy", - }, - timeout=max(self.timeout, 300)) - resp.raise_for_status() - except requests.HTTPError as exc: - status = exc.response.status_code if exc.response is not None else 0 - detail = exc.response.text if exc.response is not None else str(exc) - raise ApiError(status, detail) from exc - except requests.RequestException as exc: - raise ApiError(0, str(exc)) from exc + self._openapi.request( + "PUT", url=url, + data=data, + headers={ + "Content-Type": "application/octet-stream", + "x-oss-meta-author": "aliy", + }, + require_token=False, + unwrap=False, + timeout=float(max(self.timeout, 300)), + ) + except HubError as exc: + raise _wrap(exc) from exc def upload_file(self, resources: Dict[str, bytes]) -> str: """Two-step upload: get signed URLs → PUT to OSS → return Gid. diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index fbec77f..1f69fce 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -62,9 +62,11 @@ def _handle_term(signum, frame): # Unix: register signal handlers for graceful stop via kill(1). # Windows: SIGTERM triggers TerminateProcess (hard kill), so signals are # unreliable; the stop-file mechanism below is the primary channel. - if hasattr(signal, "SIGTERM"): - signal.signal(signal.SIGTERM, _handle_term) - signal.signal(signal.SIGINT, _handle_term) + # signal.signal() can only be called from the main thread. + if threading.current_thread() is threading.main_thread(): + if hasattr(signal, "SIGTERM"): + signal.signal(signal.SIGTERM, _handle_term) + signal.signal(signal.SIGINT, _handle_term) # Remove any stale stop file from a previous session. sf.unlink(missing_ok=True) @@ -129,8 +131,11 @@ def _handle_term(signum, frame): logger.info("Watch stopped (signal received).") pf = pid_file() - if pf.exists(): - pf.unlink(missing_ok=True) + try: + if pf.read_text(encoding="utf-8").strip() == str(os.getpid()): + pf.unlink(missing_ok=True) + except Exception: + pass sf.unlink(missing_ok=True) @@ -162,17 +167,21 @@ def _sync_action( state, ) -> bool: """Execute the appropriate sync action. Returns True if something changed.""" + # Backup naming convention: {framework}_{agent_name} so that + # ``cmd_recover --framework`` can filter watch-created backups. + backup_label = f"{framework}_{spec.agent_name}" + if push_only: if not local_changed: return False return _push_local(client, username, name, framework, local_resources, state, logger) if remote_changed and local_changed: - backup_path = backup_local(spec, name) + backup_path = backup_local(spec, backup_label) pull_incremental(client, username, name, spec, remote_files, local_resources) logger.warning("Conflict: remote wins. Local backup: %s", backup_path) elif remote_changed: - backup_path = backup_local(spec, name) + backup_path = backup_local(spec, backup_label) pull_incremental(client, username, name, spec, remote_files, local_resources) logger.info("Pulled remote changes (backup: %s).", backup_path) elif local_changed: @@ -247,7 +256,11 @@ def _daemonize_unix(target, *args, **kwargs): try: target(*args, **kwargs) finally: - pf.unlink(missing_ok=True) + try: + if pf.exists() and pf.read_text(encoding="utf-8").strip() == str(os.getpid()): + pf.unlink(missing_ok=True) + except Exception: + pass os._exit(0) From fce565b6eda50a81bce238ff7819b593098b1983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 12:33:38 +0800 Subject: [PATCH 06/14] fix --- ultron/cli/client.py | 96 ++++++++++++++++++++++++++---------------- ultron/cli/commands.py | 2 +- ultron/cli/watcher.py | 29 ++++--------- 3 files changed, 68 insertions(+), 59 deletions(-) diff --git a/ultron/cli/client.py b/ultron/cli/client.py index ab90680..c03a196 100644 --- a/ultron/cli/client.py +++ b/ultron/cli/client.py @@ -16,6 +16,7 @@ from typing import Dict, List, Optional from urllib.parse import unquote +import requests from modelscope_hub._openapi import OpenAPIClient from modelscope_hub.config import HubConfig from modelscope_hub.errors import HubError, NotExistError @@ -64,7 +65,7 @@ def login(self, token: str) -> str: data = self._openapi.get_current_user() except HubError as exc: raise _wrap(exc) from exc - return data.get("username") or data.get("Username") or "" + return data.get("username", data.get("Username", "")) # ---- repository ---- @@ -95,12 +96,27 @@ def list_agents(self, owner: Optional[str] = None, page_number: int = 1, page_si "GET", "/agents", params=params, require_token=False) except HubError as exc: raise _wrap(exc) from exc - if isinstance(data, list): - return {"items": data, "total_count": len(data)} + # Normalize response: server may return {Data: [...], Total: N} + # or {items: [...], total_count: N}. if isinstance(data, dict): - items = data.get("Data") or [] - total = data.get("Total") or data.get("TotalCount") or len(items) + items = None + for key in ("Data", "items", "data"): + if key in data: + items = data[key] + break + if items is None: + items = [] + total = data.get("Total") + if total is None: + total = data.get("total_count") + if total is None: + total = data.get("TotalCount") + if total is None: + total = len(items) return {"items": items, "total_count": total} + # If response is a list directly + if isinstance(data, list): + return {"items": data, "total_count": len(data)} return {"items": [], "total_count": 0} def create_repo( @@ -172,7 +188,7 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] raw = [] if isinstance(data, dict): - raw = data.get("Trees") or data.get("trees") or [] + raw = data.get("trees") or data.get("Trees") or [] elif isinstance(data, list): raw = data @@ -180,10 +196,10 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] if not isinstance(item, dict): continue all_entries.append({ - "path": item.get("Path") or item.get("path") or "", - "type": item.get("Type") or item.get("type") or "", - "sha256": item.get("Sha256") or item.get("sha256") or "", - "committed_date": item.get("Committed_date") or item.get("committed_date") or 0, + "path": item.get("path") or item.get("Path") or "", + "type": item.get("type") or item.get("Type") or "", + "sha256": item.get("sha256") or item.get("Sha256") or "", + "committed_date": item.get("committed_date") or item.get("Committed_date") or 0, }) if len(raw) < page_size: @@ -206,13 +222,16 @@ def download_repo_file(self, path: str, name: str, file_path: str, Returns bytes when *binary=True*, otherwise str. """ url = f"{self.server}/agents/{path}/{name}/resolve/{revision}/{file_path}" + headers = {"Authorization": f"Bearer {self.token}"} if self.token else {} try: - resp = self._openapi.request( - "GET", url=url, unwrap=False, - timeout=float(self.timeout), - ) - except HubError as exc: - raise _wrap(exc) from exc + resp = requests.get(url, headers=headers, timeout=self.timeout) + resp.raise_for_status() + except requests.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else 0 + detail = exc.response.text if exc.response is not None else str(exc) + raise ApiError(status, detail) from exc + except requests.RequestException as exc: + raise ApiError(0, str(exc)) from exc return resp.content if binary else resp.text # ---- upload (two-step OSS) ---- @@ -224,15 +243,18 @@ def _request_upload_urls(self, filenames: List[str]) -> dict: capitalised keys: {"Code": 200, "Data": {...}, "Success": true}. """ url = f"{self.server}/api/v1/agents/repo/files/upload" + headers = {"Authorization": f"Bearer {self.token}", + "Content-Type": "application/json"} try: - resp = self._openapi.request( - "POST", url=url, - json_body={"FileNames": filenames}, - unwrap=False, - timeout=float(self.timeout), - ) - except HubError as exc: - raise _wrap(exc) from exc + resp = requests.post(url, json={"FileNames": filenames}, + headers=headers, timeout=self.timeout) + resp.raise_for_status() + except requests.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else 0 + detail = exc.response.text if exc.response is not None else str(exc) + raise ApiError(status, detail) from exc + except requests.RequestException as exc: + raise ApiError(0, str(exc)) from exc body = resp.json() if not body.get("Success"): raise ApiError(body.get("Code", 0), body.get("Message", "upload credential failed")) @@ -270,19 +292,19 @@ def _upload_to_oss(self, signed_url: str, data: bytes) -> None: """ url = self._normalize_oss_url(signed_url) try: - self._openapi.request( - "PUT", url=url, - data=data, - headers={ - "Content-Type": "application/octet-stream", - "x-oss-meta-author": "aliy", - }, - require_token=False, - unwrap=False, - timeout=float(max(self.timeout, 300)), - ) - except HubError as exc: - raise _wrap(exc) from exc + resp = requests.put(url, data=data, + headers={ + "Content-Type": "application/octet-stream", + "x-oss-meta-author": "aliy", + }, + timeout=max(self.timeout, 300)) + resp.raise_for_status() + except requests.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else 0 + detail = exc.response.text if exc.response is not None else str(exc) + raise ApiError(status, detail) from exc + except requests.RequestException as exc: + raise ApiError(0, str(exc)) from exc def upload_file(self, resources: Dict[str, bytes]) -> str: """Two-step upload: get signed URLs → PUT to OSS → return Gid. diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 4b4b1d0..ba893a5 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -465,7 +465,7 @@ def cmd_watch(args) -> int: try: info = client.repo_info(group, repo) if info: - remote_fw = info.get("Framework", "") + remote_fw = info.get("Framework") or info.get("framework") or "" if remote_fw and remote_fw != framework: return _fail( f"framework mismatch: local={framework}, remote={remote_fw}. " diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index 1f69fce..fbec77f 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -62,11 +62,9 @@ def _handle_term(signum, frame): # Unix: register signal handlers for graceful stop via kill(1). # Windows: SIGTERM triggers TerminateProcess (hard kill), so signals are # unreliable; the stop-file mechanism below is the primary channel. - # signal.signal() can only be called from the main thread. - if threading.current_thread() is threading.main_thread(): - if hasattr(signal, "SIGTERM"): - signal.signal(signal.SIGTERM, _handle_term) - signal.signal(signal.SIGINT, _handle_term) + if hasattr(signal, "SIGTERM"): + signal.signal(signal.SIGTERM, _handle_term) + signal.signal(signal.SIGINT, _handle_term) # Remove any stale stop file from a previous session. sf.unlink(missing_ok=True) @@ -131,11 +129,8 @@ def _handle_term(signum, frame): logger.info("Watch stopped (signal received).") pf = pid_file() - try: - if pf.read_text(encoding="utf-8").strip() == str(os.getpid()): - pf.unlink(missing_ok=True) - except Exception: - pass + if pf.exists(): + pf.unlink(missing_ok=True) sf.unlink(missing_ok=True) @@ -167,21 +162,17 @@ def _sync_action( state, ) -> bool: """Execute the appropriate sync action. Returns True if something changed.""" - # Backup naming convention: {framework}_{agent_name} so that - # ``cmd_recover --framework`` can filter watch-created backups. - backup_label = f"{framework}_{spec.agent_name}" - if push_only: if not local_changed: return False return _push_local(client, username, name, framework, local_resources, state, logger) if remote_changed and local_changed: - backup_path = backup_local(spec, backup_label) + backup_path = backup_local(spec, name) pull_incremental(client, username, name, spec, remote_files, local_resources) logger.warning("Conflict: remote wins. Local backup: %s", backup_path) elif remote_changed: - backup_path = backup_local(spec, backup_label) + backup_path = backup_local(spec, name) pull_incremental(client, username, name, spec, remote_files, local_resources) logger.info("Pulled remote changes (backup: %s).", backup_path) elif local_changed: @@ -256,11 +247,7 @@ def _daemonize_unix(target, *args, **kwargs): try: target(*args, **kwargs) finally: - try: - if pf.exists() and pf.read_text(encoding="utf-8").strip() == str(os.getpid()): - pf.unlink(missing_ok=True) - except Exception: - pass + pf.unlink(missing_ok=True) os._exit(0) From ffc0d23f74f42d5e5e7f4ca8d97b128c31b1faeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 14:21:07 +0800 Subject: [PATCH 07/14] fix --- ultron/cli/__init__.py | 4 ++-- ultron/cli/watcher.py | 34 +++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ultron/cli/__init__.py b/ultron/cli/__init__.py index 5dce964..2ef49b8 100644 --- a/ultron/cli/__init__.py +++ b/ultron/cli/__init__.py @@ -156,8 +156,8 @@ def build_parser() -> argparse.ArgumentParser: help="Local sub-agent name (default: global/shared files only).", ) p_watch.add_argument( - "--repo", "-r", - help="Remote repository name. Supports 'group/name' format. Defaults to local name.", + "--repo", "-r", required=True, + help="Remote repository name (required). Supports 'group/name' format.", ) p_watch.add_argument("--local_dir", "-d", help="Override workspace root.") p_watch.add_argument("--server", help="Server URL override.") diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index fbec77f..629355a 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -11,7 +11,7 @@ from typing import List, Optional from .cache import load_sync_state, log_file, pid_file, save_sync_state, stop_file -from .client import ApiError +from .client import ApiError, UltronClient from .sync import ( backup_local, detect_local_changes, @@ -46,6 +46,12 @@ def watch_loop(spec, client, username: str, repo: str, framework: str, interval: False = full bidirectional sync (remote wins on conflict). """ logger = _get_logger() + + # After double-fork, the parent's requests.Session connection pool holds + # stale file descriptors that cause EBADF on new connections. Rebuild the + # entire client so the daemon starts with a fresh session. + client = UltronClient(client.server, client.token, client.timeout) + logger.info("Watch started for %s/%s (root=%s, interval=%ds, push_only=%s)", username, repo, spec.workspace_root, interval, push_only) @@ -134,7 +140,7 @@ def _handle_term(signum, frame): sf.unlink(missing_ok=True) -def _push_local(client, username, name, framework, local_resources, state, logger) -> bool: +def _push_local(client, username, name, framework, local_resources, state, logger, *, remote_paths=None) -> bool: """Push local changes: full upload on first time, incremental thereafter. Returns True if something was actually pushed, False otherwise. @@ -149,7 +155,23 @@ def _push_local(client, username, name, framework, local_resources, state, logge else: changed = detect_local_changes(local_resources, state["remote_files"]) if changed: - push_incremental(client, username, name, changed, set(state["remote_files"].keys())) + # Filter stale DELETEs: only delete files that actually exist on + # the remote. The baseline may be stale if the remote was + # modified outside watch (e.g. via upload or manual deletion). + if remote_paths is not None: + stale = { + p for p, c in changed.items() + if c is None and p not in remote_paths + } + for p in sorted(stale): + logger.warning(" SKIP DELETE: %s (not on remote, stale baseline)", p) + del changed[p] + if not changed: + logger.info("No real changes to push after filtering stale deletes.") + return False + # Use actual remote paths (not stale baseline) for CREATE vs UPDATE. + actual = remote_paths if remote_paths is not None else set(state["remote_files"].keys()) + push_incremental(client, username, name, changed, actual) logger.info("Pushed local changes (incremental commit).") return True return False @@ -162,10 +184,12 @@ def _sync_action( state, ) -> bool: """Execute the appropriate sync action. Returns True if something changed.""" + remote_paths = {f.path for f in remote_files} + if push_only: if not local_changed: return False - return _push_local(client, username, name, framework, local_resources, state, logger) + return _push_local(client, username, name, framework, local_resources, state, logger, remote_paths=remote_paths) if remote_changed and local_changed: backup_path = backup_local(spec, name) @@ -176,7 +200,7 @@ def _sync_action( pull_incremental(client, username, name, spec, remote_files, local_resources) logger.info("Pulled remote changes (backup: %s).", backup_path) elif local_changed: - _push_local(client, username, name, framework, local_resources, state, logger) + _push_local(client, username, name, framework, local_resources, state, logger, remote_paths=remote_paths) else: return False return True From f1192734d081ed960211152e4dda01df7a7ee4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 14:26:49 +0800 Subject: [PATCH 08/14] fix --- ultron/cli/__init__.py | 24 ++++++++++++++++++- ultron/cli/commands.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/ultron/cli/__init__.py b/ultron/cli/__init__.py index 2ef49b8..fd7ac5a 100644 --- a/ultron/cli/__init__.py +++ b/ultron/cli/__init__.py @@ -10,7 +10,7 @@ import argparse import sys -from .commands import cmd_convert, cmd_download, cmd_login, cmd_recover, cmd_status, cmd_stop, cmd_upload, cmd_watch +from .commands import cmd_convert, cmd_download, cmd_list, cmd_login, cmd_recover, cmd_status, cmd_stop, cmd_upload, cmd_watch def build_parser() -> argparse.ArgumentParser: @@ -120,6 +120,28 @@ def build_parser() -> argparse.ArgumentParser: ) p_cv.set_defaults(func=cmd_convert) + # ---- list (remote) ---- + p_list = sub.add_parser( + "list", + help="List remote agent repositories.", + description="Query and display remote agent repositories with pagination.", + ) + p_list.add_argument( + "--owner", default=None, + help="Filter by owner username or organization name.", + ) + p_list.add_argument( + "--page", dest="page_number", type=int, default=1, + help="Page number for pagination (default: 1).", + ) + p_list.add_argument( + "--page-size", dest="page_size", type=int, default=10, + help="Number of items per page (default: 10).", + ) + p_list.add_argument("--server", help="Server URL override.") + p_list.add_argument("--token", help="API token override.") + p_list.set_defaults(func=cmd_list) + # ---- status ---- p_status = sub.add_parser( "status", diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index ba893a5..7f790d6 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -161,6 +161,60 @@ def _convert(resources: dict, source_fw: str, target_fw: str) -> dict: return result.merged_files +def cmd_list(args) -> int: + """List remote agent repositories.""" + server = config.resolve_server(getattr(args, 'server', None)) + token = config.resolve_token(getattr(args, 'token', None)) + if not server: + return _fail("not logged in. Run 'ultron login' first or pass --server.") + + client = UltronClient(server, token) + try: + result = client.list_agents( + owner=getattr(args, 'owner', None), + page_number=getattr(args, 'page_number', 1), + page_size=getattr(args, 'page_size', 10), + ) + except ApiError as e: + return _fail(_api_error_message(e, "list")) + except Exception as e: + return _fail(f"list failed: {e}") + + items = result.get("items") or [] + total = result.get("total_count", len(items)) + + if not items: + print("(no agent repositories found)") + return 0 + + headers = ['repo_id', 'framework', 'visibility', 'updated'] + rows = [] + for item in items: + owner_name = item.get('Path') or item.get('path') or '' + repo_name = item.get('Name') or item.get('name') or '' + repo_id = f'{owner_name}/{repo_name}' if owner_name else repo_name + fw = item.get('Framework') or item.get('framework') or '-' + vis = item.get('Visibility') or item.get('visibility') or '-' + updated = item.get('LastUpdatedDate') or item.get('last_updated_date') or '-' + if isinstance(updated, str) and 'T' in updated: + updated = updated.split('T')[0] + rows.append((repo_id, fw, vis, updated)) + + col_widths = [len(h) for h in headers] + for row in rows: + for i, val in enumerate(row): + col_widths[i] = max(col_widths[i], len(str(val))) + + fmt = ' '.join(f'{{:<{w}}}' for w in col_widths) + print(fmt.format(*headers)) + print(fmt.format(*['-' * w for w in col_widths])) + for row in rows: + print(fmt.format(*[str(v) for v in row])) + + print(f'\npage {getattr(args, "page_number", 1)} / total {total} (page_size={getattr(args, "page_size", 10)})') + return 0 + + def cmd_status(args) -> int: """List discoverable sub-agents for a framework.""" framework = args.framework From 689c75acd757e9c24744ef486b9872f0be4ebd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 14:37:23 +0800 Subject: [PATCH 09/14] fix --- tests/cli/test_upload.py | 2 +- ultron/cli/commands.py | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index f89249b..4036ee3 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -293,7 +293,7 @@ def test_restore_last_filters_by_framework(self, mock_cache): from contextlib import redirect_stdout buf = io.StringIO() with redirect_stdout(buf): - rc = _run(["restore", "last", "--framework", "qoder"]) + rc = _run(["restore", "last", "--framework", "qoder", "--local_dir", str(self.cache_dir)]) # rc=1 because the fake zip is not a real zip, but it should attempt # the qoder_reviewer (latest qoder) not the qwenpaw one. # If it picked wrong, it would fail with "no backups found" or use qwenpaw. diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 7f790d6..97d3622 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -588,13 +588,14 @@ def cmd_recover(args) -> int: filtered = [] for f in backups: parts = f.stem.rsplit("_", 2) - if len(parts) >= 3: - prefix = parts[0] # e.g. "qwenpaw_default" - else: - prefix = f.stem - if fw_filter and not prefix.startswith(fw_filter): + prefix = parts[0] if len(parts) >= 3 else f.stem + delim = "_" if "_" in prefix else "-" + parts_fw = prefix.split(delim, 1) + fw = parts_fw[0] + name = parts_fw[1] if len(parts_fw) > 1 else "" + if fw_filter and fw != fw_filter: continue - if name_filter and f"_{name_filter}_" not in f.stem: + if name_filter and name != name_filter: continue filtered.append(f) backups = filtered @@ -622,9 +623,15 @@ def cmd_recover(args) -> int: if fw_filter or name_filter: filtered = [] for f in backups: - if fw_filter and not f.stem.startswith(fw_filter): + parts = f.stem.rsplit("_", 2) + prefix = parts[0] if len(parts) >= 3 else f.stem + delim = "_" if "_" in prefix else "-" + parts_fw = prefix.split(delim, 1) + fw = parts_fw[0] + name = parts_fw[1] if len(parts_fw) > 1 else "" + if fw_filter and fw != fw_filter: continue - if name_filter and f"_{name_filter}_" not in f.stem: + if name_filter and name != name_filter: continue filtered.append(f) backups = filtered From b05de820ad8b461913013ab2a52db7c37b82c267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 7 Jul 2026 21:19:54 +0800 Subject: [PATCH 10/14] fix --- tests/api/test_client_integration.py | 137 ++++++-------- tests/api/test_repo_api.py | 24 +-- tests/api/test_upload_download.py | 2 +- tests/api/test_watch_sync.py | 8 +- tests/cli/test_upload.py | 24 +-- ultron/api/routers/repo.py | 48 ++++- ultron/api/schemas.py | 13 +- ultron/cli/client.py | 272 +++++++++++++++------------ ultron/cli/commands.py | 12 +- ultron/cli/sync.py | 136 ++++++++++---- ultron/cli/watcher.py | 15 +- 11 files changed, 405 insertions(+), 286 deletions(-) diff --git a/tests/api/test_client_integration.py b/tests/api/test_client_integration.py index a2105c3..3faf9d6 100644 --- a/tests/api/test_client_integration.py +++ b/tests/api/test_client_integration.py @@ -29,7 +29,7 @@ # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- -SERVER = os.environ.get("SERVER", "http://pre.modelscope.cn") +SERVER = os.environ.get("SERVER", "http://www.modelscope.cn") TOKEN = os.environ.get("TOKEN", "") AGENT_NAME = "test-agent-integration" @@ -53,7 +53,7 @@ def _log_429(fn, *args, **kwargs): def _to_bytes(files: dict) -> dict: - """Convert a str-valued dict to bytes-valued for upload_file().""" + """Convert a str-valued dict to bytes-valued for commit upload.""" return { k: (v.encode("utf-8") if isinstance(v, str) else v) for k, v in files.items() @@ -202,50 +202,42 @@ def test_05_check_repo_bool(self): # 03. Upload + Create (nanobot — richest file set) # ----------------------------------------------------------------------- def test_06_upload_and_create(self): - file_id = _log_429(self.client.upload_file, _to_bytes(NANOBOT_FILES)) - self.assertTrue(file_id) - - result = _log_429( - self.client.create_repo, - path=self.username, name=AGENT_NAME, framework="nanobot", - visibility="public", system_prompt_files=file_id, + from ultron.cli.sync import push_resources + _log_429( + push_resources, + self.client, self.username, AGENT_NAME, "nanobot", + _to_bytes(NANOBOT_FILES), ) - self.assertIsInstance(result, dict) self.assertTrue(self.client.check_repo(self.username, AGENT_NAME)) # ----------------------------------------------------------------------- # 04. Repeated upload (idempotent upsert) # ----------------------------------------------------------------------- def test_07_repeated_upload(self): + from ultron.cli.sync import push_resources _wait_server(3) for i in range(2): - fid = _log_429(self.client.upload_file, _to_bytes(NANOBOT_FILES)) - self.assertTrue(fid) - result = _log_429( - self.client.create_repo, - path=self.username, name=AGENT_NAME, framework="nanobot", - visibility="public", system_prompt_files=fid, + _log_429( + push_resources, + self.client, self.username, AGENT_NAME, "nanobot", + _to_bytes(NANOBOT_FILES), ) - self.assertIsInstance(result, dict) _wait_server(REQUEST_INTERVAL) # ----------------------------------------------------------------------- # 05. Modify and re-upload # ----------------------------------------------------------------------- def test_08_modify_and_reupload(self): + from ultron.cli.sync import push_resources modified = dict(NANOBOT_FILES) modified["SOUL.md"] += "\n## Custom Section\nUser added this.\n" modified["new_file.md"] = "# New File\nAdded in update.\n" - fid = _log_429(self.client.upload_file, _to_bytes(modified)) - self.assertTrue(fid) - - result = _log_429( - self.client.create_repo, - path=self.username, name=AGENT_NAME, framework="nanobot", - visibility="public", system_prompt_files=fid, + _log_429( + push_resources, + self.client, self.username, AGENT_NAME, "nanobot", + _to_bytes(modified), ) - self.assertIsInstance(result, dict) # ----------------------------------------------------------------------- # 06. List files @@ -307,12 +299,11 @@ def test_14_repeated_download(self): # 09. E2E roundtrip # ----------------------------------------------------------------------- def test_15_e2e_roundtrip(self): - fid = _log_429(self.client.upload_file, _to_bytes(NANOBOT_FILES)) - self.assertTrue(fid) + from ultron.cli.sync import push_resources _log_429( - self.client.create_repo, - path=self.username, name=AGENT_NAME, framework="nanobot", - visibility="public", system_prompt_files=fid, + push_resources, + self.client, self.username, AGENT_NAME, "nanobot", + _to_bytes(NANOBOT_FILES), ) _wait_server(5) @@ -334,18 +325,16 @@ def test_15_e2e_roundtrip(self): # 10. Multi-framework upload # ----------------------------------------------------------------------- def test_16_multi_framework_upload(self): + from ultron.cli.sync import push_resources for fw, files in ALL_FRAMEWORK_FILES.items(): with self.subTest(framework=fw): agent = f"{AGENT_NAME}-{fw}" - fid = _log_429(self.client.upload_file, _to_bytes(files)) - self.assertTrue(fid) try: - result = _log_429( - self.client.create_repo, - path=self.username, name=agent, framework=fw, - visibility="public", system_prompt_files=fid, + _log_429( + push_resources, + self.client, self.username, agent, fw, + _to_bytes(files), ) - self.assertIsInstance(result, dict) except ApiError as e: self.fail(f"upload {fw} failed: status={e.status} {e.detail}") _wait_server(REQUEST_INTERVAL) @@ -354,20 +343,17 @@ def test_16_multi_framework_upload(self): # 11. Cross-framework conversion # ----------------------------------------------------------------------- def test_17_cross_framework_convert(self): + from ultron.cli.sync import push_resources for source_fw, target_fw in CONVERT_PAIRS: with self.subTest(pair=f"{source_fw}->{target_fw}"): source_files = ALL_FRAMEWORK_FILES[source_fw] agent = f"{AGENT_NAME}-conv-{source_fw}" - fid = _log_429(self.client.upload_file, _to_bytes(source_files)) - try: - _log_429( - self.client.create_repo, - path=self.username, name=agent, framework=source_fw, - visibility="public", system_prompt_files=fid, - ) - except ApiError: - pass # may already exist + _log_429( + push_resources, + self.client, self.username, agent, source_fw, + _to_bytes(source_files), + ) _wait_server(3) @@ -409,62 +395,62 @@ def test_17_cross_framework_convert(self): # ----------------------------------------------------------------------- # 12. Edge: empty zip # ----------------------------------------------------------------------- - def test_18_empty_zip(self): - try: - fid = _log_429(self.client.upload_file, {}) - # Accepted is fine; empty file_id is also acceptable - except ApiError: - pass # server may reject empty uploads + def test_18_empty_upload(self): + from ultron.cli.sync import push_resources + # push_resources with empty dict should be a no-op (skip). + push_resources(self.client, self.username, f"{AGENT_NAME}-empty", "qoder", {}) # ----------------------------------------------------------------------- # 13. Edge: large file # ----------------------------------------------------------------------- def test_19_large_file(self): + from ultron.cli.sync import push_resources large_content = "x" * (500 * 1024) files = {"SOUL.md": b"# Soul\nLarge file test.\n", "data/large.txt": large_content.encode("utf-8")} - fid = _log_429(self.client.upload_file, files) - self.assertTrue(fid) + _log_429( + push_resources, + self.client, self.username, f"{AGENT_NAME}-large", "qoder", files, + ) # ----------------------------------------------------------------------- # 14. Edge: special characters in path # ----------------------------------------------------------------------- def test_20_special_chars_path(self): + from ultron.cli.sync import push_resources files = { "SOUL.md": b"# Soul\nSpecial chars test.\n", "memory/user-notes (1).md": b"# Notes\nParentheses in filename.\n", "skills/web-search-v2/SKILL.md": b"# Web Search v2\nHyphen in skill name.\n", } - fid = _log_429(self.client.upload_file, files) - self.assertTrue(fid) + _log_429( + push_resources, + self.client, self.username, f"{AGENT_NAME}-special", "qoder", files, + ) # ----------------------------------------------------------------------- # 15. Edge: visibility variants # ----------------------------------------------------------------------- def test_21_visibility_variants(self): + from ultron.cli.sync import push_resources for vis in ["public", "private"]: with self.subTest(visibility=vis): files = {"SOUL.md": f"# Soul\nVisibility={vis} test.\n".encode("utf-8")} - fid = _log_429(self.client.upload_file, files) - self.assertTrue(fid) - result = _log_429( - self.client.create_repo, - path=self.username, name=f"{AGENT_NAME}-vis-{vis}", - framework="qoder", visibility=vis, - system_prompt_files=fid, + _log_429( + push_resources, + self.client, self.username, f"{AGENT_NAME}-vis-{vis}", + "qoder", files, ) - self.assertIsInstance(result, dict) _wait_server(REQUEST_INTERVAL) # ----------------------------------------------------------------------- # 16. Edge: upload then immediate download # ----------------------------------------------------------------------- def test_22_immediate_download(self): + from ultron.cli.sync import push_resources files = {"SOUL.md": b"# Soul\nImmediate download test.\n", "README.md": b"# README\n"} - fid = _log_429(self.client.upload_file, files) _log_429( - self.client.create_repo, - path=self.username, name=AGENT_NAME, framework="qoder", - visibility="public", system_prompt_files=fid, + push_resources, + self.client, self.username, AGENT_NAME, "qoder", files, ) # Immediate list — may be empty if async immediate_files = self.client.list_repo_files(self.username, AGENT_NAME) @@ -474,6 +460,7 @@ def test_22_immediate_download(self): # 17. Framework-specific structure verification # ----------------------------------------------------------------------- def test_23_framework_structure(self): + from ultron.cli.sync import push_resources framework_markers = { "nanobot": ["AGENTS.md", "TOOLS.md", "agents/test-bot.md", "memory/MEMORY.md"], "openclaw": ["IDENTITY.md", "BOOTSTRAP.md", "memory/project-notes.md"], @@ -487,15 +474,11 @@ def test_23_framework_structure(self): with self.subTest(framework=fw): files = ALL_FRAMEWORK_FILES[fw] agent = f"{AGENT_NAME}-struct-{fw}" - fid = _log_429(self.client.upload_file, _to_bytes(files)) - try: - _log_429( - self.client.create_repo, - path=self.username, name=agent, framework=fw, - visibility="public", system_prompt_files=fid, - ) - except ApiError: - pass + _log_429( + push_resources, + self.client, self.username, agent, fw, + _to_bytes(files), + ) _wait_server(REQUEST_INTERVAL) diff --git a/tests/api/test_repo_api.py b/tests/api/test_repo_api.py index 65b74d1..03d927c 100644 --- a/tests/api/test_repo_api.py +++ b/tests/api/test_repo_api.py @@ -58,7 +58,7 @@ def tearDown(self): def test_full_roundtrip(self): # 1. create r = _run(repo.create_repo( - CreateRepoRequest(Path="alice", Name="myagent", Framework="QwenPaw", Visibility="public"), + CreateRepoRequest(Path="alice", Name="myagent", Framework="QwenPaw"), self.user, )) self.assertEqual(r["data"]["Framework"], "QwenPaw") @@ -68,7 +68,7 @@ def test_full_roundtrip(self): self.assertEqual(r["data"]["Framework"], "QwenPaw") # 3. commit a normal text file - r = _run(repo.commit_repo("alice", "myagent", CommitRequest( + r = _run(repo.commit_repo("alice", "myagent", "master", CommitRequest( commit_message="add soul", actions=[CommitAction(action="update", path="SOUL.md", type="normal", size=5, sha256="", content=_b64("hello"), encoding="base64")], @@ -89,7 +89,7 @@ def test_check_missing_repo_404(self): self.assertEqual(ctx.exception.status_code, 404) def test_create_duplicate_409(self): - req = CreateRepoRequest(Path="alice", Name="dup", Framework="OpenClaw", Visibility="public") + req = CreateRepoRequest(Path="alice", Name="dup", Framework="OpenClaw") _run(repo.create_repo(req, self.user)) with self.assertRaises(HTTPException) as ctx: _run(repo.create_repo(req, self.user)) @@ -108,9 +108,9 @@ def test_lfs_batch_rejected(self): def test_lfs_action_rejected(self): _run(repo.create_repo(CreateRepoRequest( - Path="alice", Name="a2", Framework="QwenPaw", Visibility="public"), self.user)) + Path="alice", Name="a2", Framework="QwenPaw"), self.user)) with self.assertRaises(HTTPException) as ctx: - _run(repo.commit_repo("alice", "a2", CommitRequest( + _run(repo.commit_repo("alice", "a2", "master", CommitRequest( commit_message="big", actions=[CommitAction(action="update", path="data.bin", type="lfs", size=999, sha256="abc", content="", encoding="")], @@ -119,9 +119,9 @@ def test_lfs_action_rejected(self): def test_invalid_base64_rejected(self): _run(repo.create_repo(CreateRepoRequest( - Path="alice", Name="a4", Framework="QwenPaw", Visibility="public"), self.user)) + Path="alice", Name="a4", Framework="QwenPaw"), self.user)) with self.assertRaises(HTTPException) as ctx: - _run(repo.commit_repo("alice", "a4", CommitRequest( + _run(repo.commit_repo("alice", "a4", "master", CommitRequest( commit_message="bad", actions=[CommitAction(action="update", path="X.md", type="normal", size=1, sha256="", content="!!!notbase64!!!", encoding="base64")], @@ -130,13 +130,13 @@ def test_invalid_base64_rejected(self): def test_delete_action(self): _run(repo.create_repo(CreateRepoRequest( - Path="alice", Name="a3", Framework="QwenPaw", Visibility="public"), self.user)) - _run(repo.commit_repo("alice", "a3", CommitRequest( + Path="alice", Name="a3", Framework="QwenPaw"), self.user)) + _run(repo.commit_repo("alice", "a3", "master", CommitRequest( commit_message="add", actions=[CommitAction(action="update", path="X.md", type="normal", size=1, sha256="", content=_b64("x"), encoding="base64")], ), self.user)) - r = _run(repo.commit_repo("alice", "a3", CommitRequest( + r = _run(repo.commit_repo("alice", "a3", "master", CommitRequest( commit_message="del", actions=[CommitAction(action="delete", path="X.md", type="normal", size=0, sha256="", content="", encoding="")], @@ -145,8 +145,8 @@ def test_delete_action(self): def test_list_files_non_recursive_and_root(self): _run(repo.create_repo(CreateRepoRequest( - Path="alice", Name="a5", Framework="QwenPaw", Visibility="public"), self.user)) - _run(repo.commit_repo("alice", "a5", CommitRequest( + Path="alice", Name="a5", Framework="QwenPaw"), self.user)) + _run(repo.commit_repo("alice", "a5", "master", CommitRequest( commit_message="tree", actions=[ CommitAction(action="update", path="SOUL.md", type="normal", size=1, diff --git a/tests/api/test_upload_download.py b/tests/api/test_upload_download.py index a12ef6e..3a06b20 100644 --- a/tests/api/test_upload_download.py +++ b/tests/api/test_upload_download.py @@ -44,7 +44,7 @@ # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- -SERVER = os.environ.get("SERVER", "http://pre.modelscope.cn") +SERVER = os.environ.get("SERVER", "http://www.modelscope.cn") TOKEN = os.environ.get("TOKEN", "") AGENT_PREFIX = "test-updown" diff --git a/tests/api/test_watch_sync.py b/tests/api/test_watch_sync.py index f3165da..9792493 100644 --- a/tests/api/test_watch_sync.py +++ b/tests/api/test_watch_sync.py @@ -45,7 +45,7 @@ # --------------------------------------------------------------------------- # Config # --------------------------------------------------------------------------- -SERVER = os.environ.get("SERVER", "http://pre.modelscope.cn") +SERVER = os.environ.get("SERVER", "http://www.modelscope.cn") TOKEN = os.environ.get("TOKEN", "") AGENT_PREFIX = "test-watch" REQUEST_INTERVAL = int(os.environ.get("REQUEST_INTERVAL", "8")) @@ -137,13 +137,13 @@ def _cleanup(self, path: str): def _upload_remote(self, name: str, framework: str, files: dict): """Upload files directly to remote (simulates remote-side changes).""" - # Convert str values to bytes for the new upload_file API + from ultron.cli.sync import push_resources + # Convert str values to bytes for the commit-based upload API byte_files = { k: (v.encode("utf-8") if isinstance(v, str) else v) for k, v in files.items() } - file_id = self.client.upload_file(byte_files) - self.client.create_repo(self.username, name, framework, system_prompt_files=file_id) + push_resources(self.client, self.username, name, framework, byte_files) def _start_watch(self, framework: str, agent_name: str, local_dir: str, repo_name: str, push_only: bool = True) -> multiprocessing.Process: """Start a watch_loop in a child process, return the Process.""" diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index 4036ee3..5ee4c57 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -18,21 +18,24 @@ def __init__(self, server, token=None, timeout=60): self.server = server self.token = token self.created = [] - self.uploaded_resources = None + self.uploaded_resources = {} _StubClient.instances.append(self) def check_repo(self, path, name): return False - def upload_file(self, resources): - """Accept Dict[str, bytes]; return a fake Gid.""" - self.uploaded_resources = resources - return "fake-gid-uuid" - - def create_repo(self, path, name, framework, **kwargs): - self.created.append((path, name, framework, kwargs.get("system_prompt_files"))) + def create_repo(self, path, name, framework=None): + self.created.append((path, name, framework)) return {"success": True} + def commit_files(self, path, name, actions, *, revision="master", commit_message=""): + import base64 as _b64 + for a in actions: + self.uploaded_resources[a["path"]] = _b64.b64decode(a["content"]) + + def upload_lfs_file(self, path, name, file_path, content, *, action="create", revision="master", commit_message=""): + self.uploaded_resources[file_path] = content + def _run(argv): args = build_parser().parse_args(argv) @@ -84,10 +87,9 @@ def test_full_upload_creates_then_uploads_zip(self, *_): self.assertEqual(rc, 0) self.assertEqual(len(_StubClient.instances), 1) client = _StubClient.instances[0] - # create_repo called with (group, repo_name, framework, system_prompt_files) + # create_repo called with (group, repo_name, framework) self.assertEqual(len(client.created), 1) - self.assertEqual(client.created[0][:3], ("u", "qoder-reviewer", "qoder")) - self.assertEqual(client.created[0][3], "fake-gid-uuid") + self.assertEqual(client.created[0], ("u", "qoder-reviewer", "qoder")) # Verify uploaded resources are bytes-valued dict self.assertIsNotNone(client.uploaded_resources) self.assertIsInstance(client.uploaded_resources, dict) diff --git a/ultron/api/routers/repo.py b/ultron/api/routers/repo.py index 37ca3f5..0557453 100644 --- a/ultron/api/routers/repo.py +++ b/ultron/api/routers/repo.py @@ -26,6 +26,7 @@ CreateRepoRequest, LfsBatchRequest, ) +from pydantic import BaseModel, Field router = APIRouter(prefix="/api/v1", tags=["repo"]) @@ -86,10 +87,11 @@ async def create_repo( if existing: raise HTTPException(status_code=409, detail="Repository already exists") # Initialize an empty profile carrying the framework/product. + product = request.Framework or "nanobot" data = u.harness_sync_up( user_id=request.Path, agent_id=request.Name, - product=request.Framework, + product=product, resources={}, ) return { @@ -97,8 +99,7 @@ async def create_repo( "data": { "Path": request.Path, "Name": request.Name, - "Framework": request.Framework, - "Visibility": request.Visibility, + "Framework": product, "Revision": data.get("revision"), }, } @@ -128,10 +129,11 @@ async def lfs_batch( # ---- 3.2 Commit files ---- -@router.post("/repos/agents/{path}/{name}/commit/master") +@router.post("/repos/agents/{path}/{name}/commit/{revision}") async def commit_repo( path: str, name: str, + revision: str, request: CommitRequest, user: dict = Depends(get_current_user), ): @@ -145,7 +147,7 @@ async def commit_repo( if action.action == "delete": resources.pop(action.path, None) continue - if action.action != "update": + if action.action not in ("create", "update"): raise HTTPException( status_code=400, detail=f"Unsupported action '{action.action}' for {action.path}", @@ -187,6 +189,42 @@ async def commit_repo( # ---- 4.1 List repository files ---- +class DeleteFileRequest(BaseModel): + branch: str = Field("master", description="Branch name") + file_path: str = Field(..., description="File path to delete") + commit_message: str = Field("", description="Commit message") + + +@router.delete("/agents/{path}/{name}/repo/file") +async def delete_repo_file( + path: str, + name: str, + request: DeleteFileRequest, + user: dict = Depends(get_current_user), +): + """Delete a single file from the repo.""" + _ensure_owner(path, user) + u = _ultron() + profile = _get_profile_or_404(u, path, name) + resources = dict(profile.get("resources", {})) + product = profile.get("product", "nanobot") + + if request.file_path not in resources: + raise HTTPException(status_code=404, detail=f"File not found: {request.file_path}") + + resources.pop(request.file_path) + u.harness_sync_up( + user_id=path, agent_id=name, product=product, resources=resources + ) + return { + "success": True, + "data": { + "deleted": request.file_path, + "files": len(resources), + }, + } + + @router.get("/agents/{path}/{name}/repo/files") async def list_repo_files( path: str, diff --git a/ultron/api/schemas.py b/ultron/api/schemas.py index 46a3008..6aeb69b 100644 --- a/ultron/api/schemas.py +++ b/ultron/api/schemas.py @@ -110,10 +110,9 @@ class CreateRepoRequest(BaseModel): Path: str = Field(..., description="Repo path (user/org); must match the caller") Name: str = Field(..., description="Repo name; stored as agent_id") - Framework: str = Field( - ..., description="Framework/product, e.g. OpenClaw / QwenPaw / nanobot" + Framework: Optional[str] = Field( + None, description="Framework/product, e.g. OpenClaw / QwenPaw / nanobot (optional)" ) - Visibility: str = Field("public", description="Visibility: public / private") class LfsObject(BaseModel): @@ -127,18 +126,18 @@ class LfsBatchRequest(BaseModel): class CommitAction(BaseModel): - action: str = Field(..., description="Operation: update / delete") + action: str = Field(..., description="Operation: create / update / delete") path: str = Field(..., description="File path within the repo") type: str = Field( "normal", description="File type: normal (regular) / lfs (large)" ) size: int = Field(0, description="File size in bytes") - sha256: str = Field("", description="SHA256 (required for lfs, may be empty)") + sha256: str = Field("", description="SHA256 (required for lfs, may be empty for normal)") content: str = Field( - "", description="Content: base64 for normal files, empty for lfs" + "", description="Content: base64 for normal files, empty for lfs/delete" ) encoding: str = Field( - "", description="Encoding: base64 for normal files, empty for lfs" + "", description="Encoding: base64 for normal files, empty for lfs/delete" ) diff --git a/ultron/cli/client.py b/ultron/cli/client.py index c03a196..de3b0eb 100644 --- a/ultron/cli/client.py +++ b/ultron/cli/client.py @@ -5,13 +5,17 @@ * ``GET /openapi/v1/users/me`` → login * ``GET /openapi/v1/agents/{path}/{name}`` → repo metadata -* ``POST /openapi/v1/agents`` → create/update agent -* ``GET /openapi/v1/agents/{path}/{name}/repo/files`` → list files +* ``POST /openapi/v1/agents`` → create empty agent +* ``GET /api/v1/agents/{path}/{name}/repo/files`` → list files * ``GET /agents/{path}/{name}/resolve/{rev}/{file}`` → file download -* ``POST /api/v1/agents/repo/files/upload`` → two-step OSS upload (step1) -* ``POST /openapi/v1/agents/{path}/{name}/commit/{rev}`` → commit files +* ``POST /api/v1/repos/agents/{id}/commit/{rev}`` → commit files (normal/lfs) +* ``POST /api/v1/repos/agents/{id}/info/lfs/objects/batch`` → LFS batch verify +* ``DELETE /api/v1/agents/{path}/{name}/repo/file`` → delete file """ +import base64 +import hashlib import logging +import os from dataclasses import dataclass from typing import Dict, List, Optional from urllib.parse import unquote @@ -23,6 +27,37 @@ logger = logging.getLogger("ultron.cli") +# LFS file extensions that must use LFS upload pathway. +_LFS_EXTENSIONS: frozenset = frozenset({ + ".7z", ".aac", ".arrow", ".audio", ".bin", ".bmp", ".bz2", + ".ckpt", ".flac", ".ftz", ".gif", ".gz", ".h5", + ".jack", ".jpeg", ".jpg", ".joblib", ".jsonl", + ".lz4", ".mlmodel", ".model", ".mp3", ".mp4", ".msgpack", + ".npy", ".npz", ".ogg", ".onnx", ".ot", + ".parquet", ".pb", ".pcm", ".pickle", ".pkl", ".png", + ".pt", ".pth", ".rar", ".raw", + ".safetensors", ".sam", ".tar", ".tflite", ".tgz", ".tiff", + ".wasm", ".wav", ".webm", ".webp", ".xz", ".zip", ".zst", +}) + +# Files larger than this threshold (bytes) use LFS upload. +_LFS_SIZE_THRESHOLD: int = 1 * 1024 * 1024 # 1 MB + + +def is_lfs_file(file_path: str, size: int) -> bool: + """Determine whether a file should use LFS upload. + + A file is considered LFS if: + 1. Its extension is in the known LFS extension set, OR + 2. Its size exceeds the LFS threshold (1 MB). + """ + ext = os.path.splitext(file_path)[1].lower() + if ext in _LFS_EXTENSIONS: + return True + if size > _LFS_SIZE_THRESHOLD: + return True + return False + @dataclass class RemoteFileInfo: @@ -30,6 +65,7 @@ class RemoteFileInfo: path: str sha256: str committed_date: int # unix timestamp + is_lfs: bool = False class ApiError(Exception): @@ -119,24 +155,20 @@ def list_agents(self, owner: Optional[str] = None, page_number: int = 1, page_si return {"items": data, "total_count": len(data)} return {"items": [], "total_count": 0} - def create_repo( - self, path: str, name: str, framework: str, - visibility: str = "public", - system_prompt_files: Optional[str] = None, - ) -> dict: - """Create or update an agent (POST /agents). + def create_repo(self, path: str, name: str, framework: str | None = None) -> dict: + """Create an empty agent (POST /agents). + + The server creates a bare repository. Files are added separately via + :meth:`commit_files`. - When *system_prompt_files* is provided the server uses the uploaded - file as the agent content ("新增和更新重叠使用本方法"). + Args: + framework: Optional product/framework identifier stored with the + repo (e.g. "qoder", "nanobot"). Defaults to server-side + default when omitted. """ - body: dict = { - "path": path, - "name": name, - "framework": framework, - "visibility": visibility, - } - if system_prompt_files: - body["system_prompt_files"] = system_prompt_files + body: dict = {"path": path, "name": name} + if framework: + body["framework"] = framework try: return self._openapi._request("POST", "/agents", json_body=body) except HubError as exc: @@ -148,11 +180,7 @@ def list_repo_files(self, path: str, name: str, revision: str = 'master') -> Lis return [e["path"] for e in entries if e["type"] == "blob" and e["path"]] def list_repo_files_detail(self, path: str, name: str, revision: str = 'master') -> List[RemoteFileInfo]: - """All blob files with sha256 and committed_date. - - Returns a list of ``RemoteFileInfo`` for each blob entry in the repo. - Raises ``ApiError(404, ...)`` if the repo does not exist. - """ + """All blob files with sha256, committed_date, and is_lfs flag.""" entries = self._fetch_tree_entries(path, name, revision) results: List[RemoteFileInfo] = [] for item in entries: @@ -162,6 +190,7 @@ def list_repo_files_detail(self, path: str, name: str, revision: str = 'master') path=item["path"], sha256=item.get("sha256") or "", committed_date=int(item.get("committed_date") or 0), + is_lfs=bool(item.get("is_lfs", False)), )) return results @@ -172,10 +201,11 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] max_pages = 50 # safety cap: 5000 files max all_entries: List[dict] = [] + list_url = f"{self.server}/api/v1/agents/{path}/{name}/repo/files" while True: try: data = self._openapi._request( - "GET", f"/agents/{path}/{name}/repo/files", + "GET", url=list_url, params={ "recursive": "true", "page_size": str(page_size), @@ -200,6 +230,7 @@ def _fetch_tree_entries(self, path: str, name: str, revision: str) -> List[dict] "type": item.get("type") or item.get("Type") or "", "sha256": item.get("sha256") or item.get("Sha256") or "", "committed_date": item.get("committed_date") or item.get("Committed_date") or 0, + "is_lfs": bool(item.get("IsLfs") or item.get("is_lfs") or False), }) if len(raw) < page_size: @@ -234,70 +265,62 @@ def download_repo_file(self, path: str, name: str, file_path: str, raise ApiError(0, str(exc)) from exc return resp.content if binary else resp.text - # ---- upload (two-step OSS) ---- + # ---- commit (normal + LFS) ---- - def _request_upload_urls(self, filenames: List[str]) -> dict: - """Step 1: POST /api/v1/agents/repo/files/upload → {Gid, Urls}. - - Uses /api/v1/ prefix (not /openapi/v1/). Response envelope uses - capitalised keys: {"Code": 200, "Data": {...}, "Success": true}. + def commit_files(self, path: str, name: str, actions: List[dict], + revision: str = "master", commit_message: str = "sync") -> dict: + """Commit file changes via POST /api/v1/repos/agents/{path}/{name}/commit/{revision}. + + Each action dict should contain: + - action: "create" | "update" | "delete" + - path: file path in repo + - type: "normal" | "lfs" (for create/update) + - size: file size in bytes (for create/update) + - sha256: sha256 hash (required for lfs; empty string for normal) + - content: base64-encoded content (for normal) or empty (for lfs) + - encoding: "base64" (for normal) or "" (for lfs) """ - url = f"{self.server}/api/v1/agents/repo/files/upload" - headers = {"Authorization": f"Bearer {self.token}", - "Content-Type": "application/json"} + commit_url = f"{self.server}/api/v1/repos/agents/{path}/{name}/commit/{revision}" + body = {"commit_message": commit_message, "actions": actions} try: - resp = requests.post(url, json={"FileNames": filenames}, - headers=headers, timeout=self.timeout) - resp.raise_for_status() - except requests.HTTPError as exc: - status = exc.response.status_code if exc.response is not None else 0 - detail = exc.response.text if exc.response is not None else str(exc) - raise ApiError(status, detail) from exc - except requests.RequestException as exc: - raise ApiError(0, str(exc)) from exc - body = resp.json() - if not body.get("Success"): - raise ApiError(body.get("Code", 0), body.get("Message", "upload credential failed")) - return body["Data"] - - @staticmethod - def _normalize_oss_url(url: str) -> str: - """Decode %2F in the URL path so OSS signature verification passes. - - The server may return signed URLs with path separators encoded as %2F. - OSS computes the signature on the *decoded* resource path, so we must - send the request with real '/' in the path. We only decode the path - portion (before '?') to avoid corrupting query-string parameters. - """ - parts = url.split("?", 1) - path_part = parts[0] - if "%2F" not in path_part and "%2f" not in path_part: - return url - decoded_path = unquote(path_part) - if len(parts) == 2: - return decoded_path + "?" + parts[1] - return decoded_path - - def _upload_to_oss(self, signed_url: str, data: bytes) -> None: - """Step 2: PUT raw bytes to signed OSS URL. - - The server signs the URL with these headers in the StringToSign: - - Content-Type: application/octet-stream - - x-oss-meta-author: aliy (included in CanonicalizedOSSHeaders) - Both MUST be present for the signature to match. - - COUPLING: These headers are dictated by the server-side signing config. - If the server changes its signing parameters, these must be updated - in lockstep. + return self._openapi._request("POST", url=commit_url, json_body=body) + except HubError as exc: + raise _wrap(exc) from exc + + def lfs_batch(self, path: str, name: str, oid: str, size: int) -> Optional[str]: + """LFS batch verify and return upload URL (or None if already exists). + + POST /api/v1/repos/agents/{path}/{name}/info/lfs/objects/batch + Returns the upload href if the server needs the blob, None otherwise. """ - url = self._normalize_oss_url(signed_url) + batch_url = ( + f"{self.server}/api/v1/repos/agents/{path}/{name}" + f"/info/lfs/objects/batch" + ) + body = { + "operation": "upload", + "objects": [{"oid": oid, "size": size}], + } + try: + data = self._openapi._request("POST", url=batch_url, json_body=body) + except HubError as exc: + raise _wrap(exc) from exc + objects = [] + if isinstance(data, dict): + objects = data.get("objects") or [] + if not objects: + return None + upload_info = objects[0].get("actions", {}).get("upload", {}) + return upload_info.get("href") or None + + def lfs_upload_blob(self, upload_url: str, data: bytes) -> None: + """PUT binary data to the LFS upload URL.""" try: - resp = requests.put(url, data=data, - headers={ - "Content-Type": "application/octet-stream", - "x-oss-meta-author": "aliy", - }, - timeout=max(self.timeout, 300)) + resp = requests.put( + upload_url, data=data, + headers={"Content-Type": "application/octet-stream"}, + timeout=max(self.timeout, 300), + ) resp.raise_for_status() except requests.HTTPError as exc: status = exc.response.status_code if exc.response is not None else 0 @@ -306,46 +329,47 @@ def _upload_to_oss(self, signed_url: str, data: bytes) -> None: except requests.RequestException as exc: raise ApiError(0, str(exc)) from exc - def upload_file(self, resources: Dict[str, bytes]) -> str: - """Two-step upload: get signed URLs → PUT to OSS → return Gid. - - The returned Gid (UUID) is used as ``system_prompt_files`` in - :meth:`create_repo`. - - Returns empty string if *resources* is empty (nothing to upload). + def upload_lfs_file(self, path: str, name: str, file_path: str, + content: bytes, action: str = "create", + revision: str = "master", + commit_message: str = "sync") -> dict: + """Full LFS upload flow: batch verify -> PUT blob -> commit reference.""" + oid = hashlib.sha256(content).hexdigest() + size = len(content) + + # Step 1: batch verify + upload_url = self.lfs_batch(path, name, oid, size) + # Step 2: PUT blob if needed + if upload_url: + self.lfs_upload_blob(upload_url, content) + + # Step 3: commit LFS reference + actions = [{ + "action": action, + "path": file_path, + "type": "lfs", + "size": size, + "sha256": oid, + "content": "", + "encoding": "", + }] + return self.commit_files(path, name, actions, revision=revision, + commit_message=commit_message) + + def delete_file(self, path: str, name: str, file_path: str, + revision: str = "master", + commit_message: Optional[str] = None) -> dict: + """Delete a file from the repo. + + DELETE /api/v1/agents/{path}/{name}/repo/file """ - if not resources: - logger.warning("upload_file called with empty resources; skipping.") - return "" - filenames = list(resources.keys()) - data = self._request_upload_urls(filenames) - gid = data["Gid"] - url_map = {item["Filename"]: item["Url"] for item in data["Urls"]} - for fname, content in resources.items(): - signed_url = url_map.get(fname) - if not signed_url: - raise ApiError( - 0, - f"Server did not return a signed URL for '{fname}'. " - f"Available: {list(url_map.keys())}", - ) - self._upload_to_oss(signed_url, content) - return gid - - # ---- commit (incremental) ---- - - def commit_files(self, path: str, name: str, actions: List[dict], - revision: str = "master", commit_message: str = "sync") -> dict: - """Commit file changes via POST /openapi/v1/agents/{path}/{name}/commit/{revision}. - - *actions* example:: - - [{"action": "create", "file_path": "a.md", - "content": "hello", "encoding": "text"}] - """ - body = {"commit_message": commit_message, "actions": actions} + delete_url = f"{self.server}/api/v1/agents/{path}/{name}/repo/file" + body = { + "branch": revision, + "file_path": file_path, + "commit_message": commit_message or f"Delete {file_path}", + } try: - return self._openapi._request( - "POST", f"/agents/{path}/{name}/commit/{revision}", json_body=body) + return self._openapi._request("DELETE", url=delete_url, json_body=body) except HubError as exc: raise _wrap(exc) from exc diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 97d3622..c375a26 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -282,16 +282,14 @@ def cmd_upload(args) -> int: username=username, ) - # Step 1: upload files -> get file_id + # Step 1: upload files via commit interface try: - file_id = client.upload_file(resources) - # Step 2: create/update agent with file_id - result = client.create_repo( - group, repo, framework, - system_prompt_files=file_id, - ) + from .sync import push_resources + push_resources(client, group, repo, framework, resources) except ApiError as e: return _fail(_api_error_message(e, "upload")) + except Exception as e: + return _fail(f"upload failed: {e}") print( f"\nUploaded {len(resources)} file(s) to " diff --git a/ultron/cli/sync.py b/ultron/cli/sync.py index 36f8950..b5d195d 100644 --- a/ultron/cli/sync.py +++ b/ultron/cli/sync.py @@ -19,11 +19,10 @@ def zip_resources(resources: Dict[str, Union[str, bytes]], wrapper: str = "agent") -> bytes: - """Pack resources into a deterministic in-memory zip. + """Pack resources into a deterministic in-memory zip for local backup. - The server always strips the first directory level from zip entries, so we - wrap all files under a top-level folder (``wrapper/``). This ensures that - after stripping, the remaining path matches the original ``rel_path``. + Used by :func:`backup_local` to create timestamped backups before + destructive operations (pull, convert, restore). Args: resources: A dict {rel_path: content}. Values are written directly @@ -89,22 +88,60 @@ def push_resources( framework: str, resources: Dict[str, bytes], ) -> None: - """Full upload via two-step OSS, then create/update agent repo. + """Full upload via commit interface (normal + LFS). + Creates the repo if needed, then commits all files in batches. Raises on failure (caller should NOT update baseline on exception). - Does nothing if *resources* is empty. """ + from .client import is_lfs_file + if not resources: logger.warning("push_resources called with empty resources; skipping.") return - gid = client.upload_file(resources) - if not gid: - logger.warning("upload_file returned empty gid; skipping create_repo.") - return - client.create_repo(username, name, framework, system_prompt_files=gid) - for rel in sorted(resources): - logger.info(" UPLOAD: %s (%d B)", rel, len(resources[rel])) - logger.info("Pushed %d file(s) via OSS (gid=%s).", len(resources), gid) + + # Ensure repo exists (idempotent create). + try: + if not client.check_repo(username, name): + client.create_repo(username, name, framework=framework) + logger.info("Created empty agent repo %s/%s (framework=%s).", username, name, framework) + except Exception as exc: + logger.warning("create_repo check failed (%s), proceeding anyway.", exc) + + # Split into normal and LFS files. + normal_actions: List[dict] = [] + lfs_files: List[tuple] = [] + + for rel, content in sorted(resources.items()): + size = len(content) + if is_lfs_file(rel, size): + lfs_files.append((rel, content)) + else: + b64 = base64.b64encode(content).decode("ascii") + normal_actions.append({ + "action": "create", + "path": rel, + "type": "normal", + "size": size, + "sha256": "", + "content": b64, + "encoding": "base64", + }) + + # Commit normal files in one request. + if normal_actions: + client.commit_files(username, name, normal_actions, + commit_message="sync: upload normal files") + for a in normal_actions: + logger.info(" CREATE: %s (%d B)", a["path"], a["size"]) + + # Upload LFS files one-by-one (batch verify + PUT + commit). + for rel, content in lfs_files: + client.upload_lfs_file(username, name, rel, content, + action="create", commit_message=f"sync: upload LFS {rel}") + logger.info(" CREATE (LFS): %s (%d B)", rel, len(content)) + + logger.info("Pushed %d file(s) (%d normal, %d LFS).", + len(resources), len(normal_actions), len(lfs_files)) def push_incremental( @@ -113,32 +150,67 @@ def push_incremental( name: str, changed: Dict[str, Union[bytes, None]], remote_paths: set, + remote_lfs_paths: set = None, ) -> None: """Incremental push via commit interface. - Builds create/update/delete actions and commits in one request. - Raises on failure (caller should NOT update baseline on exception). + Builds create/update/delete actions and commits. LFS files are + uploaded via the LFS batch+PUT flow before committing their reference. """ - actions: List[dict] = [] + from .client import is_lfs_file + + normal_actions: List[dict] = [] + lfs_items: List[tuple] = [] # (path, content, action_type) + delete_paths: List[str] = [] + for fpath, content in changed.items(): - if content is None: # None = delete - actions.append({"action": "delete", "file_path": fpath}) + if content is None: + delete_paths.append(fpath) else: action_type = "update" if fpath in remote_paths else "create" - # Try UTF-8 decode; fall back to base64 for binary. - try: - text = content.decode("utf-8") - actions.append({"action": action_type, "file_path": fpath, - "content": text, "encoding": "text"}) - except UnicodeDecodeError: + size = len(content) + # Determine if the file needs LFS: check remote flag or local heuristic. + use_lfs = False + if remote_lfs_paths and fpath in remote_lfs_paths: + use_lfs = True + elif is_lfs_file(fpath, size): + use_lfs = True + + if use_lfs: + lfs_items.append((fpath, content, action_type)) + else: b64 = base64.b64encode(content).decode("ascii") - actions.append({"action": action_type, "file_path": fpath, - "content": b64, "encoding": "base64"}) - if actions: - for a in actions: - logger.info(" %s: %s", a["action"].upper(), a["file_path"]) - client.commit_files(username, name, actions, commit_message="watch sync") - logger.info("Committed %d action(s) incrementally.", len(actions)) + normal_actions.append({ + "action": action_type, + "path": fpath, + "type": "normal", + "size": size, + "sha256": "", + "content": b64, + "encoding": "base64", + }) + + # Commit normal file actions in one batch. + if normal_actions: + for a in normal_actions: + logger.info(" %s: %s", a["action"].upper(), a["path"]) + client.commit_files(username, name, normal_actions, + commit_message="watch sync") + + # Upload LFS files one-by-one. + for fpath, content, action_type in lfs_items: + logger.info(" %s (LFS): %s", action_type.upper(), fpath) + client.upload_lfs_file(username, name, fpath, content, + action=action_type, commit_message="watch sync") + + # Delete files via the DELETE endpoint. + for fpath in delete_paths: + logger.info(" DELETE: %s", fpath) + client.delete_file(username, name, fpath) + + total = len(normal_actions) + len(lfs_items) + len(delete_paths) + if total: + logger.info("Committed %d action(s) incrementally.", total) def pull_incremental( diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index 629355a..912b10b 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -108,8 +108,7 @@ def _handle_term(signum, frame): remote_sha_map = {f.path: f.sha256 for f in remote_files if f.path in scope} remote_changed = ( - max((f.committed_date for f in remote_files), default=0) > state["last_commit_date"] - or set(remote_sha_map.keys()) != set(state.get("remote_files", {}).keys()) + remote_sha_map != state.get("remote_files", {}) ) local_changed = bool(detect_local_changes(local_resources, state["remote_files"])) @@ -140,7 +139,7 @@ def _handle_term(signum, frame): sf.unlink(missing_ok=True) -def _push_local(client, username, name, framework, local_resources, state, logger, *, remote_paths=None) -> bool: +def _push_local(client, username, name, framework, local_resources, state, logger, *, remote_paths=None, remote_lfs_paths=None) -> bool: """Push local changes: full upload on first time, incremental thereafter. Returns True if something was actually pushed, False otherwise. @@ -171,7 +170,8 @@ def _push_local(client, username, name, framework, local_resources, state, logge return False # Use actual remote paths (not stale baseline) for CREATE vs UPDATE. actual = remote_paths if remote_paths is not None else set(state["remote_files"].keys()) - push_incremental(client, username, name, changed, actual) + push_incremental(client, username, name, changed, actual, + remote_lfs_paths=remote_lfs_paths) logger.info("Pushed local changes (incremental commit).") return True return False @@ -185,11 +185,13 @@ def _sync_action( ) -> bool: """Execute the appropriate sync action. Returns True if something changed.""" remote_paths = {f.path for f in remote_files} + remote_lfs_paths = {f.path for f in remote_files if getattr(f, 'is_lfs', False)} if push_only: if not local_changed: return False - return _push_local(client, username, name, framework, local_resources, state, logger, remote_paths=remote_paths) + return _push_local(client, username, name, framework, local_resources, state, logger, + remote_paths=remote_paths, remote_lfs_paths=remote_lfs_paths) if remote_changed and local_changed: backup_path = backup_local(spec, name) @@ -200,7 +202,8 @@ def _sync_action( pull_incremental(client, username, name, spec, remote_files, local_resources) logger.info("Pulled remote changes (backup: %s).", backup_path) elif local_changed: - _push_local(client, username, name, framework, local_resources, state, logger, remote_paths=remote_paths) + _push_local(client, username, name, framework, local_resources, state, logger, + remote_paths=remote_paths, remote_lfs_paths=remote_lfs_paths) else: return False return True From a5a12465ad900949f7fec8e9200c1fe95fc2f15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Wed, 8 Jul 2026 10:41:32 +0800 Subject: [PATCH 11/14] fix --- ultron/cli/__init__.py | 3 +- ultron/cli/commands.py | 2 +- ultron/cli/watcher.py | 114 +++++++++++++++++------------------------ 3 files changed, 49 insertions(+), 70 deletions(-) diff --git a/ultron/cli/__init__.py b/ultron/cli/__init__.py index fd7ac5a..49e945c 100644 --- a/ultron/cli/__init__.py +++ b/ultron/cli/__init__.py @@ -253,6 +253,7 @@ def _run_watch_daemon(param_path: str) -> int: interval = payload.get("interval", 120) push_only = payload.get("push_only", True) local_name = payload.get("local_name") or ALL_AGENT_NAME + local_dir = payload.get("local_dir") or None if not repo: repo = "default" @@ -264,7 +265,7 @@ def _run_watch_daemon(param_path: str) -> int: if not server or not token or not username: return 1 - spec = _build_allowlist(framework, local_name, None) + spec = _build_allowlist(framework, local_name, local_dir) client = UltronClient(server, token) # Redirect stdout/stderr to log file. diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index c375a26..5028e89 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -528,7 +528,7 @@ def cmd_watch(args) -> int: return _fail(_api_error_message(e, "watch")) # repo not found or unreachable — proceed, first push will create it - interval = 120 + interval = 60 push_only = not getattr(args, "pull", False) print(f"Starting sync for {group}/{repo} (interval={interval}s)...") print(f" Framework: {framework}") diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index 912b10b..27aaa4f 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -47,9 +47,9 @@ def watch_loop(spec, client, username: str, repo: str, framework: str, interval: """ logger = _get_logger() - # After double-fork, the parent's requests.Session connection pool holds - # stale file descriptors that cause EBADF on new connections. Rebuild the - # entire client so the daemon starts with a fresh session. + # The daemon runs in a freshly exec'd interpreter, but watch_loop may also + # be invoked directly (tests / foreground). Rebuild the client so we always + # start with a clean requests.Session connection pool. client = UltronClient(client.server, client.token, client.timeout) logger.info("Watch started for %s/%s (root=%s, interval=%ds, push_only=%s)", @@ -234,64 +234,24 @@ def _refresh_baseline(client, username: str, name: str, local_resources: dict, s def daemonize(target, *args, **kwargs): - """Launch *target* as a background process. - - Unix: classic double-fork. - Windows: subprocess.Popen with DETACHED_PROCESS. - """ - if hasattr(os, "fork"): - _daemonize_unix(target, *args, **kwargs) - else: - _daemonize_windows(target, *args, **kwargs) - - -def _daemonize_unix(target, *args, **kwargs): - """Double-fork daemon (Unix only).""" - pf = pid_file() - - pid = os.fork() - if pid > 0: - return # Parent returns immediately. - - os.setsid() - - pid = os.fork() - if pid > 0: - os._exit(0) # First child exits; grandchild is the actual daemon. - - # Grandchild: write PID and redirect stdio. - pf.write_text(str(os.getpid()), encoding="utf-8") - - sys.stdout.flush() - sys.stderr.flush() - with open(os.devnull, "r") as devnull: - os.dup2(devnull.fileno(), sys.stdin.fileno()) - log_fd = os.open(str(log_file()), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) - os.dup2(log_fd, sys.stdout.fileno()) - os.dup2(log_fd, sys.stderr.fileno()) - os.close(log_fd) - - try: - target(*args, **kwargs) - finally: - pf.unlink(missing_ok=True) - os._exit(0) - - -def _daemonize_windows(target, *args, **kwargs): - """Spawn a detached background process (Windows). - - Re-launches Python with ``ultron _watch_daemon`` internal command, - passing serialized arguments via a temp JSON file. + """Launch the watch loop as a fresh background process (fork + exec). + + Spawns a brand-new Python interpreter running the ``_watch_daemon`` entry + point on ALL platforms. Using exec (rather than a bare ``os.fork()``) + guarantees the daemon starts from a clean process image. This is essential + on macOS, where calling into system frameworks (e.g. ``_scproxy`` / + ``SystemConfiguration`` for proxy detection during an HTTPS request) inside + a fork-without-exec child crashes with SIGSEGV. It also avoids stale file + descriptors inherited from the parent's connection pool. """ import json import tempfile - # Serialize the arguments that watch_loop needs. - # spec (args[0]) carries the agent_name used to build the allowlist scope. + # spec (args[0]) carries the agent_name/local_dir used to build the scope. # client (args[1]) carries server/token for the child process. spec_obj = args[0] if len(args) > 0 else None client_obj = args[1] if len(args) > 1 else None + local_dir = getattr(spec_obj, "_local_dir", None) if spec_obj else None payload = { "username": args[2] if len(args) > 2 else kwargs.get("username", ""), "repo": args[3] if len(args) > 3 else kwargs.get("repo", ""), @@ -299,28 +259,46 @@ def _daemonize_windows(target, *args, **kwargs): "interval": args[5] if len(args) > 5 else kwargs.get("interval", 120), "push_only": kwargs.get("push_only", True), "local_name": getattr(spec_obj, "agent_name", "") if spec_obj else "", + "local_dir": str(local_dir) if local_dir else "", "server": getattr(client_obj, "server", "") if client_obj else "", "token": getattr(client_obj, "token", "") if client_obj else "", - # spec and client are rebuilt in the child from stored config. } - # Write to a temp file that the child will read and delete. fd, param_path = tempfile.mkstemp(suffix=".json", prefix="ultron_watch_") with os.fdopen(fd, "w") as f: json.dump(payload, f) - # Launch detached subprocess. - CREATE_NO_WINDOW = 0x08000000 - DETACHED_PROCESS = 0x00000008 - proc = subprocess.Popen( - [sys.executable, "-m", "ultron.cli", "_watch_daemon", param_path], - creationflags=DETACHED_PROCESS | CREATE_NO_WINDOW, - close_fds=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - stdin=subprocess.DEVNULL, - ) - # Write PID file. + cmd = [sys.executable, "-m", "ultron.cli", "_watch_daemon", param_path] pf = pid_file() + + if hasattr(os, "fork"): + # Unix: detach into a new session (setsid equivalent) and redirect the + # daemon's stdio to the log file so tracebacks are never lost. + lf = log_file() + lf.parent.mkdir(parents=True, exist_ok=True) + log_fd = os.open(str(lf), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) + try: + proc = subprocess.Popen( + cmd, + start_new_session=True, + close_fds=True, + stdin=subprocess.DEVNULL, + stdout=log_fd, + stderr=log_fd, + ) + finally: + os.close(log_fd) + else: + CREATE_NO_WINDOW = 0x08000000 + DETACHED_PROCESS = 0x00000008 + proc = subprocess.Popen( + cmd, + creationflags=DETACHED_PROCESS | CREATE_NO_WINDOW, + close_fds=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + ) + pf.write_text(str(proc.pid), encoding="utf-8") From cc9ceed40f40b29a8337a24d505864155ef686e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 9 Jul 2026 18:52:37 +0800 Subject: [PATCH 12/14] fix --- tests/cli/test_download_convert.py | 77 ++++++++++++++++++++++++ tests/test_harness.py | 36 +++++++++++ ultron/cli/commands.py | 89 ++++++++++++++++++++++++---- ultron/services/harness/allowlist.py | 62 +++++++++++++++++++ 4 files changed, 251 insertions(+), 13 deletions(-) diff --git a/tests/cli/test_download_convert.py b/tests/cli/test_download_convert.py index 0b08abf..355b31f 100644 --- a/tests/cli/test_download_convert.py +++ b/tests/cli/test_download_convert.py @@ -33,11 +33,39 @@ def download_repo_file(self, path, name, file_path): return self.STORE[file_path] +class _QwenpawAllStub: + """Serves a qwenpaw all-mode repo (agent-prefixed paths) for convert tests.""" + + instances = [] + STORE = { + ".gitattributes": "x", + "README.md": "readme", + "default/AGENTS.md": "# default agents", + "default/SOUL.md": "# default soul", + "bot-a/AGENTS.md": "# bot-a agents", + "bot-a/SOUL.md": "# bot-a soul", + "bot-a/PROFILE.md": "# bot-a profile", + } + + def __init__(self, server, token=None, timeout=60): + _QwenpawAllStub.instances.append(self) + + def repo_info(self, path, name): + return {"Path": path, "Name": name, "Framework": "qwenpaw", "Revision": 1} + + def list_repo_files(self, path, name): + return list(self.STORE) + + def download_repo_file(self, path, name, file_path): + return self.STORE[file_path] + + class TestDownload(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.out = Path(self.tmp.name) / "ws" _DownloadStub.instances = [] + _QwenpawAllStub.instances = [] def tearDown(self): self.tmp.cleanup() @@ -136,6 +164,55 @@ def test_download_repo_with_slash(self, *_): # Should still write files (stub doesn't care about group). self.assertTrue((self.out / "SOUL.md").is_file()) + @mock.patch.object(commands.config, "resolve_username", return_value="u") + @mock.patch.object(commands.config, "resolve_token", return_value="tok") + @mock.patch.object(commands.config, "resolve_server", return_value="http://s") + @mock.patch.object(commands, "UltronClient", _QwenpawAllStub) + def test_download_convert_all_root_to_root(self, *_): + """qwenpaw -> openclaw with --name all: per-agent convert + re-prefix.""" + rc = _run([ + "download", "--repo", "qw", "--framework", "qwenpaw", + "--name", "all", "--target", "openclaw", "--local_dir", str(self.out), + ]) + self.assertEqual(rc, 0) + # default -> workspace/, bot-a -> workspace-bot-a/ (openclaw convention) + self.assertTrue((self.out / "workspace" / "AGENTS.md").is_file()) + self.assertTrue((self.out / "workspace-bot-a" / "AGENTS.md").is_file()) + self.assertTrue((self.out / "workspace-bot-a" / "SOUL.md").is_file()) + # qwenpaw-only PROFILE.md has no openclaw equivalent: must NOT land as-is. + self.assertFalse((self.out / "workspace-bot-a" / "PROFILE.md").exists()) + # top-level non-agent files (README) are dropped, never mis-prefixed. + self.assertFalse((self.out / "README.md").exists()) + + @mock.patch.object(commands.config, "resolve_username", return_value="u") + @mock.patch.object(commands.config, "resolve_token", return_value="tok") + @mock.patch.object(commands.config, "resolve_server", return_value="http://s") + @mock.patch.object(commands, "UltronClient", _QwenpawAllStub) + def test_download_convert_all_cross_layout_rejected(self, *_): + """qwenpaw -> qoder with --name all is cross-layout: must be rejected.""" + rc = _run([ + "download", "--repo", "qw", "--framework", "qwenpaw", + "--name", "all", "--target", "qoder", "--local_dir", str(self.out), + ]) + self.assertEqual(rc, 1) + + @mock.patch.object(commands.config, "resolve_username", return_value="u") + @mock.patch.object(commands.config, "resolve_token", return_value="tok") + @mock.patch.object(commands.config, "resolve_server", return_value="http://s") + @mock.patch.object(commands, "UltronClient", _QwenpawAllStub) + def test_download_all_same_framework_keeps_prefixed_paths(self, *_): + """qwenpaw -> qwenpaw with --name all: no convert, agent prefixes kept.""" + rc = _run([ + "download", "--repo", "qw", "--framework", "qwenpaw", + "--name", "all", "--local_dir", str(self.out), + ]) + self.assertEqual(rc, 0) + self.assertTrue((self.out / "default" / "AGENTS.md").is_file()) + self.assertTrue((self.out / "bot-a" / "AGENTS.md").is_file()) + self.assertTrue((self.out / "bot-a" / "PROFILE.md").is_file()) + # non-spec top-level files are skipped. + self.assertFalse((self.out / "README.md").exists()) + class TestConvert(unittest.TestCase): def setUp(self): diff --git a/tests/test_harness.py b/tests/test_harness.py index 320bae6..35cc352 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -156,6 +156,42 @@ def test_registry_contains_all_products(self): self.assertIn("hermes", ALLOWLIST_REGISTRY) +class TestAllPathPrefix(unittest.TestCase): + """split_all_path / join_all_path for cross-framework all-mode convert.""" + + def _spec(self, fw): + return ALLOWLIST_REGISTRY[fw](agent_name="all") + + def test_qwenpaw_split_join(self): + spec = self._spec("qwenpaw") + self.assertTrue(spec.is_root_per_agent) + self.assertEqual(spec.split_all_path("bot-a/AGENTS.md"), ("bot-a", "AGENTS.md")) + self.assertEqual(spec.split_all_path("default/SOUL.md"), ("default", "SOUL.md")) + self.assertEqual(spec.split_all_path("README.md"), (None, "README.md")) + self.assertEqual(spec.join_all_path("bot-a", "AGENTS.md"), "bot-a/AGENTS.md") + + def test_openclaw_split_join(self): + spec = self._spec("openclaw") + self.assertTrue(spec.is_root_per_agent) + self.assertEqual(spec.split_all_path("workspace/AGENTS.md"), ("default", "AGENTS.md")) + self.assertEqual(spec.split_all_path("workspace-bot-a/SOUL.md"), ("bot-a", "SOUL.md")) + self.assertEqual(spec.split_all_path("README.md"), (None, "README.md")) + self.assertEqual(spec.join_all_path("default", "AGENTS.md"), "workspace/AGENTS.md") + self.assertEqual(spec.join_all_path("bot-a", "SOUL.md"), "workspace-bot-a/SOUL.md") + + def test_roundtrip_qwenpaw_to_openclaw(self): + src = self._spec("qwenpaw") + dst = self._spec("openclaw") + agent, bare = src.split_all_path("bot-a/AGENTS.md") + self.assertEqual(dst.join_all_path(agent, bare), "workspace-bot-a/AGENTS.md") + + def test_non_root_per_agent_passthrough(self): + spec = self._spec("qoder") + self.assertFalse(spec.is_root_per_agent) + self.assertEqual(spec.split_all_path("agents/x.md"), (None, "agents/x.md")) + self.assertEqual(spec.join_all_path("x", "agents/x.md"), "agents/x.md") + + class TestHarnessBundle(unittest.TestCase): """Bundle serialization round-trip.""" diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index 5028e89..f7a6261 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -82,10 +82,13 @@ def _resolve_local_name(name: Optional[str], framework: str, local_dir=None): Returns (resolved_name, error_message). - If name is given → use it directly. - - If omitted → check list_agents(): - - 0 or only 'default' → use GLOBAL_AGENT_NAME (shared files only) - - exactly 1 non-default agent → auto-select it - - multiple → return error + - If omitted: + - root-per-agent / single-agent layout (no ``{name}`` placeholder) → + always DEFAULT_AGENT_NAME. 'default' is a real workspace directory, so + sibling sub-agents never trigger auto-select or an error. + - file-per-agent+shared layout (patterns use ``{name}``) → inspect the + ``agents/`` files: exactly 1 non-default → auto-select it; 0 → + GLOBAL_AGENT_NAME (shared files only); multiple → error. """ if name: return name, None @@ -94,15 +97,20 @@ def _resolve_local_name(name: Optional[str], framework: str, local_dir=None): spec_cls = ALLOWLIST_REGISTRY[framework] local = Path(local_dir).expanduser() if local_dir else None tmp_spec = spec_cls(agent_name=DEFAULT_AGENT_NAME, local_dir=local) - agents = tmp_spec.list_agents() - # Filter out "default" to find real sub-agents. - real_agents = [a for a in agents if a != DEFAULT_AGENT_NAME] + # root-per-agent / single-agent: an omitted --name always means the default + # agent. Only layouts with a ``{name}`` placeholder (file-per-agent+shared) + # have a meaningful shared/global mode or per-agent auto-selection. + has_shared_mode = any("{name}" in p for p in tmp_spec.patterns) + if not has_shared_mode: + return DEFAULT_AGENT_NAME, None - if len(real_agents) == 0: - return GLOBAL_AGENT_NAME, None + agents = tmp_spec.list_agents() + real_agents = [a for a in agents if a != DEFAULT_AGENT_NAME] if len(real_agents) == 1: return real_agents[0], None + if len(real_agents) == 0: + return GLOBAL_AGENT_NAME, None return None, ( f"multiple sub-agents found: {', '.join(agents)}. " f"Please specify --name to select one." @@ -142,15 +150,23 @@ def _build_allowlist(framework: str, name: str, local_dir): return spec_cls(agent_name=name, local_dir=local) -def _convert(resources: dict, source_fw: str, target_fw: str) -> dict: +def _convert(resources: dict, source_fw: str, target_fw: str, + *, all_mode: bool = False, src_spec=None, dst_spec=None) -> dict: """Convert workspace resources from one framework's format to another. Reuses the server-side cross-product migration (``merge_resources``), so the output paths follow the target framework's conventions. A no-op when source and target are the same. + + When *all_mode* is True (root-per-agent -> root-per-agent), paths carry an + agent prefix; each agent is split out, converted independently as a single + agent, then re-prefixed for the target framework. Requires *src_spec* and + *dst_spec*. """ if source_fw == target_fw: return resources + if all_mode: + return _convert_all(resources, source_fw, target_fw, src_spec, dst_spec) result = merge_resources( incoming=resources, source_product=source_fw, @@ -161,6 +177,37 @@ def _convert(resources: dict, source_fw: str, target_fw: str) -> dict: return result.merged_files +def _convert_all(resources: dict, source_fw: str, target_fw: str, src_spec, dst_spec) -> dict: + """All-mode cross-framework convert (root-per-agent -> root-per-agent). + + Group incoming files by their source agent prefix, convert each agent as an + isolated single-agent workspace, then re-prefix the results using the target + framework's convention. Top-level files without an agent prefix (e.g. + README.md) belong to no agent and are dropped. + """ + groups: dict = {} + for path, content in resources.items(): + agent, bare = src_spec.split_all_path(path) + if agent is None: + continue + groups.setdefault(agent, {})[bare] = content + + src_defaults = get_defaults(source_fw) + tgt_defaults = get_defaults(target_fw) + out: dict = {} + for agent, bare_files in groups.items(): + result = merge_resources( + incoming=bare_files, + source_product=source_fw, + target_product=target_fw, + source_defaults=src_defaults, + target_defaults=tgt_defaults, + ) + for bare_path, content in result.merged_files.items(): + out[dst_spec.join_all_path(agent, bare_path)] = content + return out + + def cmd_list(args) -> int: """List remote agent repositories.""" server = config.resolve_server(getattr(args, 'server', None)) @@ -345,15 +392,31 @@ def cmd_download(args) -> int: target_fw = args.target or framework if target_fw not in ALLOWLIST_REGISTRY: return _fail(f"unknown target framework '{target_fw}'. Available: {_frameworks()}") - if target_fw != framework: - resources = _convert(resources, framework, target_fw) - print(f"Converted {framework} -> {target_fw} ({len(resources)} file(s)).") # Resolve local agent name for writing. local_name = args.name or DEFAULT_AGENT_NAME spec = _build_allowlist(target_fw, local_name, args.local_dir) root = spec.workspace_root + if target_fw != framework: + if args.name == ALL_AGENT_NAME: + # All-mode conversion only makes sense between two root-per-agent + # frameworks (1:1 agent-directory mapping); other layouts (e.g. + # file-per-agent qoder, single-agent) collapse N agents into a + # shared root, which is lossy and ambiguous -- reject explicitly. + src_spec = _build_allowlist(framework, local_name, args.local_dir) + if not (src_spec.is_root_per_agent and spec.is_root_per_agent): + return _fail( + "cross-framework conversion with --name all is only supported " + "between root-per-agent frameworks (e.g. qwenpaw <-> openclaw). " + "For other layouts, convert one agent at a time: " + "-n --target .") + resources = _convert(resources, framework, target_fw, + all_mode=True, src_spec=src_spec, dst_spec=spec) + else: + resources = _convert(resources, framework, target_fw) + print(f"Converted {framework} -> {target_fw} ({len(resources)} file(s)).") + # Filter downloaded resources by allowlist patterns. patterns = spec.resolved_patterns() filtered = {k: v for k, v in resources.items() if spec.matches(k, patterns)} diff --git a/ultron/services/harness/allowlist.py b/ultron/services/harness/allowlist.py index 5a4ce14..2a72843 100644 --- a/ultron/services/harness/allowlist.py +++ b/ultron/services/harness/allowlist.py @@ -110,6 +110,34 @@ def _effective_patterns(self) -> List[str]: add an agent-name prefix in all mode.""" return self.patterns + # ------------------------------------------------------------------ + # All-mode path prefixing (for cross-framework conversion) + # ------------------------------------------------------------------ + + @property + def is_root_per_agent(self) -> bool: + """Whether sub-agents are separate directories (root-per-agent layout). + + All-mode cross-framework conversion is only well-defined between two + root-per-agent frameworks, where every agent maps 1:1 to a directory + prefix. Other layouts return ``False``. + """ + return False + + def split_all_path(self, rel_path: str) -> Tuple[Optional[str], str]: + """Split an all-mode path into ``(agent_name, bare_path)``. + + ``bare_path`` is the path relative to a single agent's root. Returns + ``(None, rel_path)`` when the path has no recognizable agent prefix + (e.g. top-level ``README.md``). Root-per-agent classes override this. + """ + return (None, rel_path) + + def join_all_path(self, agent_name: str, bare_path: str) -> str: + """Inverse of :meth:`split_all_path`: build this framework's all-mode + path for *agent_name* + *bare_path*.""" + return bare_path + # ------------------------------------------------------------------ # Core helpers # ------------------------------------------------------------------ @@ -314,6 +342,26 @@ def _effective_patterns(self) -> List[str]: return [f"workspace*/{p}" for p in self.patterns] return self.patterns + @property + def is_root_per_agent(self) -> bool: + return True + + def split_all_path(self, rel_path: str) -> Tuple[Optional[str], str]: + # agent lives in ``workspace/`` (default) or ``workspace-/``. + if "/" not in rel_path: + return (None, rel_path) + head, rest = rel_path.split("/", 1) + if head == "workspace": + return (DEFAULT_AGENT_NAME, rest) + if head.startswith("workspace-"): + return (head[len("workspace-"):], rest) + return (None, rel_path) + + def join_all_path(self, agent_name: str, bare_path: str) -> str: + if agent_name in ("", DEFAULT_AGENT_NAME): + return f"workspace/{bare_path}" + return f"workspace-{agent_name}/{bare_path}" + def list_agents(self) -> List[str]: base = Path.home() / ".openclaw" agents: List[str] = [] @@ -396,6 +444,20 @@ def _effective_patterns(self) -> List[str]: return [f"*/{p}" for p in self.patterns] return self.patterns + @property + def is_root_per_agent(self) -> bool: + return True + + def split_all_path(self, rel_path: str) -> Tuple[Optional[str], str]: + # agent directory name IS the agent name: ``/``. + if "/" in rel_path: + head, rest = rel_path.split("/", 1) + return (head, rest) + return (None, rel_path) + + def join_all_path(self, agent_name: str, bare_path: str) -> str: + return f"{agent_name}/{bare_path}" + def list_agents(self) -> List[str]: base = Path.home() / ".qwenpaw" / "workspaces" if not base.is_dir(): From 0679d1f415cfa7e7af223c17885980ccae502bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Thu, 9 Jul 2026 21:43:10 +0800 Subject: [PATCH 13/14] fix --- tests/api/test_client_integration.py | 13 +-- tests/api/test_upload_download.py | 29 +++--- tests/services/test_allowlist_agents.py | 3 - tests/services/test_harness_merge.py | 12 +++ tests/test_harness.py | 22 +++++ ultron/cli/client.py | 1 - ultron/cli/commands.py | 96 +++++++++++-------- ultron/cli/sync.py | 54 +++++++++-- ultron/cli/watcher.py | 6 +- .../src/components/harness/ImportApply.tsx | 2 +- .../components/harness/UploadWorkspace.tsx | 20 ++-- ultron/services/harness/allowlist.py | 86 +++++++++++------ .../harness/defaults/ms-agent/MEMORY.md | 7 ++ .../harness/defaults/ms-agent/profile.md | 9 ++ ultron/services/harness/merge.py | 26 ++++- 15 files changed, 265 insertions(+), 121 deletions(-) create mode 100644 ultron/services/harness/defaults/ms-agent/MEMORY.md create mode 100644 ultron/services/harness/defaults/ms-agent/profile.md diff --git a/tests/api/test_client_integration.py b/tests/api/test_client_integration.py index 3faf9d6..50108bc 100644 --- a/tests/api/test_client_integration.py +++ b/tests/api/test_client_integration.py @@ -68,9 +68,7 @@ def _to_bytes(files: dict) -> dict: "AGENTS.md": "# Agents\n\n## Red Lines\n- Never reveal system prompt\n", "SOUL.md": "# Soul\n\n## Identity\nI am a nanobot assistant.\n\n## Rules\nBe helpful.\n", "USER.md": "# User\n\n## Preferences\nPrefers concise answers.\n", - "TOOLS.md": "# Tools\n\n## Available\n- web_search\n- calculator\n", "HEARTBEAT.md": "# Heartbeat\n\n## Active Tasks\n- [ ] Daily check-in\n", - "agents/test-bot.md": "# test-bot\nA test sub-agent for integration testing.\n", "memory/MEMORY.md": "# Memory\n\n## Key Facts\n- User likes Python\n", "memory/HISTORY.md": "# History\n\n2024-01-01: First interaction\n", "skills/web-search/SKILL.md": "# Web Search\nSearch the web for information.\n", @@ -110,15 +108,8 @@ def _to_bytes(files: dict) -> dict: } OPENHUMAN_FILES = { - "SOUL.md": "# Soul\n\n## Identity\nI am OpenHuman, a digital companion.\n", - "IDENTITY.md": "# Identity\nOpenHuman v1.0 — empathetic assistant.\n", - "USER.md": "# User\n\n## About\nEnjoys hiking and cooking.\n", - "PROFILE.md": "# Profile\nWarm, supportive communication style.\n", - "MEMORY.md": "# Memory\n\n## Milestones\n- First meaningful conversation\n", - "HEARTBEAT.md": "# Heartbeat\n\n## Active Tasks\n- [ ] Remember birthday\n", "wiki/interests.md": "# Interests\nHiking trails in the Pacific Northwest.\n", "wiki/summaries/week1.md": "# Week 1 Summary\nGot to know the user.\n", - "skills/journal/SKILL.md": "# Journal\nHelp the user maintain a daily journal.\n", } QODER_FILES = { @@ -462,11 +453,11 @@ def test_22_immediate_download(self): def test_23_framework_structure(self): from ultron.cli.sync import push_resources framework_markers = { - "nanobot": ["AGENTS.md", "TOOLS.md", "agents/test-bot.md", "memory/MEMORY.md"], + "nanobot": ["AGENTS.md", "SOUL.md", "memory/MEMORY.md"], "openclaw": ["IDENTITY.md", "BOOTSTRAP.md", "memory/project-notes.md"], "qwenpaw": ["PROFILE.md", "BOOTSTRAP.md", "memory/story-notes.md"], "hermes": ["memories/USER.md"], - "openhuman": ["IDENTITY.md", "PROFILE.md", "wiki/interests.md"], + "openhuman": ["wiki/interests.md", "wiki/summaries/week1.md"], "qoder": ["agents/code-reviewer.md", "commands/review.md", "rules/style-guide.md"], } diff --git a/tests/api/test_upload_download.py b/tests/api/test_upload_download.py index 3a06b20..0563325 100644 --- a/tests/api/test_upload_download.py +++ b/tests/api/test_upload_download.py @@ -92,8 +92,6 @@ def _log_429(fn, *args, **kwargs): NANOBOT_ALL_FILES = { "AGENTS.md": "# Agents\n", "SOUL.md": "# Soul\nI am nanobot.\n", - "agents/helper.md": "# Helper\nA helper sub-agent.\n", - "agents/writer.md": "# Writer\nA writer sub-agent.\n", "memory/MEMORY.md": "# Memory\n", "skills/search/SKILL.md": "# Search\nWeb search.\n", } @@ -329,10 +327,10 @@ def test_09_upload_empty_workspace(self): self._cleanup_dir(local) # ----------------------------------------------------------------------- - # 10. Upload: nanobot all mode (file-per-agent + shared) + # 10. Upload: nanobot all mode (single-agent -> full workspace) # ----------------------------------------------------------------------- def test_10_upload_all_nanobot(self): - """Upload all nanobot sub-agents at once.""" + """Upload the whole nanobot workspace in all mode.""" local = self._create_local_workspace(NANOBOT_ALL_FILES) try: args = self._upload_args("nanobot", ALL_AGENT_NAME, local_dir=local) @@ -463,7 +461,7 @@ def test_16_roundtrip_content_verify(self): _wait(5) # Verify via client API - server_files = self.client.list_repo_files(self.username, agent_name) + server_files = self.client.list_repo_files(self.username, _repo_name("qoder", agent_name)) uploaded_keys = set(files.keys()) server_set = set(server_files) missing = uploaded_keys - server_set @@ -472,7 +470,7 @@ def test_16_roundtrip_content_verify(self): # Download and verify content for rel, expected in files.items(): if rel in server_set: - actual = self.client.download_repo_file(self.username, agent_name, rel) + actual = self.client.download_repo_file(self.username, _repo_name("qoder", agent_name), rel) self.assertEqual(actual.strip(), expected.strip(), f"content mismatch for {rel}") @@ -590,7 +588,7 @@ def test_21_upload_modify_reupload(self): _wait(5) # Verify V2 content - content = self.client.download_repo_file(self.username, agent_name, "AGENTS.md") + content = self.client.download_repo_file(self.username, _repo_name("qoder", agent_name), "AGENTS.md") self.assertIn("V2", content) self.assertIn("Modified", content) @@ -625,7 +623,7 @@ def test_23_upload_each_framework(self): if fw == "qoder": files = {"AGENTS.md": "# Agents\n", f"agents/{agent_name}.md": "# X\n"} elif fw == "nanobot": - files = {"SOUL.md": "# Soul\n", f"agents/{agent_name}.md": "# X\n"} + files = {"SOUL.md": "# Soul\n"} elif fw == "openclaw": files = {"SOUL.md": "# Soul\n", "IDENTITY.md": "# ID\n"} elif fw == "qwenpaw": @@ -633,7 +631,9 @@ def test_23_upload_each_framework(self): elif fw == "hermes": files = {"SOUL.md": "# Soul\n"} elif fw == "openhuman": - files = {"SOUL.md": "# Soul\n", "IDENTITY.md": "# ID\n"} + files = {"wiki/identity.md": "# Identity\n"} + elif fw == "ms-agent": + files = {"profile.md": "# Profile\n", "MEMORY.md": "# Memory\n"} else: files = {"SOUL.md": "# Soul\n"} local = self._create_local_workspace(files) @@ -777,18 +777,23 @@ def test_29_download_all_roundtrip(self): # 30. supports_individual_watch property check # ----------------------------------------------------------------------- def test_30_supports_individual_watch(self): - """Verify file-per-agent frameworks disallow individual watch.""" + """Only file-per-agent + shared frameworks (qoder) disallow individual watch.""" qoder = ALLOWLIST_REGISTRY["qoder"](agent_name="reviewer") nanobot = ALLOWLIST_REGISTRY["nanobot"](agent_name="helper") qwenpaw = ALLOWLIST_REGISTRY["qwenpaw"](agent_name="bot-a") openclaw = ALLOWLIST_REGISTRY["openclaw"](agent_name="helper") hermes = ALLOWLIST_REGISTRY["hermes"](agent_name="default") + ms_agent = ALLOWLIST_REGISTRY["ms-agent"](agent_name="default") + # file-per-agent + shared: shared files cascade, individual watch unsupported self.assertFalse(qoder.supports_individual_watch) - self.assertFalse(nanobot.supports_individual_watch) + # single-agent installs: the whole workspace is the one agent + self.assertTrue(nanobot.supports_individual_watch) + self.assertTrue(hermes.supports_individual_watch) + self.assertTrue(ms_agent.supports_individual_watch) + # root-per-agent: each agent is its own directory self.assertTrue(qwenpaw.supports_individual_watch) self.assertTrue(openclaw.supports_individual_watch) - self.assertTrue(hermes.supports_individual_watch) if __name__ == "__main__": diff --git a/tests/services/test_allowlist_agents.py b/tests/services/test_allowlist_agents.py index bc576e9..f3dd3c7 100644 --- a/tests/services/test_allowlist_agents.py +++ b/tests/services/test_allowlist_agents.py @@ -63,13 +63,10 @@ def test_qwenpaw_default_root_uses_agent_name(self): def test_local_dir_override_wins(self): (self.root / "SOUL.md").write_text("soul") - (self.root / "agents").mkdir() - (self.root / "agents" / "main.md").write_text("main") spec = NanobotWorkspaceAllowlist(agent_name="main", local_dir=self.root) self.assertEqual(spec.workspace_root, self.root) collected = spec.collect() self.assertIn("SOUL.md", collected) - self.assertIn("agents/main.md", collected) def test_missing_root_returns_empty(self): spec = QoderWorkspaceAllowlist( diff --git a/tests/services/test_harness_merge.py b/tests/services/test_harness_merge.py index 96c0ee0..cca515a 100644 --- a/tests/services/test_harness_merge.py +++ b/tests/services/test_harness_merge.py @@ -208,6 +208,18 @@ def test_memory_cross_new_products(self): self.assertEqual(_resolve_target_path("openhuman", "MEMORY.md", "qwenpaw"), "MEMORY.md") self.assertEqual(_resolve_target_path("qwenpaw", "MEMORY.md", "nanobot"), "memory/MEMORY.md") + def test_ms_agent_profile(self): + # ms-agent profile.md <-> qwenpaw/openhuman PROFILE.md (persona group) + self.assertEqual(_resolve_target_path("ms-agent", "profile.md", "qwenpaw"), "PROFILE.md") + self.assertEqual(_resolve_target_path("qwenpaw", "PROFILE.md", "ms-agent"), "profile.md") + # nanobot has no PROFILE concept + self.assertIsNone(_resolve_target_path("ms-agent", "profile.md", "nanobot")) + + def test_ms_agent_memory(self): + self.assertEqual(_resolve_target_path("ms-agent", "MEMORY.md", "openclaw"), "MEMORY.md") + self.assertEqual(_resolve_target_path("openclaw", "MEMORY.md", "ms-agent"), "MEMORY.md") + self.assertEqual(_resolve_target_path("ms-agent", "MEMORY.md", "nanobot"), "memory/MEMORY.md") + class TestMergeResources(unittest.TestCase): def test_same_product_imports_directly(self): diff --git a/tests/test_harness.py b/tests/test_harness.py index 35cc352..6d4ca78 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -154,6 +154,28 @@ def test_registry_contains_all_products(self): self.assertIn("nanobot", ALLOWLIST_REGISTRY) self.assertIn("openclaw", ALLOWLIST_REGISTRY) self.assertIn("hermes", ALLOWLIST_REGISTRY) + self.assertIn("ms-agent", ALLOWLIST_REGISTRY) + + def test_ms_agent_single_agent_layout(self): + """ms-agent is single-agent: no {name} placeholder, collects persona/ + memory/skills/config under ~/.ms_agent.""" + al = ALLOWLIST_REGISTRY["ms-agent"](local_dir=self.root) + self.assertEqual(al.product_name, "ms-agent") + # single-agent: no {name} placeholder in patterns + self.assertFalse(any("{name}" in p for p in al.patterns)) + (self.root / "profile.md").write_text("p") + (self.root / "MEMORY.md").write_text("m") + (self.root / "facts.json").write_text("{}") + (self.root / "settings.json").write_text("{}") + (self.root / "skill.json").write_text("{}") + (self.root / "random.txt").write_text("x") + (self.root / "skills" / "foo").mkdir(parents=True) + (self.root / "skills" / "foo" / "SKILL.md").write_text("s") + got = al.collect() + for f in ("profile.md", "MEMORY.md", "facts.json", "settings.json", + "skill.json", "skills/foo/SKILL.md"): + self.assertIn(f, got) + self.assertNotIn("random.txt", got) class TestAllPathPrefix(unittest.TestCase): diff --git a/ultron/cli/client.py b/ultron/cli/client.py index de3b0eb..f070c13 100644 --- a/ultron/cli/client.py +++ b/ultron/cli/client.py @@ -12,7 +12,6 @@ * ``POST /api/v1/repos/agents/{id}/info/lfs/objects/batch`` → LFS batch verify * ``DELETE /api/v1/agents/{path}/{name}/repo/file`` → delete file """ -import base64 import hashlib import logging import os diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index f7a6261..cf6f107 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -603,7 +603,7 @@ def cmd_watch(args) -> int: print(f" Logs: {pid_file().parent / 'logs' / 'watch.log'}") print(f" Stop: ultron stop") - daemonize(watch_loop, spec, client, username, repo, framework, interval, push_only=push_only) + daemonize(watch_loop, spec, client, group, repo, framework, interval, push_only=push_only) # If we reach here, we are the parent process (daemon forked successfully). print(f" Watch started (PID file: {pf}).") return 0 @@ -621,6 +621,48 @@ def cmd_stop(args) -> int: return 0 +def _strip_backup_wrapper(filename: str) -> str: + """Strip the legacy ``agent/`` wrapper dir from a backup zip entry. + + Older backups stored files under an ``agent/`` prefix; new backups don't. + Stripping keeps restore aligned with the workspace-relative layout used by + ``collect``/``apply`` regardless of which format the zip uses. + """ + prefix = "agent/" + if filename.startswith(prefix): + return filename[len(prefix):] + return filename + + +def _parse_backup_meta(stem: str): + """Parse a backup filename stem into ``(framework, name)``. + + Filenames look like ``{fw}{delim}{name}_{date}_{time}``; the leading + ``{fw}{delim}{name}`` prefix is split on ``_`` (watch backups) or ``-`` + (upload backups). + """ + parts = stem.rsplit("_", 2) + prefix = parts[0] if len(parts) >= 3 else stem + delim = "_" if "_" in prefix else "-" + fw, _, nm = prefix.partition(delim) + return fw, nm + + +def _filter_backups(backups, fw_filter, name_filter): + """Filter backup files by framework/name parsed from their filenames.""" + if not (fw_filter or name_filter): + return backups + out = [] + for f in backups: + fw, nm = _parse_backup_meta(f.stem) + if fw_filter and fw != fw_filter: + continue + if name_filter and nm != name_filter: + continue + out.append(f) + return out + + def cmd_recover(args) -> int: """Restore agent files from a backup zip. @@ -643,23 +685,8 @@ def cmd_recover(args) -> int: # --list mode: enumerate backups and exit if args.list: # Filter by framework/name if provided (filename: {fw}_{name}_{date}_{time}.zip) - fw_filter = getattr(args, 'framework', None) - name_filter = getattr(args, 'name', None) - if fw_filter or name_filter: - filtered = [] - for f in backups: - parts = f.stem.rsplit("_", 2) - prefix = parts[0] if len(parts) >= 3 else f.stem - delim = "_" if "_" in prefix else "-" - parts_fw = prefix.split(delim, 1) - fw = parts_fw[0] - name = parts_fw[1] if len(parts_fw) > 1 else "" - if fw_filter and fw != fw_filter: - continue - if name_filter and name != name_filter: - continue - filtered.append(f) - backups = filtered + backups = _filter_backups( + backups, getattr(args, 'framework', None), getattr(args, 'name', None)) if not backups: print("No backups found.") @@ -679,23 +706,8 @@ def cmd_recover(args) -> int: return _fail("specify a target: 'last' or a backup filename. Use --list to see available backups.") # Filter backups by framework/name if provided - fw_filter = getattr(args, 'framework', None) - name_filter = getattr(args, 'name', None) - if fw_filter or name_filter: - filtered = [] - for f in backups: - parts = f.stem.rsplit("_", 2) - prefix = parts[0] if len(parts) >= 3 else f.stem - delim = "_" if "_" in prefix else "-" - parts_fw = prefix.split(delim, 1) - fw = parts_fw[0] - name = parts_fw[1] if len(parts_fw) > 1 else "" - if fw_filter and fw != fw_filter: - continue - if name_filter and name != name_filter: - continue - filtered.append(f) - backups = filtered + backups = _filter_backups( + backups, getattr(args, 'framework', None), getattr(args, 'name', None)) # Resolve target to a zip path if target == "last": @@ -743,11 +755,12 @@ def cmd_recover(args) -> int: else: print("No existing files to backup.") - # ---- Step 2: Determine which files are in the zip ---- + # ---- Step 2: Determine which files are in the zip (strip legacy prefix) ---- with zipfile.ZipFile(zip_path, "r") as zf: - zip_entries = set( - info.filename for info in zf.infolist() if not info.is_dir() - ) + zip_entries = { + _strip_backup_wrapper(info.filename) + for info in zf.infolist() if not info.is_dir() + } # ---- Step 3: Delete local files that are NOT in the zip ---- deleted = 0 @@ -767,13 +780,14 @@ def cmd_recover(args) -> int: for info in zf.infolist(): if info.is_dir(): continue - file_target = (resolved_root / info.filename).resolve() + rel = _strip_backup_wrapper(info.filename) + file_target = (resolved_root / rel).resolve() if not file_target.is_relative_to(resolved_root): print(f" Skipped (path traversal): {info.filename}") continue file_target.parent.mkdir(parents=True, exist_ok=True) file_target.write_bytes(zf.read(info.filename)) - print(f" Restored: {info.filename}") + print(f" Restored: {rel}") restored += 1 print(f"\nRestored {restored} file(s), removed {deleted} extra file(s).") diff --git a/ultron/cli/sync.py b/ultron/cli/sync.py index b5d195d..221705e 100644 --- a/ultron/cli/sync.py +++ b/ultron/cli/sync.py @@ -18,21 +18,25 @@ -def zip_resources(resources: Dict[str, Union[str, bytes]], wrapper: str = "agent") -> bytes: +def zip_resources(resources: Dict[str, Union[str, bytes]], wrapper: str = "") -> bytes: """Pack resources into a deterministic in-memory zip for local backup. - Used by :func:`backup_local` to create timestamped backups before - destructive operations (pull, convert, restore). + Entries are stored with workspace-relative paths (no wrapper directory) so + that restore can extract them back to the exact locations reported by + ``collect``/``apply``. Used by :func:`backup_local` to create timestamped + backups before destructive operations (pull, convert, restore). Args: resources: A dict {rel_path: content}. Values are written directly via ``ZipFile.writestr`` (accepts both str and bytes). - wrapper: Name of the top-level wrapper directory (default: "agent"). + wrapper: Optional top-level wrapper directory (kept only for backward + compatibility; empty by default). """ buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: for rel, value in sorted(resources.items()): - zf.writestr(f"{wrapper}/{rel}", value) + key = f"{wrapper}/{rel}" if wrapper else rel + zf.writestr(key, value) return buf.getvalue() @@ -81,6 +85,32 @@ def detect_local_changes( return changed +def _retry_on_master_missing(fn, retries: int = 3, delay: float = 2.0): + """Run *fn*, retrying the create-then-commit race. + + A freshly created repo may not have its ``master`` branch ready when the + first commit fires, yielding a 400 "branch or revision master not found". + Wrapping any commit-bearing call (normal commit *and* LFS upload) means + every first-push path gets the same protection. + """ + import time + + from .client import ApiError + + for attempt in range(retries): + try: + return fn() + except ApiError as e: + msg = (getattr(e, "detail", "") or getattr(e, "message", "") or str(e)).lower() + branch_missing = getattr(e, "status", None) == 400 and ( + "branch" in msg or "revision" in msg + ) + if branch_missing and attempt < retries - 1: + time.sleep(delay) + continue + raise + + def push_resources( client: "UltronClient", username: str, @@ -129,15 +159,19 @@ def push_resources( # Commit normal files in one request. if normal_actions: - client.commit_files(username, name, normal_actions, - commit_message="sync: upload normal files") + _retry_on_master_missing(lambda: client.commit_files( + username, name, normal_actions, + commit_message="sync: upload normal files")) for a in normal_actions: logger.info(" CREATE: %s (%d B)", a["path"], a["size"]) - # Upload LFS files one-by-one (batch verify + PUT + commit). + # Upload LFS files one-by-one (batch verify + PUT + commit). The commit + # inside upload_lfs_file hits the same fresh-repo master race, so it needs + # the retry too (LFS-only first push would otherwise fail). for rel, content in lfs_files: - client.upload_lfs_file(username, name, rel, content, - action="create", commit_message=f"sync: upload LFS {rel}") + _retry_on_master_missing(lambda rel=rel, content=content: client.upload_lfs_file( + username, name, rel, content, + action="create", commit_message=f"sync: upload LFS {rel}")) logger.info(" CREATE (LFS): %s (%d B)", rel, len(content)) logger.info("Pushed %d file(s) (%d normal, %d LFS).", diff --git a/ultron/cli/watcher.py b/ultron/cli/watcher.py index 27aaa4f..6ecb2a0 100644 --- a/ultron/cli/watcher.py +++ b/ultron/cli/watcher.py @@ -96,7 +96,11 @@ def _handle_term(signum, frame): try: remote_files = client.list_repo_files_detail(username, repo) except ApiError as e: - if e.status in (404, 500): + # A non-existent remote repo returns 400 (code 10025801016), and + # 404/500 cover transient/unreadable states. Treat all as an empty + # baseline so the first push can create the repo instead of looping + # forever (an empty-but-created repo returns 200 with a tree). + if e.status in (400, 404, 500): remote_files = [] else: logger.error("Failed to list remote files: %s", e) diff --git a/ultron/dashboard/src/components/harness/ImportApply.tsx b/ultron/dashboard/src/components/harness/ImportApply.tsx index b4ecd2c..79cface 100644 --- a/ultron/dashboard/src/components/harness/ImportApply.tsx +++ b/ultron/dashboard/src/components/harness/ImportApply.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { useLocale } from '../../contexts/LocaleContext'; import HarnessIoBlurb from './HarnessIoBlurb'; -const PRODUCTS = ['nanobot', 'openclaw', 'hermes', 'qwenpaw', 'openhuman']; +const PRODUCTS = ['nanobot', 'openclaw', 'hermes', 'qwenpaw', 'openhuman', 'ms-agent']; export default function ImportApply({ onImported }: { onImported?: () => void }) { const { t } = useLocale(); diff --git a/ultron/dashboard/src/components/harness/UploadWorkspace.tsx b/ultron/dashboard/src/components/harness/UploadWorkspace.tsx index 0d5160d..eead803 100644 --- a/ultron/dashboard/src/components/harness/UploadWorkspace.tsx +++ b/ultron/dashboard/src/components/harness/UploadWorkspace.tsx @@ -6,10 +6,9 @@ import HarnessIoBlurb from './HarnessIoBlurb'; const PRODUCT_ALLOWLISTS: Record = { nanobot: [ - 'AGENTS.md', 'SOUL.md', 'USER.md', 'TOOLS.md', 'HEARTBEAT.md', + 'AGENTS.md', 'SOUL.md', 'USER.md', 'HEARTBEAT.md', 'memory/MEMORY.md', 'memory/HISTORY.md', - 'skills/*/SKILL.md', 'skills/*/_meta.json', 'skills/*/scripts/*', - 'skills/*/setup.md', 'skills/*/operations.md', 'skills/*/boundaries.md', + 'skills/*/SKILL.md', 'skills/*/scripts/*', ], openclaw: [ 'AGENTS.md', 'SOUL.md', 'USER.md', 'TOOLS.md', 'HEARTBEAT.md', @@ -28,14 +27,17 @@ const PRODUCT_ALLOWLISTS: Record = { 'skills/*/SKILL.md', 'skills/*/_meta.json', 'skills/*/scripts/*', ], openhuman: [ - 'SOUL.md', 'IDENTITY.md', 'USER.md', 'PROFILE.md', 'MEMORY.md', 'HEARTBEAT.md', - 'wiki/*.md', 'wiki/summaries/*.md', 'wiki/notes/*.md', - 'skills/*/SKILL.md', 'skills/*/_meta.json', 'skills/*/scripts/*', + 'wiki/*.md', + ], + 'ms-agent': [ + 'profile.md', 'config.yaml', 'settings.json', 'agent.yaml', + 'MEMORY.md', 'facts.json', + 'skill.json', 'skills/*/SKILL.md', ], }; -const PRODUCT_DIRS: Record = { nanobot: '.nanobot', openclaw: '.openclaw', hermes: '.hermes', qwenpaw: '.qwenpaw', openhuman: '.openhuman' }; -const PRODUCT_MARKERS: Record = { '.nanobot': 'nanobot', '.openclaw': 'openclaw', '.hermes': 'hermes', '.qwenpaw': 'qwenpaw', '.openhuman': 'openhuman' }; +const PRODUCT_DIRS: Record = { nanobot: '.nanobot', openclaw: '.openclaw', hermes: '.hermes', qwenpaw: '.qwenpaw', openhuman: '.openhuman', 'ms-agent': '.ms_agent' }; +const PRODUCT_MARKERS: Record = { '.nanobot': 'nanobot', '.openclaw': 'openclaw', '.hermes': 'hermes', '.qwenpaw': 'qwenpaw', '.openhuman': 'openhuman', '.ms_agent': 'ms-agent' }; function globMatch(pattern: string, path: string) { const re = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^/]*'); @@ -189,7 +191,7 @@ export default function UploadWorkspace({ onUploaded }: { onUploaded?: () => voi {t('harness.upload.hintPaths')}{' '} ~/.nanobot, ~/.openclaw,{' '} ~/.hermes, ~/.qwenpaw,{' '} - ~/.openhuman + ~/.openhuman, ~/.ms_agent )} diff --git a/ultron/services/harness/allowlist.py b/ultron/services/harness/allowlist.py index 2a72843..3bc27d5 100644 --- a/ultron/services/harness/allowlist.py +++ b/ultron/services/harness/allowlist.py @@ -19,9 +19,10 @@ * **file-per-agent + shared** -- the sub-agent is one file inside a shared root, collected alongside the shared resources; a ``{name}`` placeholder in ``patterns`` is formatted with the sub-agent name so only the selected agent's - file matches (qoder ``agents/.md``, nanobot). + file matches (qoder ``agents/.md``). * **single-agent** -- one persona per install; the sub-agent name is only the - repository identity and does not affect file selection (hermes, openhuman). + repository identity and does not affect file selection (hermes, openhuman, + nanobot, ms-agent). The dashboard re-implements the same root/pattern logic in TypeScript (``ultron/dashboard/src/components/harness/UploadWorkspace.tsx``); keep the two @@ -258,16 +259,17 @@ def apply(self, resources: Dict[str, str]) -> List[str]: class NanobotWorkspaceAllowlist(ClawWorkspaceAllowlist): - """Allowlist for the nanobot agent workspace (file-per-agent + shared).""" + """Allowlist for the nanobot agent workspace (single-agent install). + + Nanobot keeps a single shared workspace at ``~/.nanobot/workspace``; its + sub-agents run as background sessions with no on-disk per-agent files, so + this is a single-agent layout (the agent name is only the repo identity). + """ @property def product_name(self) -> str: return "nanobot" - @property - def supports_individual_watch(self) -> bool: - return False - @property def default_workspace_root(self) -> Path: return Path.home() / ".nanobot" / "workspace" @@ -278,22 +280,13 @@ def patterns(self) -> List[str]: "AGENTS.md", "SOUL.md", "USER.md", - "TOOLS.md", "HEARTBEAT.md", - "agents/{name}.md", "memory/MEMORY.md", "memory/HISTORY.md", "skills/*/SKILL.md", - "skills/*/_meta.json", "skills/*/scripts/*", - "skills/*/setup.md", - "skills/*/operations.md", - "skills/*/boundaries.md", ] - def list_agents(self) -> List[str]: - return self._list_agents_from_dir(self.workspace_root / "agents") - class OpenclawWorkspaceAllowlist(ClawWorkspaceAllowlist): """Allowlist for the OpenClaw agent workspace (root-per-agent). @@ -469,8 +462,10 @@ def list_agents(self) -> List[str]: class OpenhumanWorkspaceAllowlist(ClawWorkspaceAllowlist): """Allowlist for the OpenHuman agent workspace (single-agent install). - OpenHuman keeps its hidden workspace at ``~/.openhuman/workspace`` with an - Obsidian-style ``wiki/`` memory vault alongside the persona files. + OpenHuman is a Rust/Tauri desktop app whose brain is a local Memory Tree + mirrored as an Obsidian-style ``wiki/`` Markdown vault. Only the + human-readable ``wiki/`` vault is portable; OpenHuman has no OpenClaw-style + persona files (SOUL/IDENTITY/USER/...). """ @property @@ -483,19 +478,9 @@ def default_workspace_root(self) -> Path: @property def patterns(self) -> List[str]: + # ``*`` in fnmatch spans ``/`` so this recurses the whole wiki vault. return [ - "SOUL.md", - "IDENTITY.md", - "USER.md", - "PROFILE.md", - "MEMORY.md", - "HEARTBEAT.md", "wiki/*.md", - "wiki/summaries/*.md", - "wiki/notes/*.md", - "skills/*/SKILL.md", - "skills/*/_meta.json", - "skills/*/scripts/*", ] @@ -536,6 +521,48 @@ def list_agents(self) -> List[str]: return self._list_agents_from_dir(self.workspace_root / "agents") +class MsAgentWorkspaceAllowlist(ClawWorkspaceAllowlist): + """Allowlist for the ms-agent agent workspace (single-agent install). + + ms-agent keeps its persona, memory and skills under ``~/.ms_agent``: + + * **persona** -- a single ``profile.md`` augmented by injected + configuration (project-level ``config.yaml``, global ``settings.json`` + and a user-specified ``agent.yaml``). + * **memory** -- ``MEMORY.md`` plus a structured ``facts.json``. + * **skills** -- ``skills//SKILL.md`` with a workspace-level + ``skill.json`` metadata index. + + Only ``profile.md`` (persona) and ``MEMORY.md`` (memory) carry + cross-framework semantics; the YAML/JSON config and metadata files are + ms-agent specific and are preserved on same-framework sync only. + """ + + @property + def product_name(self) -> str: + return "ms-agent" + + @property + def default_workspace_root(self) -> Path: + return Path.home() / ".ms_agent" + + @property + def patterns(self) -> List[str]: + return [ + # Persona + injected configuration + "profile.md", + "config.yaml", + "settings.json", + "agent.yaml", + # Memory + "MEMORY.md", + "facts.json", + # Skills + "skill.json", + "skills/*/SKILL.md", + ] + + def _list_agent_files(agents_dir: Path) -> List[str]: """Return the stems of ``*.md`` files in an ``agents/`` directory.""" if not agents_dir.is_dir(): @@ -550,4 +577,5 @@ def _list_agent_files(agents_dir: Path) -> List[str]: "qwenpaw": QwenpawWorkspaceAllowlist, "openhuman": OpenhumanWorkspaceAllowlist, "qoder": QoderWorkspaceAllowlist, + "ms-agent": MsAgentWorkspaceAllowlist, } diff --git a/ultron/services/harness/defaults/ms-agent/MEMORY.md b/ultron/services/harness/defaults/ms-agent/MEMORY.md new file mode 100644 index 0000000..9fd9cef --- /dev/null +++ b/ultron/services/harness/defaults/ms-agent/MEMORY.md @@ -0,0 +1,7 @@ +# Memory + +## Facts + +## Preferences + +## History diff --git a/ultron/services/harness/defaults/ms-agent/profile.md b/ultron/services/harness/defaults/ms-agent/profile.md new file mode 100644 index 0000000..2328c89 --- /dev/null +++ b/ultron/services/harness/defaults/ms-agent/profile.md @@ -0,0 +1,9 @@ +# Profile + +## Identity + +## Values + +## Expertise + +## Preferences diff --git a/ultron/services/harness/merge.py b/ultron/services/harness/merge.py index 62f605b..6733a76 100644 --- a/ultron/services/harness/merge.py +++ b/ultron/services/harness/merge.py @@ -346,6 +346,16 @@ def merge( ]), "heartbeat": "HEARTBEAT.md", }, + "ms-agent": { + # Only persona + memory carry cross-framework semantics; the + # config.yaml/settings.json/agent.yaml/facts.json/skill.json files are + # ms-agent specific and preserved on same-framework sync only. + "portable": frozenset([ + "profile.md", "MEMORY.md", + ]), + "config": frozenset([]), + "heartbeat": "", + }, } # Fallback for unknown products @@ -380,11 +390,11 @@ def merge( "openhuman": "USER.md"}, # Long-term curated memory {"nanobot": "memory/MEMORY.md", "openclaw": "MEMORY.md", - "qwenpaw": "MEMORY.md", "openhuman": "MEMORY.md"}, + "qwenpaw": "MEMORY.md", "openhuman": "MEMORY.md", "ms-agent": "MEMORY.md"}, # Agent's own identity card {"openclaw": "IDENTITY.md", "openhuman": "IDENTITY.md"}, # Combined identity + user profile (QwenPaw/OpenHuman concept) - {"qwenpaw": "PROFILE.md", "openhuman": "PROFILE.md"}, + {"qwenpaw": "PROFILE.md", "openhuman": "PROFILE.md", "ms-agent": "profile.md"}, # Workspace operating instructions {"nanobot": "AGENTS.md", "openclaw": "AGENTS.md", "qwenpaw": "AGENTS.md"}, # Periodic task list @@ -398,7 +408,7 @@ def merge( {"nanobot": "memory/HISTORY.md"}, ] -_ALL_PRODUCTS = ["nanobot", "openclaw", "hermes", "qwenpaw", "openhuman"] +_ALL_PRODUCTS = ["nanobot", "openclaw", "hermes", "qwenpaw", "openhuman", "ms-agent"] def _build_path_map(): @@ -437,6 +447,10 @@ def _build_path_map(): "SOUL.md", "IDENTITY.md", "USER.md", "PROFILE.md", "MEMORY.md", "HEARTBEAT.md", ]), + "ms-agent": frozenset([ + "profile.md", "MEMORY.md", "config.yaml", "settings.json", + "agent.yaml", "facts.json", "skill.json", + ]), } @@ -497,6 +511,12 @@ def _catch_all_file(product: str) -> str: return "AGENTS.md" if "AGENTS.md" in known: return "AGENTS.md" + if "SOUL.md" in known: + return "SOUL.md" + # Products without AGENTS.md/SOUL.md (e.g. ms-agent) fall back to their + # persona file so overflow lands in a file the harness actually loads. + if "profile.md" in known: + return "profile.md" return "SOUL.md" From b198ca9e6a5381e535625e2b398da202553455c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Fri, 10 Jul 2026 08:33:01 +0800 Subject: [PATCH 14/14] fix --- tests/cli/test_download_convert.py | 29 ++++++++++++++++ ultron/cli/commands.py | 54 +++++++++++++++++++++++++----- ultron/services/harness/merge.py | 9 ++++- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/tests/cli/test_download_convert.py b/tests/cli/test_download_convert.py index 355b31f..1fdcc32 100644 --- a/tests/cli/test_download_convert.py +++ b/tests/cli/test_download_convert.py @@ -259,6 +259,35 @@ def test_convert_no_source_files_fails(self): ]) self.assertEqual(rc, 1) + def test_convert_qwenpaw_to_qoder_persona_lands_in_agents_file(self): + """file-per-agent target: -n routes persona to agents/{name}.md. + + qwenpaw SOUL/PROFILE have no shared mapping on qoder, so they must be + routed to the per-agent file agents/bot-a.md rather than polluting the + shared AGENTS.md. + """ + src = Path(self.tmp.name) / "qp" + out = Path(self.tmp.name) / "qo" + src.mkdir(parents=True) + (src / "SOUL.md").write_text("# Soul\nQP_SOUL_IDENTITY.\n") + (src / "PROFILE.md").write_text("# Profile\nQP_PROFILE_MARKER.\n") + (src / "AGENTS.md").write_text("# Agents\nSHARED_QP_AGENTS.\n") + rc = _run([ + "convert", "--from", "qwenpaw", "--to", "qoder", "-n", "bot-a", + "--local_dir", str(src), "--out-dir", str(out), + ]) + self.assertEqual(rc, 0) + agent_file = out / "agents" / "bot-a.md" + self.assertTrue(agent_file.is_file(), + "persona must land in agents/bot-a.md") + body = agent_file.read_text() + self.assertIn("QP_SOUL_IDENTITY.", body) + self.assertIn("QP_PROFILE_MARKER.", body) + shared = out / "AGENTS.md" + if shared.is_file(): + self.assertNotIn("QP_SOUL_IDENTITY.", shared.read_text(), + "shared AGENTS.md must stay free of per-agent identity") + if __name__ == "__main__": unittest.main() diff --git a/ultron/cli/commands.py b/ultron/cli/commands.py index cf6f107..95a3d76 100644 --- a/ultron/cli/commands.py +++ b/ultron/cli/commands.py @@ -442,6 +442,23 @@ def cmd_download(args) -> int: return 0 +def _file_per_agent_identity_path(dst_spec) -> Optional[str]: + """Resolve the per-agent identity file for a file-per-agent target. + + File-per-agent frameworks (e.g. qoder) declare a ``{name}`` placeholder + pattern such as ``agents/{name}.md``. Format it with the destination + agent name so converted persona content can be routed into that file. + Returns ``None`` when the layout has no single ``{name}`` file pattern. + """ + name = dst_spec.agent_name or DEFAULT_AGENT_NAME + for pattern in dst_spec.patterns: + # Only single-file placeholders (no wildcard) identify the persona file; + # skip glob patterns like ``skills/{name}/*`` if any exist. + if "{name}" in pattern and "*" not in pattern: + return pattern.format(name=name) + return None + + def convert_workspace( src_spec, source_fw: str, target_fw: str, dst_spec, dry_run: bool = False ) -> int: @@ -461,12 +478,20 @@ def convert_workspace( converted = resources default_paths = set() else: + # File-per-agent targets (e.g. qoder ``agents/{name}.md``) keep + # per-agent identity in a dedicated sub-agent file; route overflow + # (persona content with no shared mapping) there instead of the + # shared catch-all so it does not pollute other sub-agents. + overflow_target = None + if any("{name}" in p for p in dst_spec.patterns): + overflow_target = _file_per_agent_identity_path(dst_spec) result = merge_resources( incoming=resources, source_product=source_fw, target_product=target_fw, source_defaults=get_defaults(source_fw), target_defaults=get_defaults(target_fw), + overflow_target=overflow_target, ) default_paths = { a.path for a in result.actions if a.action == 'default' @@ -730,20 +755,31 @@ def cmd_recover(args) -> int: return _fail(f"unknown framework '{framework}'. Available: {_frameworks()}") name = args.name - if not name: - # Infer name from filename: "qoder_20260609_143022.zip" -> "qoder" - stem = zip_path.stem - parts = stem.rsplit("_", 2) - name = parts[0] if len(parts) >= 3 else stem + # Parse (framework, name) from the zip filename once, honoring both the + # ``-`` (upload) and ``_`` (watch) delimiters, and fill in whatever the + # caller left unspecified. Reusing _parse_backup_meta keeps this aligned + # with the --list/restore filtering above. + parsed_fw, parsed_name = _parse_backup_meta(zip_path.stem) if not framework: - # Try to infer framework from the name (e.g., "qoder" -> framework "qoder") - if name in ALLOWLIST_REGISTRY: - framework = name + # Try to infer framework from the parsed prefix (e.g., "qoder"). + if parsed_fw in ALLOWLIST_REGISTRY: + framework = parsed_fw else: return _fail("cannot infer framework. Pass --framework explicitly.") - spec = _build_allowlist(framework, "all", args.local_dir) + # Determine the restore SCOPE (which agent directory the backup belongs to). + # An all-scope backup is named ``{fw}_{date}_{time}`` (no name segment), so + # ``parsed_name`` is empty -> restore into the all-root. A single-agent + # backup is ``{fw}{delim}{name}_...`` -> restore into that agent only. + # A hardcoded "all" here would lift root-per-agent frameworks to the shared + # workspaces/ parent and, because a single-agent zip stores bare (unprefixed) + # paths, wrongly treat every sibling agent's files as "extra" and delete them. + restore_name = name or parsed_name or ALL_AGENT_NAME + if not name: + name = parsed_name or parsed_fw + + spec = _build_allowlist(framework, restore_name, args.local_dir) root = spec.workspace_root # ---- Step 1: Backup current local files before any modification ---- diff --git a/ultron/services/harness/merge.py b/ultron/services/harness/merge.py index 6733a76..13dfade 100644 --- a/ultron/services/harness/merge.py +++ b/ultron/services/harness/merge.py @@ -527,6 +527,7 @@ def merge_resources( source_defaults: Dict[str, str], target_defaults: Dict[str, str], existing_skills: Optional[List[str]] = None, + overflow_target: Optional[str] = None, ) -> FullMergeResult: """Merge incoming resources into a target product workspace. @@ -537,6 +538,12 @@ def merge_resources( source_defaults: default templates for source product target_defaults: default templates for target product existing_skills: list of skill dir names already on target (for skip detection) + overflow_target: when given, mutually-exclusive ("overflow") content + that has no semantic mapping on the target is routed to *this* path + instead of the shared catch-all file. Used for file-per-agent + targets (e.g. qoder ``agents/{name}.md``) so a converted persona + lands in its own sub-agent file rather than polluting the shared + ``AGENTS.md``. Returns: FullMergeResult with merged_files and actions list. @@ -600,7 +607,7 @@ def merge_resources( detail=f"{path} has no equivalent in {target_product} and no user changes, skipped", )) continue - catch_all = _catch_all_file(target_product) + catch_all = overflow_target or _catch_all_file(target_product) block = f"## Imported from {source_product} {path}\n\n{user_diff}\n" overflow_blocks.append((catch_all, block)) result.actions.append(MergeAction(