diff --git a/.env b/.env deleted file mode 100755 index b18fc6bfb..000000000 --- a/.env +++ /dev/null @@ -1,4 +0,0 @@ -BASE_URL= -API_KEY= -SERPER_DEV_API_KEY= -JINA_API_KEY= \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..088d3f63f --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# ============================================================================ +# LLM Provider Configuration +# ============================================================================ +# BASE_URL and API_KEY are the standard configurations for model call authentication. +# These variables support OpenAI, Gemini, LM Studio, Ollama, and other providers. + +BASE_URL=https://api.openai.com/v1 +API_KEY=sk-your-openai-api-key-here + +# Example BASE_URL values: +# - OpenAI: https://api.openai.com/v1 +# - Gemini: https://generativelanguage.googleapis.com/v1beta/openai/ +# - LM Studio: http://localhost:1234/v1 +# - Ollama: http://localhost:11434/v1 + +# ============================================================================ +# Optional: Web Search and Reading Tools +# ============================================================================ + +# SERPER_DEV_API_KEY=your-serper-api-key-here +# Get from: https://serper.dev + +# JINA_API_KEY=your-jina-api-key-here +# Get from: https://jina.ai \ No newline at end of file diff --git a/.github/workflows/validate-yamls.yml b/.github/workflows/validate-yamls.yml new file mode 100644 index 000000000..cde6d64b4 --- /dev/null +++ b/.github/workflows/validate-yamls.yml @@ -0,0 +1,65 @@ +name: Validate YAML Workflows + +on: + pull_request: + paths: + - 'yaml_instance/**/*.yaml' + - '.github/workflows/**/*.yml' + - 'tools/validate_all_yamls.py' + - 'check/**/*.py' + + push: + branches: + - main + paths: + - 'yaml_instance/**/*.yaml' + - '.github/workflows/**/*.yml' + - 'tools/validate_all_yamls.py' + - 'check/**/*.py' + + workflow_dispatch: + +jobs: + validate: + name: Validate YAML Configuration Files + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Cache uv dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install dependencies + run: uv sync + + - name: Run YAML validation + run: uv run python tools/validate_all_yamls.py + + - name: Report validation results + if: always() + run: | + if [ $? -eq 0 ]; then + echo "All YAML workflow files passed validation" + else + echo "YAML validation failed - check the logs above for details" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 64b57f840..96b6481f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,29 @@ -*.pyc -.DS_Store -.idea -.vscode +# Python __pycache__/ -.env/ +*.pyc + +# Virtual environments .venv/ -env/ venv/ -.idea -.venv -.uv-cache -logs -node_modules -frontend/.vscode -WareHouse/ +env/ + +# uv +.uv-cache/ + +# IDEs +.idea/ +.vscode/ +frontend/.vscode/ + +# OS +.DS_Store + +# Environment +.env + +# Project Specific +logs/ +node_modules/ data/ -temp/ \ No newline at end of file +temp/ +WareHouse/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..73c7e953a --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +# ============================================================================== +# Development Commands +# ============================================================================== + +.PHONY: dev +dev: server client ## Run both backend and frontend development servers + +.PHONY: server +server: ## Start the backend server in the background + @echo "Starting server in background..." + @uv run python server_main.py --port 6400 --reload & + +.PHONY: client +client: ## Start the frontend development server + @cd frontend && VITE_API_BASE_URL=http://localhost:6400 npm run dev + +.PHONY: stop +stop: ## Stop backend and frontend servers + @echo "Stopping backend server (port 6400)..." + @lsof -t -i:6400 | xargs kill -9 2>/dev/null || echo "Backend server not found on port 6400." + @echo "Stopping frontend server (port 5173)..." + @lsof -t -i:5173 | xargs kill -9 2>/dev/null || echo "Frontend server not found on port 5173." + +# ============================================================================== +# Tools & Maintenance +# ============================================================================== + +.PHONY: sync +sync: ## Sync Vue graphs to the server database + @uv run python tools/sync_vuegraphs.py + +.PHONY: validate-yamls +validate-yamls: ## Validate all YAML configuration files + @uv run python tools/validate_all_yamls.py + +# ============================================================================== +# Help +# ============================================================================== + +.PHONY: help +help: ## Display this help message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index e4328cc1b..4a640606a 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,30 @@ See our paper in [Multi-Agent Collaboration via Evolving Orchestration](https:// cd frontend && npm install ``` +### 🔑 Configuration + +* **Environment Variables**: + ```bash + cp .env.example .env + ``` +* **Model Keys**: Set `API_KEY` and `BASE_URL` in `.env` for your LLM provider. +* **YAML placeholders**: Use `${VAR}`(e.g., `${API_KEY}`)in configuration files to reference these variables. + ### ⚡️ Run the Application -1. **Start Backend** : + +#### Using Makefile (Recommended) + +**Start both Backend and Frontent**: +```bash +make dev +``` + +> Then access the Web Console at **[http://localhost:5173](http://localhost:5173)**. + +#### Manual Commands + +1. **Start Backend**: ```bash # Run from the project root uv run python server_main.py --port 6400 --reload @@ -143,12 +164,25 @@ See our paper in [Multi-Agent Collaboration via Evolving Orchestration](https:// > * **Backend**: start with `--port 6401` > * **Frontend**: set `VITE_API_BASE_URL=http://localhost:6401` +#### Utility Commands -### 🔑 Configuration +* **Help command**: + ```bash + make help + ``` + +* **Sync YAML workflows to frontend**: + ```bash + make sync + ``` + Uploads all workflow files from `yaml_instance/` to the database. + +* **Validate all YAML workflows**: + ```bash + make validate-yamls + ``` + Checks all YAML files for syntax and schema errors. -* **Environment Variables**: Create a `.env` file in the project root. -* **Model Keys**: Set `API_KEY` and `BASE_URL` in `.env` for your LLM provider. -* **YAML placeholders**: Use `${VAR}`(e.g., `${API_KEY}`)in configuration files to reference these variables. --- diff --git a/check/check.py b/check/check.py index 39ba04079..12dcd6aa6 100755 --- a/check/check.py +++ b/check/check.py @@ -118,4 +118,4 @@ def check_config(yaml_content: Any) -> str: except Exception as e: return str(e) - return "" + return "" \ No newline at end of file diff --git a/tools/sync_vuegraphs.py b/tools/sync_vuegraphs.py new file mode 100644 index 000000000..60bb7ee9f --- /dev/null +++ b/tools/sync_vuegraphs.py @@ -0,0 +1,65 @@ +""" +Synchronize YAML Configurations to VueGraph Database + +This tool uploads local YAML workflow configurations from the yaml_instance/ +directory to the VueGraph database via the API endpoint. This is essential for +making workflow configurations available to the frontend visualization system. + +Purpose: +- Ensures the database reflects the latest YAML configurations +- Required after modifying workflow YAML files to see changes in the UI +- Useful for development and deployment workflows + +Usage: + python tools/sync_vuegraphs.py + # or via Makefile: + make sync +""" + +import os +import glob +import requests +import yaml +from pathlib import Path + +# Configuration +API_URL = "http://localhost:6400/api/vuegraphs/upload/content" +YAML_DIR = "yaml_instance" + + +def sync_yaml_to_vuegraphs(): + """Reads all YAML files and uploads them to the VueGraph database.""" + print(f"Syncing YAML files from {YAML_DIR} to {API_URL}...") + + yaml_files = glob.glob(os.path.join(YAML_DIR, "*.yaml")) + + for file_path in yaml_files: + try: + filename = Path(file_path).stem # simulation_hospital_lmstudio + + with open(file_path, "r") as f: + content = f.read() + + # Basic validation to ensure it's a valid YAML + try: + yaml.safe_load(content) + except yaml.YAMLError as e: + print(f"Skipping {filename}: Invalid YAML - {e}") + continue + + # Upload to VueGraph API + payload = {"filename": filename, "content": content} + + response = requests.post(API_URL, json=payload) + + if response.status_code == 200: + print(f"Synced: {filename}") + else: + print(f"Failed: {filename} - {response.status_code} {response.text}") + + except Exception as e: + print(f"Error processing {file_path}: {e}") + + +if __name__ == "__main__": + sync_yaml_to_vuegraphs() diff --git a/tools/validate_all_yamls.py b/tools/validate_all_yamls.py new file mode 100644 index 000000000..19f89d49b --- /dev/null +++ b/tools/validate_all_yamls.py @@ -0,0 +1,104 @@ +""" +Validate All YAML Workflow Configurations + +This tool performs strict validation on all YAML workflow configuration files +in the yaml_instance/ directory. It ensures configuration integrity and prevents +runtime errors by catching issues early in the development process. + +Purpose: +- Validates YAML syntax and schema compliance for all workflow configurations +- Prevents invalid configurations from causing runtime failures +- Essential for CI/CD pipelines to ensure code quality +- Provides detailed error reporting for debugging + +Usage: + python tools/validate_all_yamls.py + # or via Makefile: + make validate-yamls +""" + +import sys +import subprocess +from pathlib import Path + + +def validate_all(): + base_dir = Path("yaml_instance") + if not base_dir.exists(): + print(f"Directory {base_dir} not found.") + sys.exit(1) + + # Recursive search for all .yaml files + files = sorted(list(base_dir.rglob("*.yaml"))) + + if not files: + print("No YAML files found.") + return + + print( + f"Found {len(files)} YAML files. Running FULL validation via check.check...\n" + ) + + passed = 0 + failed = 0 + failed_files = [] + + for yaml_file in files: + # Use relative path for cleaner output + try: + rel_path = yaml_file.relative_to(Path.cwd()) + except ValueError: + rel_path = yaml_file + + # NOW we run check.check, which we just patched to have a main() + # This performs the stricter load_config() validation + cmd = [sys.executable, "-m", "check.check", "--path", str(yaml_file)] + + try: + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + print(f"{rel_path}") + passed += 1 + else: + print(f"{rel_path}") + # Indent error output + if result.stdout: + print(" stdout:", result.stdout.strip().replace("\n", "\n ")) + # Validation errors usually print to stdout/stderr depending on impl + # Our new main prints to stdout for success/failure message + failed += 1 + failed_files.append(str(rel_path)) + except Exception as e: + print(f"{rel_path} (Execution Failed)") + print(f" Error: {e}") + failed += 1 + failed_files.append(str(rel_path)) + + print("\n" + "=" * 40) + print(f"YAML Validation Summary") + print("=" * 40) + print(f"Total Files: {len(files)}") + print(f"Passed: {passed}") + print(f"Failed: {failed}") + + if failed > 0: + print("\nFailed Files:") + for f in failed_files: + print(f"- {f}") + + # Overall validation status + print("\n" + "=" * 40) + print("Overall Validation Status") + print("=" * 40) + + if failed > 0: + print("YAML validation: FAILED") + sys.exit(1) + else: + print("All validations passed successfully.") + sys.exit(0) + + +if __name__ == "__main__": + validate_all()