From d1a1649520c91e4bab345dcbd292c2c9793114dc Mon Sep 17 00:00:00 2001 From: Dan Merino Date: Thu, 21 May 2026 16:26:04 -0700 Subject: [PATCH 01/11] r2 docker compose (#263) --- Dockerfile | 4 + appArmor/README.md | 55 +++ appArmor/docker-default | 59 +++ compose.yaml | 145 +++++++ gvisor-seccomp.json | 447 ++++++++++++++++++++ install.sh | 70 +++- nsjail-seccomp.json | 692 +++++++++++++++++++++++++++++++ scripts/setup-docker-apparmor.sh | 135 ++++++ 8 files changed, 1606 insertions(+), 1 deletion(-) create mode 100644 appArmor/docker-default create mode 100644 gvisor-seccomp.json create mode 100644 nsjail-seccomp.json create mode 100755 scripts/setup-docker-apparmor.sh diff --git a/Dockerfile b/Dockerfile index 9f295bd..12a66f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,12 @@ ARG VERSION=X.Y.Z-stable +FROM tryretool/agent-sandbox-service:${VERSION} AS agent-sandbox + FROM tryretool/code-executor-service:${VERSION} AS code-executor +FROM tryretool/js-executor-service:${VERSION} AS js-executor + FROM tryretool/backend:${VERSION} CMD ./docker_scripts/start_api.sh diff --git a/appArmor/README.md b/appArmor/README.md index 4e15056..6f749a7 100644 --- a/appArmor/README.md +++ b/appArmor/README.md @@ -20,3 +20,58 @@ sudo apt install apparmor-profiles sudo apparmor_parser /etc/apparmor.d/usr.bin.nsjail ``` +## Agent Sandbox: replacing `docker-default` + +The agent-sandbox-controller spawns sandbox containers over the Docker socket. Inside each sandbox, two mount-using tools run: + +- [pasta](https://passt.top/) sets up isolated networking and remounts `/` to make the root mount private. +- [runsc (gVisor)](https://gvisor.dev/) in `--rootless` mode mounts `proc`, `tmpfs`, `/dev`, and calls `pivot_root` to assemble its second-level sandbox — even when networking is disabled. + +The Docker daemon auto-applies its built-in `docker-default` AppArmor profile to every container it starts. The bundled `gvisor-seccomp.json` profile already permits `mount` / `umount2` / `pivot_root` (those are explicitly allow-listed for gVisor + pasta), but AppArmor sits in front of seccomp and rejects the syscall first. The visible symptom is pasta exiting with `Failed to remount /: Permission denied`; runsc's mount calls would be the next failure if pasta were skipped. + +`docker-default` is loaded into the kernel by the Docker daemon at startup; it usually isn't present as a file in `/etc/apparmor.d/`. We ship a replacement profile (`docker-default` in this directory) with the same name. Loading it with `apparmor_parser -r` replaces the in-kernel version, and new containers — including those spawned by the agent-sandbox-controller — pick up our variant. The rest of upstream `docker-default`'s hardening (the `/proc`, `/sys`, sysrq, ptrace denylists) is kept intact. + +### What's different from upstream `docker-default` + +Three deltas: + +1. **Replaced `deny mount,` with `mount,`.** Upstream blocks all mount operations at the AppArmor layer. Agent-sandbox needs `mount` for pasta (remount `/` private) and runsc (mount `proc` / `tmpfs` / `/dev`). A bare `mount,` rule is a broad allow. +2. **Added `pivot_root,`.** `pivot_root` is a distinct AppArmor mediation class from `mount` and is also default-deny once any other mount-class rule exists in the profile. pasta and runsc both call `pivot_root` to swap rootfs into their sandboxes, so we need an explicit allow. Symptom without it: `apparmor="DENIED" operation="pivotroot" ...` for `comm="passt.avx2"` and `comm="exe"` (runsc) on `/tmp/` and `/proc/fs/`. +3. **Explicit `signal` rules.** Upstream historically didn't include any `signal` rules. AppArmor 4.x (Ubuntu 24.04+) treats a profile that has *any* rules but no `signal` rules as default-deny for cross-profile signals, which surfaces as `apparmor="DENIED" operation="signal" ... comm="runc"` audit lines during container teardown. We add explicit rules for peer=docker-default and receive-from-unconfined to match the implicit behavior on older kernels. + +All `/proc`, `/sys`, sysrq, kcore, powercap, security and ptrace deny rules are kept verbatim from upstream so the rest of the hardening surface is unchanged. + +### Why an explicit `mount,` rule is needed (and not just dropping the `deny`) + +`#include ` brings in narrow mount rules on Ubuntu 24.04+ (e.g. specific tmpfs / proc allowances). That puts the profile into "default-deny for unmatched mounts" mode — once any mount rule exists in the evaluated profile, every other mount must match an allow rule. Removing `deny mount,` alone leaves only the abstraction's narrow allows, so pasta's `mount / / -o rw,rslave` falls through to a default-deny and the audit log shows `info="failed mntpnt match"`. The broad `mount,` rule explicitly permits unmatched mount syscalls. + +### Installation + +Run the bundled script: + +``` +sudo ./scripts/setup-docker-apparmor.sh +``` + +It is idempotent and skips cleanly on hosts without AppArmor (e.g. macOS Docker Desktop). What it does: + +1. Copies the bundled `docker-default` profile to `/etc/apparmor.d/docker-default`. +2. Loads it into the kernel with `apparmor_parser -r` so new containers pick it up. +3. Installs a systemd drop-in at `/etc/systemd/system/docker.service.d/retool-apparmor.conf` containing `ExecStartPre=-/usr/sbin/apparmor_parser -r /etc/apparmor.d/docker-default`. This makes the docker daemon re-apply our profile on every start, so the override survives `systemctl restart docker` and host reboots without manual intervention. + +`./install.sh` invokes this script automatically; only run it directly if you're applying the AppArmor change to an existing install or to a host where `install.sh` was run on an older version of this repo. + +You do **not** need to restart Docker after a first-time install — already-running containers keep their existing profile, and any new container will use the replaced version. + +### Verifying it's in effect + +After loading, trigger a fresh sandbox spawn and tail the kernel audit log: + +``` +sudo dmesg -wT | grep -iE 'apparmor|audit' +``` + +You should see no `apparmor="DENIED"` lines for `operation="mount"` or `operation="signal"`. If you do, capture the line — it points at the next rule that needs widening. + +This step only applies on Linux hosts where AppArmor is active. macOS hosts running Docker Desktop don't enforce AppArmor (the LinuxKit VM kernel ships without it), so no override is needed there; `apparmor_parser` won't be available on those machines. + diff --git a/appArmor/docker-default b/appArmor/docker-default new file mode 100644 index 0000000..596de87 --- /dev/null +++ b/appArmor/docker-default @@ -0,0 +1,59 @@ +#include + + +profile docker-default flags=(attach_disconnected,mediate_deleted) { + + #include + + + network, + capability, + file, + umount, + # NOTE: upstream docker-default has `deny mount,` here. It is replaced with an + # explicit broad `mount,` allow so that — inside containers the + # agent-sandbox-controller spawns — pasta can remount `/` for its filesystem + # isolation AND runsc (gVisor) can mount proc / tmpfs / /dev when assembling + # its second-level sandbox. The bundled gvisor-seccomp.json already permits + # mount / umount2 / pivot_root, but AppArmor sits in front of seccomp. + # + # An explicit `mount,` rule is required (rather than just dropping the deny): + # `#include ` brings in some narrow mount rules on + # Ubuntu 24.04+, which puts the profile into "default-deny for unmatched + # mounts" mode. Without a broad allow, pasta's `mount / / -o rw,rslave` + # fails with `failed mntpnt match`. + # + # The other denylists below are kept verbatim from upstream so the rest of + # docker-default's hardening (proc, sys, sysrq, kcore, ptrace) is preserved. + mount, + + # pivot_root is a distinct AppArmor mediation class from mount. pasta and + # runsc both pivot_root when assembling their respective sandboxes, so we + # need an explicit allow. + pivot_root, + + deny @{PROC}/* w, # deny write for all files directly in /proc + deny @{PROC}/sys/[^k]*/** wklx, # deny write to /proc/sys except /proc/sys/k* + deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} wklx, # deny /proc/sys/kernel/* except *shm* + deny @{PROC}/sysrq-trigger rwklx, + deny @{PROC}/kcore rwklx, + + deny /sys/[^f]*/** wklx, + deny /sys/f[^s]*/** wklx, + deny /sys/fs/[^c]*/** wklx, + deny /sys/fs/c[^g]*/** wklx, + deny /sys/fs/cg[^r]*/** wklx, + deny /sys/firmware/** rwklx, + deny /sys/devices/virtual/powercap/** rwklx, + deny /sys/kernel/security/** rwklx, + + # Suppress ptrace denials when running `ps` / `docker ps` inside a container. + ptrace (trace,read,tracedby,readby) peer=docker-default, + + # Signal mediation. AppArmor 4.x (Ubuntu 24.04+) defaults to deny when a + # profile has any rules but no signal rules; this allows signals between + # docker-default-confined processes (container peers) and from the + # unconfined Docker daemon (e.g. SIGKILL during teardown). + signal (send,receive) peer=docker-default, + signal (receive) peer=unconfined, +} diff --git a/compose.yaml b/compose.yaml index c1a4365..84c26ee 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,8 +15,10 @@ services: - frontend - backend - code-executor + - js-executor depends_on: - postgres + - minio restart: always jobs-runner: @@ -40,6 +42,7 @@ services: networks: - backend - code-executor + - js-executor depends_on: - postgres restart: always @@ -54,6 +57,7 @@ services: networks: - backend - code-executor + - js-executor depends_on: - postgres restart: always @@ -68,6 +72,7 @@ services: networks: - backend - code-executor + - js-executor depends_on: - postgres restart: always @@ -82,6 +87,7 @@ services: networks: - backend - code-executor + - js-executor depends_on: - postgres restart: always @@ -103,6 +109,141 @@ services: - code-executor restart: always + js-executor: + build: + context: . + target: js-executor + + # js-executor reads its memory budget from the cgroup limit, so this + # must be set explicitly. Matches the helm chart default (2048Mi). + mem_limit: 4g + + # Custom seccomp profile is required for nsjail's syscalls to pass Docker's + # default seccomp filter. Applies under both options below. + security_opt: + - seccomp=./nsjail-seccomp.json + + # Option 1 (preferred): Grant NET_ADMIN so the executor can install + # iptables egress rules. Requires the host kernel to expose the + # netfilter `filter` table. + cap_add: + - NET_ADMIN + + # Option 2: Skip iptables-based egress filtering. Use this if NET_ADMIN + # isn't available or the host kernel lacks netfilter support. nsjail still + # wraps user code; only the network-egress filter is skipped. Note: do + # not also drop to a non-root user here — the entrypoint uses `runuser` + # internally to drop privileges and that requires starting as root. + # environment: + # - DISABLE_IPTABLES_SECURITY_CONFIGURATION=true + + networks: + - js-executor + restart: always + + agent-sandbox-controller: + build: + context: . + target: agent-sandbox + user: root + env_file: docker.env + environment: + - NODE_ENV=production + - AGENT_EXECUTOR_ROLE=controller + - ORCHESTRATOR=docker + - EXECUTOR_IMAGE=${EXECUTOR_IMAGE:-retool-onpremise-agent-sandbox-controller} + - DOCKER_NETWORK=agent-sandbox + - EXECUTOR_SERVICE_NAME=agent-sandbox + - SANDBOX_BACKEND_URL=http://api:3000 + - EXECUTOR_EXTRA_ENV=SANDBOX_NETWORK_ENABLED=false,SANDBOX_HTTP_PROXY=http://agent-sandbox-proxy:3019 + - EXECUTOR_SECCOMP_PROFILE_PATH=/seccomp/gvisor-seccomp.json + # Lower pool sizes for local development + - PREWARM_POOL_SIZE=1 + - MAX_TOTAL_JOBS=10 + - MAX_CONCURRENT_CREATES=2 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./gvisor-seccomp.json:/seccomp/gvisor-seccomp.json:ro + networks: + - backend + - agent-sandbox + depends_on: + - postgres + restart: always + + agent-sandbox-proxy: + build: + context: . + target: agent-sandbox + env_file: docker.env + environment: + - NODE_ENV=production + - AGENT_EXECUTOR_ROLE=proxy + - ORCHESTRATOR=docker + - PROXY_PORT=3019 + - EXECUTOR_SERVICE_NAME=agent-sandbox + - BACKEND_URL=http://api:3000 + # Uncomment to restrict which external domains sandboxes can reach + # - ALLOWED_DOMAINS=api.openai.com,api.anthropic.com + ports: + - 3019:3019 + networks: + - backend + - agent-sandbox + depends_on: + - postgres + restart: always + + r2-agent-worker: + build: + context: . + env_file: docker.env + environment: + - SERVICE_TYPE=R2_AGENT_TEMPORAL_WORKER + - WORKER_TEMPORAL_TASKQUEUE=r2-agent + networks: + - backend + - code-executor + - js-executor + - agent-sandbox + depends_on: + - postgres + restart: always + + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + - MINIO_ROOT_USER=retool + - MINIO_ROOT_PASSWORD=retoolminio + - MINIO_DOMAIN=minio + ports: + - 9000:9000 + - 9001:9001 + networks: + backend: + aliases: + - retool-rr-git.minio + - retool-rr-snapshots.minio + volumes: + - minio-data:/data + restart: always + + minio-init: + image: minio/mc:latest + depends_on: + - minio + entrypoint: > + /bin/sh -c " + sleep 3; + mc alias set local http://minio:9000 retool retoolminio; + mc mb --ignore-existing local/retool-rr-git; + mc mb --ignore-existing local/retool-rr-snapshots; + echo 'Buckets created successfully'; + " + networks: + - backend + # Retool's internal DB, we recommend using an externally hosted database: https://docs.retool.com/docs/configuring-retools-storage-database postgres: image: postgres:16.8 @@ -146,7 +287,11 @@ networks: frontend: backend: code-executor: + js-executor: + agent-sandbox: + name: agent-sandbox volumes: data: retooldb-data: + minio-data: diff --git a/gvisor-seccomp.json b/gvisor-seccomp.json new file mode 100644 index 0000000..9b2a1de --- /dev/null +++ b/gvisor-seccomp.json @@ -0,0 +1,447 @@ +{ + "comment": "Docker default seccomp profile extended with syscalls required by gVisor runsc (systrap platform, rootless mode). Use with: docker run --security-opt seccomp=gvisor-seccomp.json", + "defaultAction": "SCMP_ACT_ERRNO", + "defaultErrnoRet": 1, + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": ["SCMP_ARCH_X86", "SCMP_ARCH_X32"] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": [] + } + ], + "syscalls": [ + { + "comment": "Docker default allowlist (Docker 27.x, x86_64 + aarch64)", + "names": [ + "_llseek", + "_newselect", + "accept", + "accept4", + "access", + "acct", + "adjtimex", + "alarm", + "arch_prctl", + "bind", + "bpf", + "brk", + "cachestat", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "chroot", + "clock_adjtime", + "clock_adjtime64", + "clock_getres", + "clock_getres_time64", + "clock_gettime", + "clock_gettime64", + "clock_nanosleep", + "clock_nanosleep_time64", + "clock_settime", + "clock_settime64", + "close", + "close_range", + "connect", + "copy_file_range", + "creat", + "delete_module", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_pwait2", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "faccessat2", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_init", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchmodat2", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "finit_module", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsconfig", + "fsetxattr", + "fsmount", + "fsopen", + "fspick", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futex_requeue", + "futex_time64", + "futex_wait", + "futex_waitv", + "futex_wake", + "futimesat", + "get_mempolicy", + "get_robust_list", + "get_thread_area", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "init_module", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "io_cancel", + "io_destroy", + "io_getevents", + "io_pgetevents", + "io_pgetevents_time64", + "io_setup", + "io_submit", + "io_uring_enter", + "io_uring_register", + "io_uring_setup", + "ioctl", + "ioperm", + "iopl", + "ioprio_get", + "ioprio_set", + "ipc", + "kcmp", + "kill", + "landlock_add_rule", + "landlock_create_ruleset", + "landlock_restrict_self", + "lchown", + "lchown32", + "lgetxattr", + "link", + "linkat", + "listen", + "listxattr", + "llistxattr", + "lookup_dcookie", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "map_shadow_stack", + "mbind", + "membarrier", + "memfd_create", + "memfd_secret", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "modify_ldt", + "mount_setattr", + "move_mount", + "mprotect", + "mq_getsetattr", + "mq_notify", + "mq_open", + "mq_timedreceive", + "mq_timedreceive_time64", + "mq_timedsend", + "mq_timedsend_time64", + "mq_unlink", + "mremap", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munlock", + "munlockall", + "munmap", + "name_to_handle_at", + "nanosleep", + "newfstatat", + "open", + "open_by_handle_at", + "open_tree", + "openat", + "openat2", + "pause", + "perf_event_open", + "pidfd_getfd", + "pidfd_open", + "pidfd_send_signal", + "pipe", + "pipe2", + "pkey_alloc", + "pkey_free", + "pkey_mprotect", + "poll", + "ppoll", + "ppoll_time64", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "process_madvise", + "process_mrelease", + "process_vm_readv", + "process_vm_writev", + "pselect6", + "pselect6_time64", + "pwrite64", + "pwritev", + "pwritev2", + "quotactl", + "quotactl_fd", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "reboot", + "recv", + "recvfrom", + "recvmmsg", + "recvmmsg_time64", + "recvmsg", + "remap_file_pages", + "removexattr", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "rmdir", + "rseq", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_sigtimedwait_time64", + "rt_tgsigqueueinfo", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_rr_get_interval_time64", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "semtimedop_time64", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "set_mempolicy", + "set_mempolicy_home_node", + "set_robust_list", + "set_thread_area", + "set_tid_address", + "set_tls", + "setdomainname", + "setfsgid", + "setfsgid32", + "setfsuid", + "setfsuid32", + "setgid", + "setgid32", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setresuid", + "setresuid32", + "setreuid", + "setreuid32", + "setrlimit", + "setsid", + "setsockopt", + "settimeofday", + "setuid", + "setuid32", + "setxattr", + "shmat", + "shmctl", + "shmdt", + "shmget", + "shutdown", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socket", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statx", + "stime", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "sync_file_range2", + "syncfs", + "sysinfo", + "syslog", + "tee", + "tgkill", + "time", + "timer_create", + "timer_delete", + "timer_getoverrun", + "timer_gettime", + "timer_gettime64", + "timer_settime", + "timer_settime64", + "timerfd_create", + "timerfd_gettime", + "timerfd_gettime64", + "timerfd_settime", + "timerfd_settime64", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "umount", + "uname", + "unlink", + "unlinkat", + "utime", + "utimensat", + "utimensat_time64", + "utimes", + "vfork", + "vhangup", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev" + ], + "action": "SCMP_ACT_ALLOW" + }, + { + "comment": "gVisor + pasta: namespace creation and entry (clone/unshare with CLONE_NEW* flags, setns to join namespaces)", + "names": ["clone", "clone3", "unshare", "setns"], + "action": "SCMP_ACT_ALLOW" + }, + { + "comment": "pasta: set hostname inside namespace (cosmetic, avoids warning)", + "names": ["sethostname"], + "action": "SCMP_ACT_ALLOW" + }, + { + "comment": "gVisor: sandbox filesystem setup (tmpfs, proc, bind mounts)", + "names": ["mount", "umount2"], + "action": "SCMP_ACT_ALLOW" + }, + { + "comment": "gVisor: filesystem root isolation for sentry and gofer", + "names": ["pivot_root"], + "action": "SCMP_ACT_ALLOW" + }, + { + "comment": "gVisor systrap platform: workload executor thread initialization", + "names": ["ptrace"], + "action": "SCMP_ACT_ALLOW" + } + ] +} diff --git a/install.sh b/install.sh index efc8df1..2c67370 100755 --- a/install.sh +++ b/install.sh @@ -61,6 +61,41 @@ if [ -f /etc/os-release ]; then fi fi +# Install the customized docker-default AppArmor profile + systemd drop-in +# so it survives docker restarts. The script handles its own preflight +# (skips silently on hosts without AppArmor); --check tells us whether any +# persistent changes are needed so we only prompt the user when there's +# actually work to do. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APPARMOR_SCRIPT="${SCRIPT_DIR}/scripts/setup-docker-apparmor.sh" +if [ -x "$APPARMOR_SCRIPT" ]; then + if "$APPARMOR_SCRIPT" --check; then + echo " ✅ docker-default AppArmor profile already configured (or not applicable on this host)." + else + echo "" + echo " Agent-sandbox needs the bundled docker-default AppArmor profile installed." + echo " The script will:" + echo " - copy appArmor/docker-default to /etc/apparmor.d/docker-default" + echo " - load it into the kernel via 'apparmor_parser -r'" + echo " - install a systemd drop-in at" + echo " /etc/systemd/system/docker.service.d/retool-apparmor.conf so the" + echo " profile is re-applied on every docker.service start" + echo " Requires sudo. See appArmor/README.md for details." + read -p " Run it now? [Y/n]: " apparmor_confirm + case "${apparmor_confirm:-y}" in + [yY]|[yY][eE][sS]) + "$APPARMOR_SCRIPT" || \ + echo " ⚠️ docker-default AppArmor setup failed; agent-sandbox may not work. See appArmor/README.md for manual steps." + ;; + *) + echo " ⏭️ Skipped docker-default AppArmor setup." + echo " ⚠️ Agent-sandbox will fail with apparmor=\"DENIED\" errors until you run" + echo " '${APPARMOR_SCRIPT}' manually." + ;; + esac + fi +fi + echo "" [[ -f docker.env ]] && echo "⚠️ docker.env file already exists, skipping initializing it!" && exit 1 @@ -78,6 +113,13 @@ echo "" random() { cat /dev/urandom | base64 | head -c "$1" | tr -d +/ ; } +postgres_password=$(random 64) + +ae_private_pem=$(openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 2>/dev/null) +ae_public_pem=$(echo "$ae_private_pem" | openssl ec -pubout 2>/dev/null) +ae_private_key=$(echo "$ae_private_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') +ae_public_key=$(echo "$ae_public_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') + cat << EOF > docker.env # Environment variables reference: docs.retool.com/docs/environment-variables DEPLOYMENT_TEMPLATE_TYPE=docker-compose @@ -87,7 +129,7 @@ POSTGRES_HOST=postgres POSTGRES_DB=hammerhead_production POSTGRES_PORT=5432 POSTGRES_USER=retool_internal_user -POSTGRES_PASSWORD=$(random 64) +POSTGRES_PASSWORD=$postgres_password # Retool DB credentials RETOOLDB_POSTGRES_HOST=retooldb-postgres @@ -99,6 +141,32 @@ RETOOLDB_POSTGRES_PASSWORD=$(random 64) # Workflows configuration WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 CODE_EXECUTOR_INGRESS_DOMAIN=http://code-executor:3004 +JS_EXECUTOR_INGRESS_DOMAIN=http://js-executor:3000 + +# Agent sandbox configuration +AGENT_EXECUTOR_ENABLED=true +RR_AGENT_PUBSUB_BACKEND=postgres +AGENT_EXECUTOR_CONTROLLER_INGRESS_DOMAIN=http://agent-sandbox-controller:3018 +AGENT_EXECUTOR_PROXY_INGRESS_DOMAIN=http://agent-sandbox-proxy:3019 +AGENT_EXECUTOR_FRONTEND_WS_PROXY_DOMAIN=http://$hostname:3019 +AGENT_EXECUTOR_JWT_PRIVATE_KEY="$ae_private_key" +AGENT_EXECUTOR_JWT_PUBLIC_KEY="$ae_public_key" +AGENT_EXECUTOR_ENCRYPTION_KEY=$(openssl rand -hex 32) +STATE_BACKEND=postgres +AGENT_EXECUTOR_POSTGRES_URL=postgres://retool_internal_user:$postgres_password@postgres:5432/hammerhead_production +AGENT_EXECUTOR_POSTGRES_SCHEMA=agent_executor + +# S3-compatible storage (MinIO) +AWS_ENDPOINT_URL=http://minio:9000 +RR_GIT_S3_BUCKET=retool-rr-git +RR_GIT_S3_ACCESS_KEY_ID=retool +RR_GIT_S3_SECRET_ACCESS_KEY=retoolminio +RR_GIT_S3_REGION=us-east-1 +RR_SNAPSHOTS_S3_BUCKET=retool-rr-snapshots +RR_SNAPSHOTS_S3_ACCESS_KEY_ID=retool +RR_SNAPSHOTS_S3_SECRET_ACCESS_KEY=retoolminio +RR_SNAPSHOTS_S3_REGION=us-east-1 +RR_SNAPSHOTS_S3_ENDPOINT=http://minio:9000 # Comment out below to use Retool-managed Temporal (Enterprise license) WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal diff --git a/nsjail-seccomp.json b/nsjail-seccomp.json new file mode 100644 index 0000000..42c9c5c --- /dev/null +++ b/nsjail-seccomp.json @@ -0,0 +1,692 @@ +{ + "defaultAction": "SCMP_ACT_ERRNO", + "defaultErrnoRet": 1, + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": ["SCMP_ARCH_X86", "SCMP_ARCH_X32"] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": ["SCMP_ARCH_ARM"] + }, + { + "architecture": "SCMP_ARCH_MIPS64", + "subArchitectures": ["SCMP_ARCH_MIPS", "SCMP_ARCH_MIPS64N32"] + }, + { + "architecture": "SCMP_ARCH_MIPS64N32", + "subArchitectures": ["SCMP_ARCH_MIPS", "SCMP_ARCH_MIPS64"] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64", + "subArchitectures": ["SCMP_ARCH_MIPSEL", "SCMP_ARCH_MIPSEL64N32"] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64N32", + "subArchitectures": ["SCMP_ARCH_MIPSEL", "SCMP_ARCH_MIPSEL64"] + }, + { + "architecture": "SCMP_ARCH_S390X", + "subArchitectures": ["SCMP_ARCH_S390"] + }, + { + "architecture": "SCMP_ARCH_RISCV64", + "subArchitectures": null + } + ], + "syscalls": [ + { + "names": [ + "accept", + "accept4", + "access", + "adjtimex", + "alarm", + "bind", + "brk", + "cachestat", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "clock_adjtime", + "clock_adjtime64", + "clock_getres", + "clock_getres_time64", + "clock_gettime", + "clock_gettime64", + "clock_nanosleep", + "clock_nanosleep_time64", + "close", + "close_range", + "connect", + "copy_file_range", + "creat", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_pwait2", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "faccessat2", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchmodat2", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsetxattr", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futex_requeue", + "futex_time64", + "futex_wait", + "futex_waitv", + "futex_wake", + "futimesat", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "get_robust_list", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "get_thread_area", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "getxattrat", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "io_cancel", + "ioctl", + "io_destroy", + "io_getevents", + "io_pgetevents", + "io_pgetevents_time64", + "ioprio_get", + "ioprio_set", + "io_setup", + "io_submit", + "ipc", + "kill", + "landlock_add_rule", + "landlock_create_ruleset", + "landlock_restrict_self", + "lchown", + "lchown32", + "lgetxattr", + "link", + "linkat", + "listen", + "listmount", + "listxattr", + "listxattrat", + "llistxattr", + "_llseek", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "map_shadow_stack", + "membarrier", + "memfd_create", + "memfd_secret", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "mprotect", + "mq_getsetattr", + "mq_notify", + "mq_open", + "mq_timedreceive", + "mq_timedreceive_time64", + "mq_timedsend", + "mq_timedsend_time64", + "mq_unlink", + "mremap", + "mseal", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munlock", + "munlockall", + "munmap", + "name_to_handle_at", + "nanosleep", + "newfstatat", + "_newselect", + "open", + "openat", + "openat2", + "pause", + "pidfd_open", + "pidfd_send_signal", + "pipe", + "pipe2", + "pkey_alloc", + "pkey_free", + "pkey_mprotect", + "poll", + "ppoll", + "ppoll_time64", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "process_mrelease", + "pselect6", + "pselect6_time64", + "pwrite64", + "pwritev", + "pwritev2", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "recv", + "recvfrom", + "recvmmsg", + "recvmmsg_time64", + "recvmsg", + "remap_file_pages", + "removexattr", + "removexattrat", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "riscv_hwprobe", + "rmdir", + "rseq", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_sigtimedwait_time64", + "rt_tgsigqueueinfo", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_rr_get_interval_time64", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "semtimedop_time64", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "setfsgid", + "setfsgid32", + "setfsuid", + "setfsuid32", + "setgid", + "setgid32", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setresuid", + "setresuid32", + "setreuid", + "setreuid32", + "setrlimit", + "set_robust_list", + "setsid", + "setsockopt", + "set_thread_area", + "set_tid_address", + "setuid", + "setuid32", + "setxattr", + "setxattrat", + "shmat", + "shmctl", + "shmdt", + "shmget", + "shutdown", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statmount", + "statx", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "syncfs", + "sysinfo", + "tee", + "tgkill", + "time", + "timer_create", + "timer_delete", + "timer_getoverrun", + "timer_gettime", + "timer_gettime64", + "timer_settime", + "timer_settime64", + "timerfd_create", + "timerfd_gettime", + "timerfd_gettime64", + "timerfd_settime", + "timerfd_settime64", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "uname", + "unlink", + "unlinkat", + "uretprobe", + "utime", + "utimensat", + "utimensat_time64", + "utimes", + "vfork", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev" + ], + "action": "SCMP_ACT_ALLOW" + }, + { + "names": ["process_vm_readv", "process_vm_writev", "ptrace"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "minKernel": "4.8" + } + }, + { + "names": ["socket"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 40, + "op": "SCMP_CMP_NE" + } + ] + }, + { + "names": ["personality"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 0, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": ["personality"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 8, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": ["personality"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131072, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": ["personality"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 131080, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": ["personality"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 4294967295, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "names": ["sync_file_range2", "swapcontext"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["ppc64le"] + } + }, + { + "names": ["arm_fadvise64_64", "arm_sync_file_range", "sync_file_range2", "breakpoint", "cacheflush", "set_tls"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["arm", "arm64"] + } + }, + { + "names": ["arch_prctl"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["amd64", "x32"] + } + }, + { + "names": ["modify_ldt"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["amd64", "x32", "x86"] + } + }, + { + "names": ["s390_pci_mmio_read", "s390_pci_mmio_write", "s390_runtime_instr"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["s390", "s390x"] + } + }, + { + "names": ["riscv_flush_icache"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "arches": ["riscv64"] + } + }, + { + "names": ["open_by_handle_at"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_DAC_READ_SEARCH"] + } + }, + { + "names": ["clone", "clone2", "mount", "pivot_root", "sethostname", "umount2"], + "action": "SCMP_ACT_ALLOW", + "comment": "Retool specific syscalls to enable nsjail sandboxing" + }, + { + "names": [ + "bpf", + "clone", + "clone3", + "fanotify_init", + "fsconfig", + "fsmount", + "fsopen", + "fspick", + "lookup_dcookie", + "lsm_get_self_attr", + "lsm_list_modules", + "lsm_set_self_attr", + "mount", + "mount_setattr", + "move_mount", + "open_tree", + "perf_event_open", + "quotactl", + "quotactl_fd", + "setdomainname", + "sethostname", + "setns", + "syslog", + "umount", + "umount2", + "unshare" + ], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_ADMIN"] + } + }, + { + "names": ["clone"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "excludes": { + "caps": ["CAP_SYS_ADMIN"], + "arches": ["s390", "s390x"] + } + }, + { + "names": ["clone"], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 1, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "s390 parameter ordering for clone is different", + "includes": { + "arches": ["s390", "s390x"] + }, + "excludes": { + "caps": ["CAP_SYS_ADMIN"] + } + }, + { + "names": ["clone3"], + "action": "SCMP_ACT_ERRNO", + "errnoRet": 38, + "excludes": { + "caps": ["CAP_SYS_ADMIN"] + } + }, + { + "names": ["reboot"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_BOOT"] + } + }, + { + "names": ["chroot"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_CHROOT"] + } + }, + { + "names": ["delete_module", "init_module", "finit_module"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_MODULE"] + } + }, + { + "names": ["acct"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_PACCT"] + } + }, + { + "names": ["kcmp", "pidfd_getfd", "process_madvise", "process_vm_readv", "process_vm_writev", "ptrace"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_PTRACE"] + } + }, + { + "names": ["iopl", "ioperm"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_RAWIO"] + } + }, + { + "names": ["settimeofday", "stime", "clock_settime", "clock_settime64"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_TIME"] + } + }, + { + "names": ["vhangup"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_TTY_CONFIG"] + } + }, + { + "names": ["get_mempolicy", "mbind", "set_mempolicy", "set_mempolicy_home_node"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYS_NICE"] + } + }, + { + "names": ["syslog"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_SYSLOG"] + } + }, + { + "names": ["bpf"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_BPF"] + } + }, + { + "names": ["perf_event_open"], + "action": "SCMP_ACT_ALLOW", + "includes": { + "caps": ["CAP_PERFMON"] + } + } + ] +} diff --git a/scripts/setup-docker-apparmor.sh b/scripts/setup-docker-apparmor.sh new file mode 100755 index 0000000..0677ccf --- /dev/null +++ b/scripts/setup-docker-apparmor.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Installs the Retool-customized docker-default AppArmor profile and a +# systemd drop-in that auto-reloads it on every docker.service start. +# +# Why this exists: the Docker daemon resets its built-in `docker-default` +# AppArmor profile every time it starts, so without the drop-in our overrides +# revert on any `systemctl restart docker` or host reboot. The drop-in calls +# apparmor_parser -r on each docker start so the right profile is always live. +# +# Assumes system Docker (docker.service). Rootless Docker uses a different +# unit and would need a different drop-in path. +# +# Modes: +# (no args) install / reconcile state; loud, idempotent +# --check report state and exit. 0 = no changes needed (or skip case), +# 1 = persistent changes would be made. Reads only; no sudo +# required, no mutations. Intended for install.sh gating. + +set -euo pipefail + +CHECK_ONLY=0 +case "${1:-}" in + --check) CHECK_ONLY=1 ;; + "") ;; + *) echo "Usage: $0 [--check]" >&2; exit 2 ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +SRC_PROFILE="$REPO_ROOT/appArmor/docker-default" +DST_PROFILE="/etc/apparmor.d/docker-default" +DROPIN_DIR="/etc/systemd/system/docker.service.d" +DROPIN_FILE="$DROPIN_DIR/retool-apparmor.conf" + +# --- Preflight (same in both modes) ------------------------------------- + +if ! command -v apparmor_parser >/dev/null 2>&1; then + [ "$CHECK_ONLY" = "1" ] || echo " ℹ️ apparmor_parser not found — host doesn't use AppArmor. Skipping." + exit 0 +fi + +if [ ! -d /sys/kernel/security/apparmor ]; then + [ "$CHECK_ONLY" = "1" ] || echo " ℹ️ AppArmor not active in this kernel. Skipping." + exit 0 +fi + +if [ ! -f "$SRC_PROFILE" ]; then + echo " ❌ Bundled profile not found at: $SRC_PROFILE" >&2 + exit 2 +fi + +# --- Compute desired state --------------------------------------------- + +read -r -d '' DESIRED_DROPIN_CONTENT </dev/null || $SUDO cat "$DROPIN_FILE" 2>/dev/null || true)" + if [ "$current" = "$DESIRED_DROPIN_CONTENT" ]; then + dropin_matches=1 + fi +fi + +# --- --check: report and exit ------------------------------------------ + +if [ "$CHECK_ONLY" = "1" ]; then + if [ "$profile_matches" = "1" ] && { [ ! -d /run/systemd/system ] || [ "$dropin_matches" = "1" ]; }; then + # Persistent state already correct. (Kernel state can't be cheaply + # verified from here, but a present-matching drop-in guarantees the next + # docker start will load our profile — that's the meaningful invariant.) + exit 0 + fi + exit 1 +fi + +# --- Install the profile file ------------------------------------------ + +if [ "$profile_matches" = "1" ]; then + echo " ✅ docker-default profile at $DST_PROFILE already matches bundled copy." +else + echo " 📝 Installing docker-default profile to $DST_PROFILE..." + $SUDO cp "$SRC_PROFILE" "$DST_PROFILE" +fi + +# --- Load it into the kernel now --------------------------------------- + +# apparmor_parser -r is idempotent for the same content; replaces the +# in-kernel docker-default with ours. Already-running containers keep their +# previously-loaded profile; any new container picks up the replacement. +echo " 🔄 Loading docker-default profile into the kernel..." +$SUDO apparmor_parser -r "$DST_PROFILE" + +# --- Install systemd drop-in for persistence --------------------------- + +if [ ! -d /run/systemd/system ]; then + echo " ℹ️ Not running under systemd — skipping persistent drop-in install." + echo " ⚠️ Re-run this script after any docker daemon restart or host reboot." + exit 0 +fi + +if [ "$dropin_matches" = "1" ]; then + echo " ✅ systemd drop-in at $DROPIN_FILE already up to date." +else + echo " 📝 Writing systemd drop-in to $DROPIN_FILE..." + $SUDO mkdir -p "$DROPIN_DIR" + printf '%s\n' "$DESIRED_DROPIN_CONTENT" | $SUDO tee "$DROPIN_FILE" >/dev/null + echo " 🔄 Reloading systemd..." + $SUDO systemctl daemon-reload +fi + +echo +echo " ✅ docker-default AppArmor profile is installed and persistent." +echo +echo " Verify after the next sandbox container spawns:" +echo " sudo dmesg -wT | grep -iE 'apparmor|audit'" +echo " You should see NO 'apparmor=\"DENIED\"' entries for" +echo " operation=mount / pivotroot / signal." From 252c54267c9e2a6361c7f357f381d2b81a3139a2 Mon Sep 17 00:00:00 2001 From: David Huie Date: Tue, 9 Jun 2026 11:38:24 -0700 Subject: [PATCH 02/11] Use default blob storage env vars for Docker Compose --- README.md | 10 ++++++---- compose.yaml | 2 ++ install.sh | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e920ba0..486040c 100644 --- a/README.md +++ b/README.md @@ -49,16 +49,18 @@ Configure 1. Check the generated `.env` files to make sure the license key and randomized keys were set as expected during the installation. -2. Save off the `ENCRYPTION_KEY` value, since this is needed to encrypt/decrypt values saved into the Postgres database the Retool instance runs on. +2. Review the blob storage settings in `docker.env`. The generated defaults use bundled MinIO through `RR_BLOB_STORAGE_PROVIDER=s3` and `RR_DEFAULT_S3_*` variables. For production, replace these with your external object store credentials. For AWS S3, remove `RR_DEFAULT_S3_ENDPOINT` and `AWS_ENDPOINT_URL`; for S3-compatible providers such as R2 or MinIO, set both endpoint variables to the provider endpoint. The older `RR_GIT_*`, `RR_SNAPSHOTS_*`, and `RETOOL_ORG_PACKAGES_*` variables are only needed for advanced per-use-case overrides. -3. Replace `X.Y.Z-stable` in `Dockerfile` with the desired Retool version listed in our [Dockerhub repo](https://hub.docker.com/r/tryretool/backend/tags), we recommend the latest patch of the most recent [stable version](https://hub.docker.com/r/tryretool/backend/tags?name=stable). +3. Save off the `ENCRYPTION_KEY` value, since this is needed to encrypt/decrypt values saved into the Postgres database the Retool instance runs on. -4. To set up HTTPS, you'll need your domain pointing to your server's IP address. If that's in place, make sure `DOMAINS` is correct in `docker.env`, and then set `STAGE=production` in `compose.yaml` for the `https-portal` container to attempt to get and use a free `Let's Encrypt` cert for your domain on startup. +4. Replace `X.Y.Z-stable` in `Dockerfile` with the desired Retool version listed in our [Dockerhub repo](https://hub.docker.com/r/tryretool/backend/tags), we recommend the latest patch of the most recent [stable version](https://hub.docker.com/r/tryretool/backend/tags?name=stable). + +5. To set up HTTPS, you'll need your domain pointing to your server's IP address. If that's in place, make sure `DOMAINS` is correct in `docker.env`, and then set `STAGE=production` in `compose.yaml` for the `https-portal` container to attempt to get and use a free `Let's Encrypt` cert for your domain on startup. > [!WARNING] > You must set `COOKIE_INSECURE=true` in `docker.env` to allow logging into Retool without HTTPS configured (not recommended) -5. By default, the deployment will include a Temporal container for Workflows. If you have an Enterprise license and would like to instead use Retool's managed Temporal cluster, comment out the `include` block in `compose.yaml` and the `WORKFLOW_TEMPORAL_...` environment variables in `docker.env`. Check out [our docs](https://docs.retool.com/self-hosted/concepts/temporal) for more information on Temporal deployment options. +6. By default, the deployment will include a Temporal container for Workflows. If you have an Enterprise license and would like to instead use Retool's managed Temporal cluster, comment out the `include` block in `compose.yaml` and the `WORKFLOW_TEMPORAL_...` environment variables in `docker.env`. Check out [our docs](https://docs.retool.com/self-hosted/concepts/temporal) for more information on Temporal deployment options.
diff --git a/compose.yaml b/compose.yaml index 84c26ee..597bd01 100644 --- a/compose.yaml +++ b/compose.yaml @@ -223,6 +223,7 @@ services: networks: backend: aliases: + - retool-blob-storage.minio - retool-rr-git.minio - retool-rr-snapshots.minio volumes: @@ -237,6 +238,7 @@ services: /bin/sh -c " sleep 3; mc alias set local http://minio:9000 retool retoolminio; + mc mb --ignore-existing local/retool-blob-storage; mc mb --ignore-existing local/retool-rr-git; mc mb --ignore-existing local/retool-rr-snapshots; echo 'Buckets created successfully'; diff --git a/install.sh b/install.sh index 2c67370..581150b 100755 --- a/install.sh +++ b/install.sh @@ -156,17 +156,17 @@ STATE_BACKEND=postgres AGENT_EXECUTOR_POSTGRES_URL=postgres://retool_internal_user:$postgres_password@postgres:5432/hammerhead_production AGENT_EXECUTOR_POSTGRES_SCHEMA=agent_executor -# S3-compatible storage (MinIO) +# Blob storage (bundled MinIO defaults) +# For production, replace these with your external S3-compatible object store. +# Leave RR_DEFAULT_S3_ENDPOINT and AWS_ENDPOINT_URL unset for AWS S3. +# Set both to the same endpoint for R2, MinIO, or other custom endpoints. +RR_BLOB_STORAGE_PROVIDER=s3 +RR_DEFAULT_S3_BUCKET=retool-blob-storage +RR_DEFAULT_S3_ACCESS_KEY_ID=retool +RR_DEFAULT_S3_SECRET_ACCESS_KEY=retoolminio +RR_DEFAULT_S3_REGION=us-east-1 +RR_DEFAULT_S3_ENDPOINT=http://minio:9000 AWS_ENDPOINT_URL=http://minio:9000 -RR_GIT_S3_BUCKET=retool-rr-git -RR_GIT_S3_ACCESS_KEY_ID=retool -RR_GIT_S3_SECRET_ACCESS_KEY=retoolminio -RR_GIT_S3_REGION=us-east-1 -RR_SNAPSHOTS_S3_BUCKET=retool-rr-snapshots -RR_SNAPSHOTS_S3_ACCESS_KEY_ID=retool -RR_SNAPSHOTS_S3_SECRET_ACCESS_KEY=retoolminio -RR_SNAPSHOTS_S3_REGION=us-east-1 -RR_SNAPSHOTS_S3_ENDPOINT=http://minio:9000 # Comment out below to use Retool-managed Temporal (Enterprise license) WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal From 7c9e17e60eda8f84766e808cf28b30f1e783a2e8 Mon Sep 17 00:00:00 2001 From: David Huie Date: Tue, 9 Jun 2026 11:44:14 -0700 Subject: [PATCH 03/11] Document blob storage version requirements --- README.md | 2 +- install.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 486040c..2c4521e 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Configure 1. Check the generated `.env` files to make sure the license key and randomized keys were set as expected during the installation. -2. Review the blob storage settings in `docker.env`. The generated defaults use bundled MinIO through `RR_BLOB_STORAGE_PROVIDER=s3` and `RR_DEFAULT_S3_*` variables. For production, replace these with your external object store credentials. For AWS S3, remove `RR_DEFAULT_S3_ENDPOINT` and `AWS_ENDPOINT_URL`; for S3-compatible providers such as R2 or MinIO, set both endpoint variables to the provider endpoint. The older `RR_GIT_*`, `RR_SNAPSHOTS_*`, and `RETOOL_ORG_PACKAGES_*` variables are only needed for advanced per-use-case overrides. +2. Review the blob storage settings in `docker.env`. The generated defaults use bundled MinIO through `RR_BLOB_STORAGE_PROVIDER=s3` and `RR_DEFAULT_S3_*` variables, which require Retool 3.391.0 or later. For production, replace these with your external object store credentials. For AWS S3, remove `RR_DEFAULT_S3_ENDPOINT` and `AWS_ENDPOINT_URL`; for S3-compatible providers such as R2 or MinIO, set both endpoint variables to the provider endpoint. The older `RR_GIT_*`, `RR_SNAPSHOTS_*`, and `RETOOL_ORG_PACKAGES_*` variables are only needed for older Retool versions or advanced per-use-case overrides. 3. Save off the `ENCRYPTION_KEY` value, since this is needed to encrypt/decrypt values saved into the Postgres database the Retool instance runs on. diff --git a/install.sh b/install.sh index 581150b..9a07251 100755 --- a/install.sh +++ b/install.sh @@ -157,6 +157,7 @@ AGENT_EXECUTOR_POSTGRES_URL=postgres://retool_internal_user:$postgres_password@p AGENT_EXECUTOR_POSTGRES_SCHEMA=agent_executor # Blob storage (bundled MinIO defaults) +# Requires Retool 3.391.0+; older images need the legacy per-use-case blob storage vars. # For production, replace these with your external S3-compatible object store. # Leave RR_DEFAULT_S3_ENDPOINT and AWS_ENDPOINT_URL unset for AWS S3. # Set both to the same endpoint for R2, MinIO, or other custom endpoints. From f2ad3d1d94cd5700f99905c4cf22b3f934c93786 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Thu, 11 Jun 2026 15:22:22 -0700 Subject: [PATCH 04/11] r2 fixes --- compose.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index 597bd01..7a09fcf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,7 +8,7 @@ services: context: . env_file: docker.env environment: - - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,RR_GIT_SERVER ports: - 3000:3000 networks: @@ -116,7 +116,7 @@ services: # js-executor reads its memory budget from the cgroup limit, so this # must be set explicitly. Matches the helm chart default (2048Mi). - mem_limit: 4g + mem_limit: 2g # Custom seccomp profile is required for nsjail's syscalls to pass Docker's # default seccomp filter. Applies under both options below. @@ -151,7 +151,6 @@ services: - NODE_ENV=production - AGENT_EXECUTOR_ROLE=controller - ORCHESTRATOR=docker - - EXECUTOR_IMAGE=${EXECUTOR_IMAGE:-retool-onpremise-agent-sandbox-controller} - DOCKER_NETWORK=agent-sandbox - EXECUTOR_SERVICE_NAME=agent-sandbox - SANDBOX_BACKEND_URL=http://api:3000 @@ -226,6 +225,9 @@ services: - retool-blob-storage.minio - retool-rr-git.minio - retool-rr-snapshots.minio + agent-sandbox: + aliases: + - retool-rr-snapshots.minio volumes: - minio-data:/data restart: always From 8cebdf2b664bf3a452d21aa55c2063ffcc8d5ad0 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Mon, 15 Jun 2026 13:38:45 -0700 Subject: [PATCH 05/11] comments --- compose.yaml | 4 ++-- install.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index 7a09fcf..60f426c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -114,8 +114,8 @@ services: context: . target: js-executor - # js-executor reads its memory budget from the cgroup limit, so this - # must be set explicitly. Matches the helm chart default (2048Mi). + # js-executor reads its memory budget from the cgroup limit, so this must + # be set explicitly. mem_limit: 2g # Custom seccomp profile is required for nsjail's syscalls to pass Docker's diff --git a/install.sh b/install.sh index 9a07251..581150b 100755 --- a/install.sh +++ b/install.sh @@ -157,7 +157,6 @@ AGENT_EXECUTOR_POSTGRES_URL=postgres://retool_internal_user:$postgres_password@p AGENT_EXECUTOR_POSTGRES_SCHEMA=agent_executor # Blob storage (bundled MinIO defaults) -# Requires Retool 3.391.0+; older images need the legacy per-use-case blob storage vars. # For production, replace these with your external S3-compatible object store. # Leave RR_DEFAULT_S3_ENDPOINT and AWS_ENDPOINT_URL unset for AWS S3. # Set both to the same endpoint for R2, MinIO, or other custom endpoints. From d0154e8e481d0610f7d5738ecad88e826846fb02 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Tue, 16 Jun 2026 21:34:39 -0700 Subject: [PATCH 06/11] changes --- compose.yaml | 41 +++++++++++++++++----------------- install.sh | 62 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/compose.yaml b/compose.yaml index 60f426c..e4adef7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -184,8 +184,6 @@ services: - BACKEND_URL=http://api:3000 # Uncomment to restrict which external domains sandboxes can reach # - ALLOWED_DOMAINS=api.openai.com,api.anthropic.com - ports: - - 3019:3019 networks: - backend - agent-sandbox @@ -268,24 +266,27 @@ services: - retooldb-data:/var/lib/postgresql/data restart: always - # Optional Nginx container for handling TLS for your domain (requires setting DOMAINS and STAGE) - https-portal: - image: tryretool/https-portal:latest - env_file: docker.env - environment: - # Change 'local' -> 'production' below once your domain is pointing to this server - STAGE: local - CLIENT_MAX_BODY_SIZE: 40M - KEEPALIVE_TIMEOUT: 605 - PROXY_CONNECT_TIMEOUT: 600 - PROXY_SEND_TIMEOUT: 600 - PROXY_READ_TIMEOUT: 600 - ports: - - 80:80 - - 443:443 - networks: - - frontend - restart: always + # Optional Nginx container for handling TLS for your domain (requires setting DOMAINS and STAGE). + # Disabled for local HTTP access: Retool is reached directly at http://localhost:3000 (api). + # Re-enable this (and set DOMAINS + BASE_DOMAIN to https://your-domain, STAGE: production) + # once you have a real domain with a trusted certificate. + # https-portal: + # image: tryretool/https-portal:latest + # env_file: docker.env + # environment: + # # Change 'local' -> 'production' below once your domain is pointing to this server + # STAGE: local + # CLIENT_MAX_BODY_SIZE: 40M + # KEEPALIVE_TIMEOUT: 605 + # PROXY_CONNECT_TIMEOUT: 600 + # PROXY_SEND_TIMEOUT: 600 + # PROXY_READ_TIMEOUT: 600 + # ports: + # - 80:80 + # - 443:443 + # networks: + # - frontend + # restart: always networks: frontend: diff --git a/install.sh b/install.sh index 581150b..30d4cd2 100755 --- a/install.sh +++ b/install.sh @@ -105,8 +105,9 @@ echo "Prompting for optional configuration..." read -p " Retool license key: " licenseKey licenseKey=${licenseKey:-EXPIRED-LICENSE-KEY-TRIAL} -read -p " Domain (e.g. retool.company.com) pointing to this server: " hostname -hostname=${hostname:-$(dig +short myip.opendns.com @resolver1.opendns.com)} +echo " Domain pointing to this server (e.g. retool.company.com) for HTTPS access." +echo " Leave blank for local HTTP access at http://localhost:3000 (recommended for local testing)." +read -p " Domain: " hostname echo "" # Create docker.env with values @@ -120,6 +121,43 @@ ae_public_pem=$(echo "$ae_private_pem" | openssl ec -pubout 2>/dev/null) ae_private_key=$(echo "$ae_private_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') ae_public_key=$(echo "$ae_public_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') +# Build the serving-mode config (domain/TLS vs. local HTTP). The agent-sandbox +# preview connects same-origin as the editor, so BASE_DOMAIN must match the scheme +# and host the browser actually uses. +if [[ -n "$hostname" ]]; then + serving_mode="https://$hostname (enable the https-portal service in compose.yaml)" + network_config=$(cat < http://api:3000 + +# Used to create links like user invitations and password resets +# Retool tries to guess this, but it can be incorrect if using a proxy in front of the instance +BASE_DOMAIN=https://$hostname + +# If your domain/HTTPS isn't in place yet +# COOKIE_INSECURE=true +CFG +) +else + serving_mode="http://localhost:3000 (plain HTTP, https-portal disabled)" + network_config=$(cat < http://api:3000", +# uncomment the https-portal service in compose.yaml, and set BASE_DOMAIN to +# https://your-domain. +# DOMAINS=your-domain.com -> http://api:3000 + +# Used to create links like user invitations and password resets +# Local HTTP access: reach Retool directly at the api service's published port. +BASE_DOMAIN=http://localhost:3000 + +# Required when serving over plain HTTP (no TLS) +COOKIE_INSECURE=true +CFG +) +fi + cat << EOF > docker.env # Environment variables reference: docs.retool.com/docs/environment-variables DEPLOYMENT_TEMPLATE_TYPE=docker-compose @@ -148,7 +186,6 @@ AGENT_EXECUTOR_ENABLED=true RR_AGENT_PUBSUB_BACKEND=postgres AGENT_EXECUTOR_CONTROLLER_INGRESS_DOMAIN=http://agent-sandbox-controller:3018 AGENT_EXECUTOR_PROXY_INGRESS_DOMAIN=http://agent-sandbox-proxy:3019 -AGENT_EXECUTOR_FRONTEND_WS_PROXY_DOMAIN=http://$hostname:3019 AGENT_EXECUTOR_JWT_PRIVATE_KEY="$ae_private_key" AGENT_EXECUTOR_JWT_PUBLIC_KEY="$ae_public_key" AGENT_EXECUTOR_ENCRYPTION_KEY=$(openssl rand -hex 32) @@ -181,15 +218,7 @@ JWT_SECRET=$(random 256) # License you received from my.retool.com or your Retool contact LICENSE_KEY=$licenseKey -# Make sure $hostname is your domain to set up HTTPS (e.g. retool.company.com) -DOMAINS=$hostname -> http://api:3000 - -# Used to create links like user invitations and password resets -# Retool tries to guess this, but it can be incorrect if using a proxy in front of the instance -BASE_DOMAIN=https://$hostname - -# If your domain/HTTPS isn't in place yet -# COOKIE_INSECURE=true +$network_config EOF @@ -202,7 +231,14 @@ echo "✅ Created docker.env" # Next steps echo "" -echo "Done! Check docker.env and retooldb.env files for expected values, and confirm" +echo "Done! Retool will be served at: $serving_mode" +echo "" +echo "Check docker.env and retooldb.env files for expected values, and confirm" echo "the Retool version in Dockerfile. We suggest the most recent X.Y.Z-stable version," echo "see Dockerhub for available tags: https://hub.docker.com/r/tryretool/backend/tags" +if [[ -n "$hostname" ]]; then + echo "" + echo "HTTPS mode: uncomment the https-portal service in compose.yaml before starting," + echo "and ensure $hostname resolves to this server with a trusted certificate." +fi echo "" From 1e0991c0869908a0856ad605146255181153ec97 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Tue, 16 Jun 2026 22:52:51 -0700 Subject: [PATCH 07/11] fix --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index e4adef7..0e4b114 100644 --- a/compose.yaml +++ b/compose.yaml @@ -276,6 +276,7 @@ services: # environment: # # Change 'local' -> 'production' below once your domain is pointing to this server # STAGE: local + # WEBSOCKET: 'true' # CLIENT_MAX_BODY_SIZE: 40M # KEEPALIVE_TIMEOUT: 605 # PROXY_CONNECT_TIMEOUT: 600 From 0246ef30c2ab47ec1842a98cf0ccc2f4524101c5 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Wed, 17 Jun 2026 10:23:29 -0700 Subject: [PATCH 08/11] make more similar to master --- compose.yaml | 44 ++++++++++++++++++------------------- install.sh | 61 +++++++++++----------------------------------------- 2 files changed, 34 insertions(+), 71 deletions(-) diff --git a/compose.yaml b/compose.yaml index 0e4b114..59957ca 100644 --- a/compose.yaml +++ b/compose.yaml @@ -266,28 +266,28 @@ services: - retooldb-data:/var/lib/postgresql/data restart: always - # Optional Nginx container for handling TLS for your domain (requires setting DOMAINS and STAGE). - # Disabled for local HTTP access: Retool is reached directly at http://localhost:3000 (api). - # Re-enable this (and set DOMAINS + BASE_DOMAIN to https://your-domain, STAGE: production) - # once you have a real domain with a trusted certificate. - # https-portal: - # image: tryretool/https-portal:latest - # env_file: docker.env - # environment: - # # Change 'local' -> 'production' below once your domain is pointing to this server - # STAGE: local - # WEBSOCKET: 'true' - # CLIENT_MAX_BODY_SIZE: 40M - # KEEPALIVE_TIMEOUT: 605 - # PROXY_CONNECT_TIMEOUT: 600 - # PROXY_SEND_TIMEOUT: 600 - # PROXY_READ_TIMEOUT: 600 - # ports: - # - 80:80 - # - 443:443 - # networks: - # - frontend - # restart: always + # Optional Nginx container for handling TLS for your domain (requires setting DOMAINS and STAGE) + https-portal: + image: tryretool/https-portal:latest + env_file: docker.env + environment: + # Change 'local' -> 'production' below once your domain is pointing to this server + STAGE: local + # Forwards WebSocket upgrades (proxy_http_version 1.1 + Upgrade/Connection headers). + # Required for the agent-sandbox preview and multiplayer WebSockets — without it + # those connections fail with close code 1006 even though normal HTTP works. + WEBSOCKET: 'true' + CLIENT_MAX_BODY_SIZE: 40M + KEEPALIVE_TIMEOUT: 605 + PROXY_CONNECT_TIMEOUT: 600 + PROXY_SEND_TIMEOUT: 600 + PROXY_READ_TIMEOUT: 600 + ports: + - 80:80 + - 443:443 + networks: + - frontend + restart: always networks: frontend: diff --git a/install.sh b/install.sh index 30d4cd2..4cb0c2b 100755 --- a/install.sh +++ b/install.sh @@ -105,9 +105,8 @@ echo "Prompting for optional configuration..." read -p " Retool license key: " licenseKey licenseKey=${licenseKey:-EXPIRED-LICENSE-KEY-TRIAL} -echo " Domain pointing to this server (e.g. retool.company.com) for HTTPS access." -echo " Leave blank for local HTTP access at http://localhost:3000 (recommended for local testing)." -read -p " Domain: " hostname +read -p " Domain (e.g. retool.company.com) pointing to this server: " hostname +hostname=${hostname:-$(dig +short myip.opendns.com @resolver1.opendns.com)} echo "" # Create docker.env with values @@ -121,43 +120,6 @@ ae_public_pem=$(echo "$ae_private_pem" | openssl ec -pubout 2>/dev/null) ae_private_key=$(echo "$ae_private_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') ae_public_key=$(echo "$ae_public_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') -# Build the serving-mode config (domain/TLS vs. local HTTP). The agent-sandbox -# preview connects same-origin as the editor, so BASE_DOMAIN must match the scheme -# and host the browser actually uses. -if [[ -n "$hostname" ]]; then - serving_mode="https://$hostname (enable the https-portal service in compose.yaml)" - network_config=$(cat < http://api:3000 - -# Used to create links like user invitations and password resets -# Retool tries to guess this, but it can be incorrect if using a proxy in front of the instance -BASE_DOMAIN=https://$hostname - -# If your domain/HTTPS isn't in place yet -# COOKIE_INSECURE=true -CFG -) -else - serving_mode="http://localhost:3000 (plain HTTP, https-portal disabled)" - network_config=$(cat < http://api:3000", -# uncomment the https-portal service in compose.yaml, and set BASE_DOMAIN to -# https://your-domain. -# DOMAINS=your-domain.com -> http://api:3000 - -# Used to create links like user invitations and password resets -# Local HTTP access: reach Retool directly at the api service's published port. -BASE_DOMAIN=http://localhost:3000 - -# Required when serving over plain HTTP (no TLS) -COOKIE_INSECURE=true -CFG -) -fi - cat << EOF > docker.env # Environment variables reference: docs.retool.com/docs/environment-variables DEPLOYMENT_TEMPLATE_TYPE=docker-compose @@ -218,7 +180,15 @@ JWT_SECRET=$(random 256) # License you received from my.retool.com or your Retool contact LICENSE_KEY=$licenseKey -$network_config +# Make sure $hostname is your domain to set up HTTPS (e.g. retool.company.com) +DOMAINS=$hostname -> http://api:3000 + +# Used to create links like user invitations and password resets +# Retool tries to guess this, but it can be incorrect if using a proxy in front of the instance +BASE_DOMAIN=https://$hostname + +# If your domain/HTTPS isn't in place yet +# COOKIE_INSECURE=true EOF @@ -231,14 +201,7 @@ echo "✅ Created docker.env" # Next steps echo "" -echo "Done! Retool will be served at: $serving_mode" -echo "" -echo "Check docker.env and retooldb.env files for expected values, and confirm" +echo "Done! Check docker.env and retooldb.env files for expected values, and confirm" echo "the Retool version in Dockerfile. We suggest the most recent X.Y.Z-stable version," echo "see Dockerhub for available tags: https://hub.docker.com/r/tryretool/backend/tags" -if [[ -n "$hostname" ]]; then - echo "" - echo "HTTPS mode: uncomment the https-portal service in compose.yaml before starting," - echo "and ensure $hostname resolves to this server with a trusted certificate." -fi echo "" From 00b57214f20f974a74338eb637345e74a1309d54 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Mon, 22 Jun 2026 09:56:40 -0700 Subject: [PATCH 09/11] fixes --- compose.yaml | 14 ++++++++------ install.sh | 12 ++++++++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/compose.yaml b/compose.yaml index 59957ca..c99e76d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -210,13 +210,12 @@ services: minio: image: minio/minio:latest command: server /data --console-address ":9001" + env_file: docker.env environment: - - MINIO_ROOT_USER=retool - - MINIO_ROOT_PASSWORD=retoolminio - MINIO_DOMAIN=minio ports: - - 9000:9000 - - 9001:9001 + - 127.0.0.1:9000:9000 + - 127.0.0.1:9001:9001 networks: backend: aliases: @@ -232,12 +231,15 @@ services: minio-init: image: minio/mc:latest + env_file: docker.env depends_on: - minio entrypoint: > /bin/sh -c " - sleep 3; - mc alias set local http://minio:9000 retool retoolminio; + until mc alias set local http://minio:9000 \"$$MINIO_ROOT_USER\" \"$$MINIO_ROOT_PASSWORD\"; do + echo 'Waiting for MinIO to be ready...'; + sleep 2; + done; mc mb --ignore-existing local/retool-blob-storage; mc mb --ignore-existing local/retool-rr-git; mc mb --ignore-existing local/retool-rr-snapshots; diff --git a/install.sh b/install.sh index 4cb0c2b..948975e 100755 --- a/install.sh +++ b/install.sh @@ -115,6 +115,9 @@ random() { cat /dev/urandom | base64 | head -c "$1" | tr -d +/ ; } postgres_password=$(random 64) +minio_root_user=retool +minio_root_password=$(random 32) + ae_private_pem=$(openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 2>/dev/null) ae_public_pem=$(echo "$ae_private_pem" | openssl ec -pubout 2>/dev/null) ae_private_key=$(echo "$ae_private_pem" | awk '{if(NR>1)printf "\\n";printf "%s",$0}') @@ -161,12 +164,17 @@ AGENT_EXECUTOR_POSTGRES_SCHEMA=agent_executor # Set both to the same endpoint for R2, MinIO, or other custom endpoints. RR_BLOB_STORAGE_PROVIDER=s3 RR_DEFAULT_S3_BUCKET=retool-blob-storage -RR_DEFAULT_S3_ACCESS_KEY_ID=retool -RR_DEFAULT_S3_SECRET_ACCESS_KEY=retoolminio +RR_DEFAULT_S3_ACCESS_KEY_ID=$minio_root_user +RR_DEFAULT_S3_SECRET_ACCESS_KEY=$minio_root_password RR_DEFAULT_S3_REGION=us-east-1 RR_DEFAULT_S3_ENDPOINT=http://minio:9000 AWS_ENDPOINT_URL=http://minio:9000 +# Bundled MinIO root credentials. The minio and minio-init services read these +# from docker.env; they must match the RR_DEFAULT_S3_* access key/secret above. +MINIO_ROOT_USER=$minio_root_user +MINIO_ROOT_PASSWORD=$minio_root_password + # Comment out below to use Retool-managed Temporal (Enterprise license) WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT=7233 From ec5d028d6ee0f72a36486387d6cf9fab052c8d32 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Mon, 22 Jun 2026 10:02:19 -0700 Subject: [PATCH 10/11] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c4521e..c9ddff1 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Configure 1. Check the generated `.env` files to make sure the license key and randomized keys were set as expected during the installation. -2. Review the blob storage settings in `docker.env`. The generated defaults use bundled MinIO through `RR_BLOB_STORAGE_PROVIDER=s3` and `RR_DEFAULT_S3_*` variables, which require Retool 3.391.0 or later. For production, replace these with your external object store credentials. For AWS S3, remove `RR_DEFAULT_S3_ENDPOINT` and `AWS_ENDPOINT_URL`; for S3-compatible providers such as R2 or MinIO, set both endpoint variables to the provider endpoint. The older `RR_GIT_*`, `RR_SNAPSHOTS_*`, and `RETOOL_ORG_PACKAGES_*` variables are only needed for older Retool versions or advanced per-use-case overrides. +2. Review the blob storage settings in `docker.env`. The generated defaults use bundled MinIO through `RR_BLOB_STORAGE_PROVIDER=s3` and `RR_DEFAULT_S3_*` variables. For production, replace these with your external object store credentials. For AWS S3, remove `RR_DEFAULT_S3_ENDPOINT` and `AWS_ENDPOINT_URL`; for S3-compatible providers such as MinIO, set both endpoint variables to the provider endpoint. 3. Save off the `ENCRYPTION_KEY` value, since this is needed to encrypt/decrypt values saved into the Postgres database the Retool instance runs on. From 9509237babab481e43336e47e12a8ae5bc261854 Mon Sep 17 00:00:00 2001 From: lukefoster11 Date: Tue, 23 Jun 2026 10:49:45 -0700 Subject: [PATCH 11/11] cleanup --- compose.yaml | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/compose.yaml b/compose.yaml index c99e76d..2584af7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,8 +17,10 @@ services: - code-executor - js-executor depends_on: - - postgres - - minio + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always jobs-runner: @@ -44,7 +46,10 @@ services: - code-executor - js-executor depends_on: - - postgres + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always workflows-worker: @@ -59,7 +64,10 @@ services: - code-executor - js-executor depends_on: - - postgres + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always agent-worker: @@ -74,7 +82,10 @@ services: - code-executor - js-executor depends_on: - - postgres + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always agent-eval-worker: @@ -89,7 +100,10 @@ services: - code-executor - js-executor depends_on: - - postgres + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always code-executor: @@ -156,7 +170,6 @@ services: - SANDBOX_BACKEND_URL=http://api:3000 - EXECUTOR_EXTRA_ENV=SANDBOX_NETWORK_ENABLED=false,SANDBOX_HTTP_PROXY=http://agent-sandbox-proxy:3019 - EXECUTOR_SECCOMP_PROFILE_PATH=/seccomp/gvisor-seccomp.json - # Lower pool sizes for local development - PREWARM_POOL_SIZE=1 - MAX_TOTAL_JOBS=10 - MAX_CONCURRENT_CREATES=2 @@ -204,7 +217,10 @@ services: - js-executor - agent-sandbox depends_on: - - postgres + postgres: + condition: service_started + minio-init: + condition: service_completed_successfully restart: always minio: @@ -220,11 +236,9 @@ services: backend: aliases: - retool-blob-storage.minio - - retool-rr-git.minio - - retool-rr-snapshots.minio agent-sandbox: aliases: - - retool-rr-snapshots.minio + - retool-blob-storage.minio volumes: - minio-data:/data restart: always @@ -241,9 +255,7 @@ services: sleep 2; done; mc mb --ignore-existing local/retool-blob-storage; - mc mb --ignore-existing local/retool-rr-git; - mc mb --ignore-existing local/retool-rr-snapshots; - echo 'Buckets created successfully'; + echo 'Bucket created successfully'; " networks: - backend