Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions .github/workflows/seed-reachability.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: Seed Reachability
run-name: Seed Reachability / published seeds serve P2P

# Dials the published Sei Labs seed endpoints and asserts each one actually
# speaks P2P, rather than merely accepting the TCP connection.
#
# Deliberately manual. This is a verification tool, not a monitor:
#
# - run it after the seeds are rolled onto a new build, to confirm they came
# back
# - run it before cutting a release that ships the seed defaults
# - run it when someone reports trouble bootstrapping
#
# Not a PR gate, because it talks to live external infrastructure and a seed
# hiccup or CI egress trouble must not block unrelated work. Not scheduled
# either: continuous detection of a seed going dark belongs in the monitoring
# stack, which cannot see seeds today because seed mode starts no Prometheus
# listener. A cron here would paper over that rather than fix it.
on:
workflow_dispatch:

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
reachability:
name: Reachability
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true

# Compile-checked as well as run: build-tagged files are skipped by the
# untagged vet and by golangci-lint, so without this the package can rot
# against an API change and stay green.
- name: Vet
run: go vet -tags=integration ./app/seeds/...

- name: Seed reachability
run: go test -tags=integration -v -count=1 -timeout=5m ./app/seeds/...

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for adding this monty, we can tidy this up later in terms of requirements for PR landing or alerting if seed nodes are down.

51 changes: 51 additions & 0 deletions app/seeds/seeds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Package seeds ships the Sei Labs operated P2P seed nodes for the public Sei
// networks, so a freshly initialised node bootstraps peer discovery with no
// operator configuration.
//
// Seeds are dialled to populate the address book via the PEX reactor and may
// then be dropped, which is why they belong in `bootstrap-peers` rather than
// `persistent-peers` — an operator should not hold connections open against
// them indefinitely.
package seeds

import "strings"

// chainSeeds maps a well-known chain-id to its Sei Labs seed nodes, each in
// CometBFT's `NodeID@host:port` form. Three per network, one per cell, so
// losing a region does not cost bootstrap capability. The cell is encoded in
// the hostname: unsuffixed `prod` is eu-central-1, `prod-euw1` is eu-west-1,
// and `prod-use2` is us-east-2.
//
// PERMANENCE: these strings ship inside released binaries and operators pin
// them. The secret-connection handshake verifies the NodeID, so a changed ID
// is a rejected dial, not a degraded one — and a release already in the wild
// cannot be recalled. Retiring an address therefore means keeping it dialable
// until every release carrying it is out of use. Treat edits here as one-way.
//
// arctic-1 is deliberately absent. It is a devnet: it has no Cosmos
// chain-registry entry, so it is not an operator-facing network, and a devnet
// is the most likely to be reset or re-keyed — exactly the case where baking a
// permanent address into a binary is wrong. Devnet users set bootstrap-peers
// explicitly.
//
// Source of truth: clusters/<cell>/<chain>/seeds/seed-N/seed-N.yaml in
// sei-protocol/platform (the SeiNode's externalAddress plus its NodeID).
var chainSeeds = map[string][]string{
"pacific-1": {
"0cd5f57c249b5aca815710338e1fe7a14797585d@seed-0-p2p.pacific-1.prod.platform.sei.io:26656",

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] Naming asymmetry worth a second look: seed-1/seed-2 carry an explicit cell suffix (prod-euw1, prod-use2) while seed-0 is bare prod, so the doc comment's "one per cell (eu-central-1, eu-west-1, us-east-2)" is only readable as three cells if you already know prod == eu-central-1. Not something the tests can catch (host uniqueness passes either way), and these strings are one-way once released — worth confirming against clusters/<cell>/<chain>/seeds/ that the unsuffixed name really is the eu-central-1 cell for both chains.

"f0f057f1593d28bec11591cf146bd223e0be1866@seed-1-p2p.pacific-1.prod-euw1.platform.sei.io:26656",
"8e28f62368a1ceae0102645db8584b218650930d@seed-2-p2p.pacific-1.prod-use2.platform.sei.io:26656",
},
"atlantic-2": {
"362f934ead3654fca9cafdac63b52b47b2f9a95e@seed-0-p2p.atlantic-2.prod.platform.sei.io:26656",
"1f55cd51183d3a6cad8a3667b91d08d0338bd52e@seed-1-p2p.atlantic-2.prod-euw1.platform.sei.io:26656",
"7152be2e4c1a057d2b2467723058c5f0ec790472@seed-2-p2p.atlantic-2.prod-use2.platform.sei.io:26656",
},
}
Comment thread
cursor[bot] marked this conversation as resolved.

// BootstrapPeers returns the Sei Labs seeds for a chain as the comma-separated
// value CometBFT's `bootstrap-peers` expects, or "" when the chain-id is not
// recognised (private and local chains included).
func BootstrapPeers(chainID string) string {
return strings.Join(chainSeeds[chainID], ",")
}
92 changes: 92 additions & 0 deletions app/seeds/seeds_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
//go:build integration

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] (Also raised by Codex.) Nothing invokes go test -tags=integration ./app/seeds/... — not .github/workflows/, not the Makefile, not integration-test-matrix.json (which uses the unrelated yaml_integration tag).

Worse than just not running: .golangci.yml sets build-tags: [codeanalysis] and tests: false, and go build / go vet skip tagged files by default, so this file is never even compile-checked in CI. It can break against a config API change and stay green indefinitely.

A scheduled workflow (these assert live endpoint health, so cron fits better than per-PR) plus a make target would give the reachability check a way to actually fire. At minimum, add it to a compile-only step (go vet -tags=integration ./app/seeds/...) so it can't rot silently.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question, and I went and checked: no, integration is a new tag. The repo currently uses norace, linux, inprocess, rocksdb, dummy, cgo, darwin, mock_balances, ledger, littdb_wip, windows, and the existing integration suite runs under a different one, yaml_integration, driven by .github/workflows/integration-test-matrix.json.

I looked at reusing that tag and decided against it: that suite is docker based (docker exec sei-node-0 ...) against a locally spun chain, whereas this one dials live public endpoints. Folding them together means either the docker matrix starts making external calls, or this gets dragged into a container lifecycle it does not need.

So for now I have gone the honest route rather than the half wired one: the file header now says plainly that nothing runs or compile checks it and that it is an on demand tool rather than coverage, and I have raised a follow up to add a scheduled (cron plus workflow_dispatch) job.

Scheduled rather than a PR gate, since it depends on live external infrastructure and would otherwise block unrelated PRs whenever a seed hiccups or CI egress is flaky. Sequencing it after the two silent seeds are fixed as well, because wiring it today means the job is red on its first run, which is the quickest way to train everyone to ignore it. Happy to bring it into this PR instead if you would rather it did not land separately!

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@monty-sei the reply to this comment is identical to the one below. AI burp?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is integration an existing tag used elewhere? If yes and CI is already hooked up to running them then great.

If not, i recommend to:

  • pick an existing tag to save yourself from having to set up a CI for it, unless there is a really good reason that I might be missing for keeping it separate.
  • at the very least this PR should hook these tests to some CI job that runs them.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question, and I went and checked: no, integration is a new tag. The repo currently uses norace, linux, inprocess, rocksdb, dummy, cgo, darwin, mock_balances, ledger, littdb_wip, windows, and the existing integration suite runs under a different one, yaml_integration, driven by .github/workflows/integration-test-matrix.json.

I looked at reusing that tag and decided against it: that suite is docker based (docker exec sei-node-0 ...) against a locally spun chain, whereas this one dials live public endpoints. Folding them together means either the docker matrix starts making external calls, or this gets dragged into a container lifecycle it does not need.

So for now I have gone the honest route rather than the half wired one: the file header now says plainly that nothing runs or compile checks it and that it is an on demand tool rather than coverage, and I have raised a follow up to add a scheduled (cron plus workflow_dispatch) job.

Scheduled rather than a PR gate, since it depends on live external infrastructure and would otherwise block unrelated PRs whenever a seed hiccups or CI egress is flaky. Sequencing it after the two silent seeds are fixed as well, because wiring it today means the job is red on its first run, which is the quickest way to train everyone to ignore it. Happy to bring it into this PR instead if you would rather it did not land separately!

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @masih! I've hooked it up rather than leaving it unrun, so there's now a Seed Reachability workflow that runs the tagged tests and vets the package so it can't rot. It's manual only, partly because it dials live endpoints and I'm assuming we wouldn't want it gating unrelated PRs.

To get it green today it tolerates one unreachable seed per network, which I believe is actually the more honest assertion anyway, since three per network exist for redundancy and a node bootstraps fine on two (I saw one reach 21 peers with a seed down). Anything tolerated still gets named in the output so nothing quietly hides, and I've raised a follow up to tighten it to zero once the prod seeds are serving again.

Whenever you get a chance it'd be great to get your eyes on it again, and just let me know if you'd rather it went the other way!


// Reachability checks against the live published seed endpoints, build-tagged
// off by default because they make real network calls:
//
// go test -tags=integration ./app/seeds/...
//
// Run in CI by .github/workflows/seed-reachability.yml.
package seeds

import (
"fmt"
"net"
"strconv"
"testing"
"time"

"github.com/sei-protocol/sei-chain/sei-tendermint/config"
"github.com/stretchr/testify/require"
)

const dialTimeout = 10 * time.Second

// maxUnreachablePerNetwork is how many seeds in one network may fail to speak
// before this suite fails.
//
// One rather than zero, because that is the property the seeds actually owe:
// three per network exist so losing a region does not cost bootstrap
// capability, and a node bootstraps fine on the remaining two. Failing on any
// single unreachable seed would make this a liveness alarm for individual pods
// rather than a check that the published set still does its job.
//
// A tolerated failure is still named in the output, so a seed that stays dark
// is visible rather than silently absorbed.
//
// Tighten this to zero once every published seed is serving.
const maxUnreachablePerNetwork = 1

// A seed that is reachable at the TCP layer but never speaks is the failure
// mode this exists for: the listener accepts, the pod reports Ready, seed mode
// publishes no metrics, and inbound is silently closed. Only the bytes on the
// wire distinguish that from a healthy seed, so assert them.
//
// A conforming node sends its ephemeral-key preface immediately on connect
// without waiting for the dialer, so a seed that sends nothing is broken
// regardless of why.
func TestSeedsAreReachableAndSpeakP2P(t *testing.T) {
for chainID, addrs := range chainSeeds {
t.Run(chainID, func(t *testing.T) {
var unreachable []string

for _, entry := range addrs {
addr, err := config.ParseNodeAddress(entry)
require.NoErrorf(t, err, "%s: %q", chainID, entry)

if err := speaksP2P(addr); err != nil {
unreachable = append(unreachable, fmt.Sprintf("%s: %v", addr.Hostname, err))
t.Logf("UNREACHABLE %s", addr.Hostname)
continue
}
t.Logf("ok %s", addr.Hostname)
}

require.LessOrEqualf(t, len(unreachable), maxUnreachablePerNetwork,
"%s: %d of %d seeds are not serving P2P (tolerating up to %d):\n %v",
chainID, len(unreachable), len(addrs), maxUnreachablePerNetwork, unreachable)
})
}
}

// speaksP2P dials the seed and waits for it to send its handshake preface.
func speaksP2P(addr config.NodeAddress) error {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend convention of requireXYZ where the helper takes t and asserts things for you instead of returning error which then one asserts in the test.

Then the author has the duty to write small sharp require assertions that are assembleable across different tests.

hostPort := net.JoinHostPort(addr.Hostname, strconv.Itoa(int(addr.Port)))

conn, err := net.DialTimeout("tcp", hostPort, dialTimeout)
if err != nil {
return fmt.Errorf("dial: %w", err)
}
defer conn.Close()

if err := conn.SetReadDeadline(time.Now().Add(dialTimeout)); err != nil {
return fmt.Errorf("set deadline: %w", err)
}
n, err := conn.Read(make([]byte, 64))
if err != nil {
return fmt.Errorf("accepted the connection but sent nothing: %w", err)
}
if n == 0 {
return fmt.Errorf("sent an empty preface")
}
return nil
}
96 changes: 96 additions & 0 deletions app/seeds/seeds_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package seeds

import (
"strings"
"testing"

"github.com/sei-protocol/sei-chain/app/genesis"
"github.com/sei-protocol/sei-chain/sei-tendermint/config"
"github.com/stretchr/testify/require"
)

const (
pacific = "pacific-1"
atlantic = "atlantic-2"
arctic = "arctic-1"
)

// Addresses are parsed with the same parser the router uses when it dials, so
// this cannot drift from what p2p actually accepts. What it protects against is
// a typo in the table above: the NodeID is verified during the
// secret-connection handshake, so a wrong one is a rejected dial rather than a
// degraded connection, and seed mode serves no metrics to notice it by.
//
// Uniqueness is asserted across the whole table rather than per chain: the
// likeliest copy/paste error when adding a network is a pacific-1 entry landing
// in the atlantic-2 block, which a per-chain check cannot see.
func TestSeedAddressesParseAndAreUnique(t *testing.T) {
seenID := map[string]string{}
seenHost := map[string]string{}

for chainID, addrs := range chainSeeds {
require.NotEmptyf(t, addrs, "%s has no seeds", chainID)

for _, entry := range addrs {
addr, err := config.ParseNodeAddress(entry)
require.NoErrorf(t, err, "%s: %q", chainID, entry)
Comment thread
monty-sei marked this conversation as resolved.

// Round-trip rather than parse alone. ParseNodeAddress substitutes
// 26657 for a missing port, so a dropped ":26656" parses clean and
// would ship pointing at the RPC port; re-rendering catches that,
// and any other silent normalisation, without pinning a port number
// as though the protocol required one.
require.Equalf(t, entry, strings.TrimPrefix(addr.String(), "mconn://"),
"%s: %q does not survive a parse round-trip", chainID, entry)

// Seeds publish DNS names, not bare hosts, so the address outlives
// any IP change behind it.
require.Containsf(t, addr.Hostname, ".",
"%s: %q should publish a DNS name", chainID, entry)

id := string(addr.NodeID)
require.NotContainsf(t, seenID, id,
"NodeID %s appears in both %s and %s", id, seenID[id], chainID)
seenID[id] = chainID

require.NotContainsf(t, seenHost, addr.Hostname,
"host %s appears in both %s and %s", addr.Hostname, seenHost[addr.Hostname], chainID)
seenHost[addr.Hostname] = chainID
}
}
}

// Every chain we ship seeds for must also be a chain seid can initialise, or
// the entry is a typo that would silently never apply. The converse is not
// asserted: arctic-1 is intentionally well-known for genesis but has no seeds.
func TestSeedChainsAreWellKnown(t *testing.T) {
for chainID := range chainSeeds {
require.Truef(t, genesis.IsWellKnown(chainID),
"chain %q has seeds but is not a well-known chain (typo?)", chainID)
}
}

func TestArcticIsDeliberatelyExcluded(t *testing.T) {
require.Empty(t, BootstrapPeers(arctic), "arctic-1 is a devnet and must not ship seeds")
// Guard the premise of the exclusion: arctic-1 is still initialisable.
require.True(t, genesis.IsWellKnown(arctic))
}

func TestBootstrapPeers(t *testing.T) {
for _, chainID := range []string{pacific, atlantic} {
got := BootstrapPeers(chainID)
require.NotEmptyf(t, got, "%s should ship seeds", chainID)
// Round-trip the rendered value through the parser the way seid does,
// so the joined form is asserted rather than just the table entries.
for _, entry := range strings.Split(got, ",") {
_, err := config.ParseNodeAddress(entry)
require.NoErrorf(t, err, "%s: %q", chainID, entry)
}
}

// Exact match only — a chain-id we do not recognise must contribute nothing,
// so private and local chains are unaffected.
for _, unknown := range []string{"", "unknown-1", "Pacific-1", pacific + " "} {
require.Emptyf(t, BootstrapPeers(unknown), "chain %q should ship no seeds", unknown)
}
}
Loading
Loading