Skip to content

DAOS-19087 pool: Recov conts in all reint modes #18651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
liw wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

DAOS-19087 pool: Recov conts in all reint modes #18651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
561fd1f
DAOS-19087 pool: Recov conts in all reint modes
liw Jul 31, 2026
1077b20
Serialize discard and recov_cont
liw Jul 27, 2026
1a7c0b9
Adopt pool_discard's filtering
liw Jul 31, 2026
2934059
Fix retries for -DER_AGAIN
liw Aug 4, 2026
075effa
Remove "create" from migrate_get_cont_child
liw Aug 3, 2026
b6f0675
Fix recov_cont segfaults
liw Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions src/mgmt/srv_pool.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,8 +411,7 @@ ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
pool_comp_state_t state, size_t scm_size, size_t nvme_size,
size_t meta_size, bool skip_rf_check)
{
uint64_t deadline;
int rc;
int rc;

if (state == PO_COMP_ST_UP) {
/* When doing reintegration, need to make sure the pool is created and started on
Expand All @@ -436,16 +435,8 @@ ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
}
}

deadline = mgmt_ps_call_deadline();

again:
rc = dsc_pool_svc_update_target_state(pool_uuid, svc_ranks, deadline, target_addrs, state,
skip_rf_check);
if (rc == -DER_AGAIN && state == PO_COMP_ST_UP && daos_getmtime_coarse() < deadline) {
D_WARN("Retry incremental reintegration for pool " DF_UUID " because of race\n",
DP_UUID(pool_uuid));
goto again;
}
rc = dsc_pool_svc_update_target_state(pool_uuid, svc_ranks, mgmt_ps_call_deadline(),
target_addrs, state, skip_rf_check);

return rc;
}
Expand Down
44 changes: 13 additions & 31 deletions src/object/srv_obj_migrate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1865,7 +1865,7 @@ migrate_punch(struct migrate_pool_tls *tls, struct migrate_one *mrone,

static int
migrate_get_cont_child(struct migrate_pool_tls *tls, uuid_t cont_uuid,
struct ds_cont_child **cont_p, bool create)
struct ds_cont_child **cont_p)
{
struct ds_cont_child *cont_child = NULL;
int rc;
Expand All @@ -1876,32 +1876,14 @@ migrate_get_cont_child(struct migrate_pool_tls *tls, uuid_t cont_uuid,
return 0;
}

/* For incremental reintegration, the container has already been (re)-created. */
if (create && !tls->mpt_reintegrating) {
/* Since the shard might be moved different location for any pool operation,
* so it may need create the container in all cases.
*/
rc = ds_cont_child_open_create(tls->mpt_pool_uuid, cont_uuid, false, &cont_child);
if (rc != 0) {
if (rc == -DER_CONT_DESTROYING)
D_DEBUG(DB_REBUILD,
DF_RB ": container " DF_UUID
"already destroyed or destroying\n",
DP_RB_MPT(tls), DP_UUID(cont_uuid));
D_ASSERT(cont_child == NULL);
return rc;
}
} else {
rc = ds_cont_child_lookup(tls->mpt_pool_uuid, cont_uuid, &cont_child);
if (rc != 0) {
if (rc == -DER_CONT_NONEXIST || rc == -DER_CONT_DESTROYING)
D_DEBUG(DB_REBUILD,
DF_RB ": container " DF_UUID
"already destroyed or destroying\n",
DP_RB_MPT(tls), DP_UUID(cont_uuid));
D_ASSERT(cont_child == NULL);
return rc;
}
rc = ds_cont_child_lookup(tls->mpt_pool_uuid, cont_uuid, &cont_child);
if (rc != 0) {
if (rc == -DER_CONT_NONEXIST || rc == -DER_CONT_DESTROYING)
D_DEBUG(DB_REBUILD,
DF_RB ": container " DF_UUID "already destroyed or destroying\n",
DP_RB_MPT(tls), DP_UUID(cont_uuid));
D_ASSERT(cont_child == NULL);
return rc;
}

*cont_p = cont_child;
Expand All @@ -1917,7 +1899,7 @@ migrate_dkey(struct migrate_pool_tls *tls, struct migrate_one *mrone,
int rc;

D_ASSERT(dss_get_module_info()->dmi_xs_id != 0);
rc = migrate_get_cont_child(tls, mrone->mo_cont_uuid, &cont, true);
rc = migrate_get_cont_child(tls, mrone->mo_cont_uuid, &cont);
if (rc || cont == NULL)
D_GOTO(out, rc);

Expand Down Expand Up @@ -2985,7 +2967,7 @@ migrate_enum_unpack_cb(struct dc_obj_enum_unpack_io *io, void *data)
/* Create the vos container when no record need to be rebuilt for this shard,
* for the case of reintegrate the container was discarded ahead.
*/
rc = migrate_get_cont_child(tls, arg->arg->cont_uuid, &cont, true);
rc = migrate_get_cont_child(tls, arg->arg->cont_uuid, &cont);
if (cont != NULL)
ds_cont_child_put(cont);

Expand Down Expand Up @@ -3039,7 +3021,7 @@ migrate_obj_punch_one(void *data)
D_DEBUG(DB_REBUILD, DF_RB ": tls %p version %d punch " DF_U64 " " DF_UOID "\n",
DP_RB_MPT(tls), tls, arg->version, arg->punched_epoch, DP_UOID(arg->oid));

rc = migrate_get_cont_child(tls, arg->cont_uuid, &cont, true);
rc = migrate_get_cont_child(tls, arg->cont_uuid, &cont);
if (rc != 0 || cont == NULL)
D_GOTO(out, rc);

Expand Down Expand Up @@ -3497,7 +3479,7 @@ migrate_obj_ult(void *data)
struct ds_cont_child *cont_child = NULL;

/* check again to see if the container is being destroyed. */
migrate_get_cont_child(tls, arg->cont_uuid, &cont_child, false);
migrate_get_cont_child(tls, arg->cont_uuid, &cont_child);
if (cont_child != NULL && !cont_child->sc_stopping) {
if (vos_oi_exist(cont_child->sc_hdl, arg->oid)) {
stable_epoch = vos_cont_get_global_stable_epoch(cont_child->sc_hdl);
Expand Down
5 changes: 0 additions & 5 deletions src/pool/rpc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,6 @@ enum pool_map_update_flags {
POOL_RESET_RECOV_CONT = (1 << 1),
};

enum pool_recov_cont_flags {
PRCF_BIND_BULK = (1 << 0),
};

static inline uint32_t
pool_opc_2map_opc(uint32_t pool_opc)
{
Expand Down Expand Up @@ -942,7 +938,6 @@ CRT_RPC_DECLARE(pool_eval_self_heal, DAOS_ISEQ_POOL_EVAL_SELF_HEAL, DAOS_OSEQ_PO
((uint32_t) (prci_flags) CRT_VAR) \
((uint32_t) (prci_padding) CRT_VAR) \
((uint64_t) (prci_cont_nr) CRT_VAR) \
((crt_bulk_t) (prci_cont_bulk) CRT_VAR) \

@Nasf-Fan Nasf-Fan Jul 16, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer to keep this RPC field. As you mentioned, there are some CaRT issues. Be as some temporary solution, can we assign prci_cont_bulk as NULL and use CaRT internal bulk mechanism as your patch does? On the receiver side, start bulk transfer only when prci_cont_bulk is non-NULL.

Container recovery is not the unique bulk user in pool related logic. So I think that related CaRT issue needs to resolved sooner or later. Once done, then we can explicitly use such bulk transfer without worrying about potential interoperability trouble.

That also makes the patch to be smaller.

((struct pool_target_addr) (prci_addrs) CRT_ARRAY)

#define DAOS_OSEQ_POOL_RECOV_CONT /* output fields */ \
Expand Down
25 changes: 17 additions & 8 deletions src/pool/srv_cli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* (C) Copyright 2017-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -960,12 +960,18 @@ pool_extend_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
static int
pool_extend_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_extend_arg *arg = varg;
struct pool_extend_out *out = crt_reply_get(rpc);
int rc = out->peo_op.po_rc;

if (rc != 0)
DL_ERROR(rc, DF_UUID ": Failed to set targets to UP state for reintegration",
DP_UUID(pool_uuid));
if (rc == -DER_AGAIN) {
D_DEBUG(DB_MGMT, DF_UUID ": extend to %d ranks: " DF_RC "\n", DP_UUID(pool_uuid),
arg->pea_ntargets, DP_RC(rc));
rc = DSC_POOL_SVC_CALL_AGAIN;
} else if (rc != 0) {
DL_ERROR(rc, DF_UUID ": failed to extend to %d ranks", DP_UUID(pool_uuid),
arg->pea_ntargets);
}
return rc;
}

Expand Down Expand Up @@ -1016,11 +1022,14 @@ pool_update_target_state_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
struct pool_tgt_update_out *out = crt_reply_get(rpc);
int rc = out->pto_op.po_rc;

if (rc != 0)
if (rc == -DER_AGAIN) {
D_DEBUG(DB_MGMT, DF_UUID ": set targets to state %s: " DF_RC "\n",
DP_UUID(pool_uuid), pool_map_status2name(arg->puta_state), DP_RC(rc));
rc = DSC_POOL_SVC_CALL_AGAIN;
} else if (rc != 0) {
DL_ERROR(rc, DF_UUID ": Failed to set targets to %s state", DP_UUID(pool_uuid),
arg->puta_state == PO_COMP_ST_DOWN ? "DOWN"
: arg->puta_state == PO_COMP_ST_UP ? "UP"
: "UNKNOWN");
pool_map_status2name(arg->puta_state));
}
return rc;
}

Expand Down
125 changes: 75 additions & 50 deletions src/pool/srv_pool.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8058,8 +8058,7 @@ pool_discard(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_li
struct pool_tgt_discard_in *ptdi_in;
struct pool_tgt_discard_out *ptdi_out;
crt_rpc_t *rpc;
d_rank_list_t *rank_list = NULL;
struct pool_target_addr_list valid_list = {0};
d_rank_list_t *rank_list = NULL;
crt_opcode_t opc = POOL_TGT_DISCARD;
int i;
int rc;
Expand All @@ -8072,34 +8071,23 @@ pool_discard(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_li
D_ASSERTF(svc->ps_pool->sp_incr_reint == 0,
"incremental reint should not get here\n");

rc = pool_discard_filter_tgts(svc, list, reint, &valid_list);
if (rc != 0)
D_GOTO(out, rc);

if (valid_list.pta_number == 0) {
D_INFO(DF_UUID " discard 0 valid target.", DP_UUID(svc->ps_pool->sp_uuid));
D_GOTO(out, rc = 0);
}

rank_list = d_rank_list_alloc(valid_list.pta_number);
rank_list = d_rank_list_alloc(list->pta_number);
if (rank_list == NULL)
D_GOTO(out, rc = -DER_NOMEM);

rank_list->rl_nr = 0;
/* remove the duplicate ranks from list, see reintegrate target case */
for (i = 0; i < valid_list.pta_number; i++) {
if (daos_rank_in_rank_list(rank_list, valid_list.pta_addrs[i].pta_rank))
for (i = 0; i < list->pta_number; i++) {
if (daos_rank_in_rank_list(rank_list, list->pta_addrs[i].pta_rank))
continue;

rank_list->rl_ranks[rank_list->rl_nr++] = valid_list.pta_addrs[i].pta_rank;
rank_list->rl_ranks[rank_list->rl_nr++] = list->pta_addrs[i].pta_rank;
D_DEBUG(DB_MD, DF_UUID ": discard rank %u\n", DP_UUID(svc->ps_pool->sp_uuid),
valid_list.pta_addrs[i].pta_rank);
list->pta_addrs[i].pta_rank);
}

if (rank_list->rl_nr == 0) {
D_INFO(DF_UUID " discard 0 rank.", DP_UUID(svc->ps_pool->sp_uuid));
D_GOTO(out, rc = 0);
}
D_ASSERTF(rank_list->rl_nr > 0, DF_UUID ": discard 0 rank\n",
DP_UUID(svc->ps_pool->sp_uuid));

rc = crt_corpc_req_create(ctx, NULL, rank_list, opc, NULL,
NULL, CRT_RPC_FLAG_FILTER_INVERT,
Expand All @@ -8108,8 +8096,8 @@ pool_discard(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_li
D_GOTO(out, rc);

ptdi_in = crt_req_get(rpc);
ptdi_in->ptdi_addrs.ca_arrays = valid_list.pta_addrs;
ptdi_in->ptdi_addrs.ca_count = valid_list.pta_number;
ptdi_in->ptdi_addrs.ca_arrays = list->pta_addrs;
ptdi_in->ptdi_addrs.ca_count = list->pta_number;
uuid_copy(ptdi_in->ptdi_uuid, svc->ps_pool->sp_uuid);
rc = dss_rpc_send(rpc);

Expand All @@ -8125,7 +8113,54 @@ pool_discard(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_li
out:
if (rank_list)
d_rank_list_free(rank_list);
return rc;
}

static int
pool_recov_cont(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_list *list);

/*
* Prepare targets to for reintegration or extension. Return
* - 0 if successful,
* - 1 if no target needs to join, or
* - a negative error code.
*/
static int
pool_join_pre(crt_context_t ctx, struct pool_svc *svc, crt_opcode_t opc,
struct pool_target_addr_list *list)
{
struct pool_target_addr_list valid_list = {0};
bool reint;
int rc;

D_ASSERT(opc == POOL_REINT || opc == POOL_EXTEND);
reint = (opc == POOL_REINT);

rc = pool_discard_filter_tgts(svc, list, reint, &valid_list);
if (rc != 0)
goto out;

if (valid_list.pta_number == 0) {
D_INFO(DF_UUID ": no valid target to join\n", DP_UUID(svc->ps_uuid));
rc = 1;
goto out_valid_list;
}

rc = pool_recov_cont(ctx, svc, &valid_list);
if (rc != 0) {
DL_INFO(rc, DF_UUID ": recover containers", DP_UUID(svc->ps_uuid));
goto out_valid_list;
}

if (!reint || svc->ps_pool->sp_reint_mode == DAOS_REINT_MODE_DATA_SYNC) {
rc = pool_discard(ctx, svc, &valid_list, reint);
if (rc != 0)
DL_ERROR(rc, DF_UUID ": pool_discard failed", DP_UUID(svc->ps_uuid));
}

out_valid_list:
pool_target_addr_list_free(&valid_list);
out:
return rc;
}

Expand Down Expand Up @@ -8163,18 +8198,19 @@ ds_pool_extend_handler(crt_rpc_t *rpc)
if (rc != 0)
goto out;

rc = pool_discard(rpc->cr_ctx, svc, &tgt_addr_list, false);
if (rc) {
DL_ERROR(rc, DF_UUID ": pool_discard failed.", DP_UUID(in->pei_op.pi_uuid));
goto failed;
rc = pool_join_pre(rpc->cr_ctx, svc, opc_get(rpc->cr_opc), &tgt_addr_list);
if (rc != 0) {
if (rc == 1) /* already joined */
rc = 0;
goto out_svc;
}

rc = pool_svc_update_map(svc, pool_opc_2map_opc(opc_get(rpc->cr_opc)),
false /* exclude_rank */, &rank_list, domains, ndomains, NULL,
NULL, &out->peo_op.po_map_version, &out->peo_op.po_hint, MUS_DMG,
POOL_TGT_UPDATE_SKIP_RF_CHECK);
POOL_TGT_UPDATE_SKIP_RF_CHECK | POOL_RESET_RECOV_CONT);

failed:
out_svc:
pool_svc_put_leader(svc);
out:
if (tgt_addr_list.pta_addrs != NULL)
Expand Down Expand Up @@ -8221,8 +8257,7 @@ pool_recov_cont(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr
}
}

if (unlikely(ranks->rl_nr == 0))
D_GOTO(out, rc = 0);
D_ASSERTF(ranks->rl_nr > 0, DF_UUID ": recover cont on 0 rank\n", DP_UUID(svc->ps_uuid));

rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx);
if (rc != 0)
Expand Down Expand Up @@ -8253,29 +8288,20 @@ pool_recov_cont(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr
rc = crt_bulk_create(ctx, &cont_sgl, CRT_BULK_RO, &bulk);
if (rc != 0)
goto out;

if (ranks->rl_nr > 1) {
rc = crt_bulk_bind(bulk, ctx);
if (rc != 0)
goto out;
}
}

rc = crt_corpc_req_create(ctx, NULL, ranks, opc, NULL, NULL, CRT_RPC_FLAG_FILTER_INVERT,
crt_tree_topo(CRT_TREE_KNOMIAL, 32), &rpc);
rc = crt_corpc_req_create(ctx, NULL, ranks, opc, bulk /* co_bulk_hdl */, NULL,
CRT_RPC_FLAG_FILTER_INVERT, crt_tree_topo(CRT_TREE_KNOMIAL, 32),
&rpc);
if (rc != 0)
D_GOTO(out, rc);

prci = crt_req_get(rpc);
prci->prci_cont_nr = cont_nr;
prci->prci_cont_bulk = bulk;
prci->prci_addrs.ca_count = list->pta_number;
prci->prci_addrs.ca_arrays = list->pta_addrs;
uuid_copy(prci->prci_uuid, svc->ps_uuid);
if (ranks->rl_nr > 1)
prci->prci_flags = PRCF_BIND_BULK;
else
prci->prci_flags = 0;
prci->prci_flags = 0;

rc = dss_rpc_send(rpc);
if (rc != 0)
Expand Down Expand Up @@ -8325,14 +8351,13 @@ pool_update_handler(crt_rpc_t *rpc, int handler_version)
goto out;

if (opc_get(rpc->cr_opc) == POOL_REINT) {
if (svc->ps_pool->sp_reint_mode == DAOS_REINT_MODE_DATA_SYNC) {
rc = pool_discard(rpc->cr_ctx, svc, &list, true);
} else if (svc->ps_pool->sp_reint_mode == DAOS_REINT_MODE_INCREMENTAL) {
flags |= POOL_RESET_RECOV_CONT;
rc = pool_recov_cont(rpc->cr_ctx, svc, &list);
}
if (rc)
rc = pool_join_pre(rpc->cr_ctx, svc, opc_get(rpc->cr_opc), &list);
if (rc != 0) {
if (rc == 1) /* already joined */
rc = 0;
goto out_svc;
}
flags |= POOL_RESET_RECOV_CONT;
}

rc = pool_svc_update_map(svc, pool_opc_2map_opc(opc_get(rpc->cr_opc)),
Expand Down
Loading
Loading