From ce4e019d58b128ce2986ca29ed4669f237099cf5 Mon Sep 17 00:00:00 2001 From: Cengiz Karakoyunlu Date: Fri, 11 Nov 2011 16:32:41 +0000 Subject: [PATCH] get_member_attr is all set except for a few minor things, such as the oid comparison in sys_readdirplus.sm and creating a new collection whenever a new directory is created git-svn-id: svn+ssh://137.99.13.219/srv/svn/ofsproject/ofsproject/2.8.4-stock/osd_branch@52 dfcdf660-53a5-499b-8775-f9b82cf2d3b9 --- src/client/sysint/client-state-machine.h | 3 - src/client/sysint/mgmt-noop.sm | 2 +- src/client/sysint/module.mk.in | 3 +- src/client/sysint/osd.c | 6 +- src/client/sysint/osdsm.sm | 24 +- src/client/sysint/remove.sm | 26 ++- src/client/sysint/sys-create.sm | 252 +++++++++++++------- src/client/sysint/sys-getattr.sm | 284 ++++++++++++++--------- src/client/sysint/sys-io.sm | 27 +-- src/client/sysint/sys-osd-dir.sm | 3 - src/client/sysint/sys-osd-io.sm | 3 +- src/client/sysint/sys-readdirplus.sm | 262 +++++++++++---------- src/client/sysint/sys-setattr.sm | 118 +++++++++- 13 files changed, 644 insertions(+), 369 deletions(-) diff --git a/src/client/sysint/client-state-machine.h b/src/client/sysint/client-state-machine.h index f0afc97..ca85297 100644 --- a/src/client/sysint/client-state-machine.h +++ b/src/client/sysint/client-state-machine.h @@ -793,9 +793,6 @@ extern struct PINT_state_machine_s pvfs2_client_datafile_getattr_sizes_sm; extern struct PINT_state_machine_s pvfs2_client_setattr_sm; extern struct PINT_state_machine_s pvfs2_client_io_sm; extern struct PINT_state_machine_s pvfs2_client_osd_io_sm; -extern struct PINT_state_machine_s pvfs2_client_osd_dirops_sm; -extern struct PINT_state_machine_s pvfs2_client_osd_dirops_attr1_sm; -extern struct PINT_state_machine_s pvfs2_client_osd_dirops_attr4_sm; extern struct PINT_state_machine_s pvfs2_client_small_io_sm; extern struct PINT_state_machine_s pvfs2_client_flush_sm; extern struct PINT_state_machine_s pvfs2_client_sysint_readdir_sm; diff --git a/src/client/sysint/mgmt-noop.sm b/src/client/sysint/mgmt-noop.sm index 8a625ef..d069528 100644 --- a/src/client/sysint/mgmt-noop.sm +++ b/src/client/sysint/mgmt-noop.sm @@ -184,10 +184,10 @@ static PINT_sm_action mgmt_noop_setup_msgpair( js_p->error_code = OSD_MSGPAIR; } else { PINT_SERVREQ_MGMT_NOOP_FILL(sm_p->msgarray_op.msgpair.req, *sm_p->cred_p, sm_p->hints); - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); js_p->error_code = 0; } + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } diff --git a/src/client/sysint/module.mk.in b/src/client/sysint/module.mk.in index a83139d..4f606c9 100644 --- a/src/client/sysint/module.mk.in +++ b/src/client/sysint/module.mk.in @@ -52,8 +52,7 @@ CLIENT_SMCGEN := \ $(DIR)/mgmt-create-dirent.c \ $(DIR)/mgmt-get-dirdata-handle.c \ $(DIR)/osdsm.c \ - $(DIR)/sys-osd-io.c \ - $(DIR)/sys-osd-dir.c + $(DIR)/sys-osd-io.c # track generated .c files that need to be removed during dist clean, etc. SMCGEN += $(CLIENT_SMCGEN) diff --git a/src/client/sysint/osd.c b/src/client/sysint/osd.c index 313bb70..ab01a07 100755 --- a/src/client/sysint/osd.c +++ b/src/client/sysint/osd.c @@ -265,8 +265,10 @@ void osd_find_scsi_addresses(PVFS_fs_id fs_id, int32_t *num_osd, goto out; } for (j = 0; j < num_drives; j++) { - if (strcmp(drives[j].hostname, - hmap->alias_mapping->host_alias) == 0) { + /*if (strcmp(drives[j].targetname, + hmap->alias_mapping->host_alias) == 0) {*/ + if (strcmp(drives[j].targetname, + "beaf10") == 0) { if (num >= max_addrs) { num = 0; goto out; diff --git a/src/client/sysint/osdsm.sm b/src/client/sysint/osdsm.sm index 7b3bdc9..6662247 100755 --- a/src/client/sysint/osdsm.sm +++ b/src/client/sysint/osdsm.sm @@ -33,6 +33,7 @@ #include "pint-util.h" #include "server-config-mgr.h" #include "pvfs2-internal.h" +#include "osd-util/osd-util.h" enum { @@ -75,6 +76,12 @@ nested machine pvfs2_osd_msgpairarray_sm { run osd_msgpairarray_all_complete; MSGPAIRS_RETRY => post_retry; + default => done; + } + + state done + { + run osd_msgpairarray_done; default => return; } } @@ -83,7 +90,8 @@ nested machine pvfs2_osd_msgpairarray_sm static int osd_msgpairarray_init(struct PINT_smcb *smcb, job_status_s *js_p) { - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + int i = 0; PINT_sm_msgpair_state *msg_p = NULL; @@ -127,7 +135,7 @@ static int osd_msgpairarray_init(struct PINT_smcb *smcb, job_status_s *js_p) */ static int osd_msgpairarray_post(struct PINT_smcb *smcb, job_status_s *js_p) { - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); int ret = -PVFS_EINVAL, i; PVFS_msg_tag_t session_tag; PINT_sm_msgpair_state *msg_p; @@ -213,7 +221,7 @@ static int osd_msgpairarray_post_retry(struct PINT_smcb *smcb, static int osd_msgpairarray_one_complete(struct PINT_smcb *smcb, job_status_s *js_p) { - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); PINT_sm_msgpair_state *msg_p; gossip_debug(GOSSIP_MSGPAIR_DEBUG, "%s: sm %p tag %lld count %d\n", @@ -242,7 +250,7 @@ static int osd_msgpairarray_one_complete(struct PINT_smcb *smcb, static int osd_msgpairarray_all_complete(struct PINT_smcb *smcb, job_status_s *js_p) { - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); int i; js_p->error_code = 0; @@ -285,6 +293,14 @@ static int osd_msgpairarray_all_complete(struct PINT_smcb *smcb, return SM_ACTION_COMPLETE; } +static int osd_msgpairarray_done( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int task_id, error_code, remaining; + PINT_sm_pop_frame(smcb, &task_id, &error_code, &remaining); + return SM_ACTION_COMPLETE; +} + /* * vim: ts=8 sts=4 sw=4 expandtab ft=c */ diff --git a/src/client/sysint/remove.sm b/src/client/sysint/remove.sm index 11a246f..37fb6ac 100644 --- a/src/client/sysint/remove.sm +++ b/src/client/sysint/remove.sm @@ -23,7 +23,8 @@ extern job_context_id pint_client_sm_context; enum { REMOVE_MUST_REMOVE_DATAFILES = 1, - OSD_MSGPAIR = 2001 + OSD_MSGPAIR = 2001, + OSD_MDFILE_MSGPAIR = 2002 }; #define HANDLE_REMOVE_ERROR(state) \ @@ -94,6 +95,7 @@ nested machine pvfs2_client_remove_helper_sm state object_remove_setup_msgpair { run remove_object_remove_setup_msgpair; + OSD_MDFILE_MSGPAIR => remove_helper_cleanup; OSD_MSGPAIR => object_remove_xfer_osd_msgpair; success => object_remove_xfer_msgpair; default => object_remove_failure; @@ -131,7 +133,6 @@ nested machine pvfs2_client_remove_helper_sm static PINT_sm_action remove_getattr_init( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("remove_getattr_init\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_SM_GETATTR_STATE_FILL( sm_p->getattr, @@ -145,7 +146,6 @@ static PINT_sm_action remove_getattr_init( static PINT_sm_action remove_getattr_analyze_results( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("remove_getattr_analyze_results\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_attr *attr = NULL; attr = &sm_p->getattr.attr; @@ -189,7 +189,6 @@ static PINT_sm_action remove_getattr_analyze_results( static PINT_sm_action remove_datafile_remove_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("remove_datafile_remove_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_attr *attr = NULL; int ret = -PVFS_EINVAL; @@ -247,22 +246,30 @@ static PINT_sm_action remove_datafile_remove_setup_msgpair( attr->u.meta.dfile_count, attr->u.meta.dfile_array, sm_p->hints); - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); } + + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } static PINT_sm_action remove_object_remove_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("remove_object_remove_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PINT_sm_msgpair_state *msg_p = NULL; + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); + gossip_debug(GOSSIP_CLIENT_DEBUG, "remove state: object_remove_setup_msgpair\n"); + if (is_osd_md) { + /* no mdfile to remove */ + js_p->error_code = OSD_MDFILE_MSGPAIR; + return SM_ACTION_COMPLETE; + } + PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; @@ -304,12 +311,12 @@ static PINT_sm_action remove_object_remove_setup_msgpair( sm_p->object_ref.handle, sm_p->hints); js_p->error_code = 0; - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); } msg_p->retry_flag = PVFS_MSGPAIR_RETRY; msg_p->comp_fn = NULL; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -348,7 +355,6 @@ static PINT_sm_action remove_object_remove_failure( static PINT_sm_action remove_helper_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("remove_helper_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); gossip_debug(GOSSIP_CLIENT_DEBUG, "remove state: helper cleanup\n"); @@ -358,6 +364,10 @@ static PINT_sm_action remove_helper_cleanup( PINT_msgpairarray_destroy(&sm_p->msgarray_op); + /* came directly from object_remove_setup_msgpair, no error to check */ + if (js_p->error_code == OSD_MDFILE_MSGPAIR) + js_p->error_code = 0; + if (js_p->error_code) { char buf[64] = {0}; diff --git a/src/client/sysint/sys-create.sm b/src/client/sysint/sys-create.sm index 6276f72..eeda35b 100644 --- a/src/client/sysint/sys-create.sm +++ b/src/client/sysint/sys-create.sm @@ -49,14 +49,13 @@ static int create_crdirent_comp_fn( static int create_delete_handles_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); -int osd_create_flag = 0; - /* misc helper functions */ static PINT_dist* get_default_distribution(PVFS_fs_id fs_id); enum { OSD_MSGPAIR = 2001, - OSD_META_MSGPAIR = 2002 + OSD_CREATE_MSGPAIR = 2002, + OSD_MDFILE_MSGPAIR = 2003 }; %% @@ -87,6 +86,8 @@ machine pvfs2_client_create_sm { run create_create_setup_msgpair; OSD_MSGPAIR => create_xfer_osd_msgpair; + OSD_CREATE_MSGPAIR => create_xfer_datafile_msgpair; + OSD_MDFILE_MSGPAIR => datafiles_setup_msgpair_array; success => create_xfer_msgpair; default => cleanup; } @@ -105,19 +106,17 @@ machine pvfs2_client_create_sm default => cleanup; } - state crdirent_setup_msgpair + state create_xfer_datafile_msgpair { - run create_crdirent_setup_msgpair; - OSD_MSGPAIR => datafiles_setup_msgpair_array; - OSD_META_MSGPAIR => crdirent_osd_msgpair; - success => crdirent_xfer_msgpair; - default => crdirent_failure; + jump pvfs2_msgpairarray_sm; + success => datafiles_setup_msgpair_array; + default => cleanup; } - state crdirent_osd_msgpair + state crdirent_setup_msgpair { - jump pvfs2_client_osd_dirops_sm; - success => cleanup; + run create_crdirent_setup_msgpair; + success => crdirent_xfer_msgpair; default => crdirent_failure; } @@ -131,10 +130,25 @@ machine pvfs2_client_create_sm state datafiles_xfer_osd_msgpair_array { jump pvfs2_osd_msgpairarray_sm; + /*success => test_fetch_attrs;*/ success => create_setattr_setup_msgpair; default => crdirent_failure; } + state test_fetch_attrs + { + run test_fetch_attrs_func; + success => test_fetch_attrs_xfer_msgpair; + default => cleanup; + } + + state test_fetch_attrs_xfer_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => create_setattr_setup_msgpair; + default => cleanup; + } + state create_setattr_setup_msgpair { run create_setattr_setup_msgpair; @@ -425,14 +439,8 @@ static int create_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("create_comp_fn\n"); PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p; - if (smcb->base_frame) { - sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); - } else { - sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - } + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); @@ -456,11 +464,11 @@ static int create_comp_fn(void *v_p, osd_error_xerrno(ret, "%s: attr_resolve failed", __func__); } oid = get_ntohll(sm_p->msgarray_op.msgpair.osd_command.attr[0].val); + sm_p->u.create.metafile_handle = oid; sm_p->u.create.create_resp->ref.handle = oid; sm_p->u.create.create_resp->ref.fs_id = sm_p->object_ref.fs_id; osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); - } else { assert(resp_p->op == PVFS_SERV_CREATE); @@ -486,7 +494,6 @@ static int create_comp_fn(void *v_p, resp_p->u.create.datafile_count)); sm_p->u.create.stuffed = resp_p->u.create.stuffed; } - gossip_debug( GOSSIP_CLIENT_DEBUG, "*** Got newly created handle %llu\n", llu(sm_p->u.create.metafile_handle)); @@ -494,13 +501,50 @@ static int create_comp_fn(void *v_p, return 0; } +static int test_fetch_attrs_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + gossip_err("test_fetch_attrs_comp_fn\n"); + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + PVFS_error status; + int64_t size[sm_p->msgarray_op.count]; + int i; + + status = osd_errno_from_status( + sm_p->msgarray_op.msgarray[index].osd_command.status); + + if (status != 0) + { + return status; + } + + struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; + int ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", + __func__); + return ret; + } + + for (i=0; imsgarray_op.count; i++) { + size[i] = get_ntohll(command->attr[i].val); + gossip_err("%lu\n", size[i]); + } + + osd_command_attr_free(command); + + return 0; +} + static int create_datafiles_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { PVFS_error status; PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); @@ -581,17 +625,10 @@ static int create_setattr_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("create_setattr_comp_fn\n"); int res, status; PVFS_object_ref refn; PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p; - if (smcb->base_frame) - { - sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); - } else { - sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - } + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); gossip_debug(GOSSIP_CLIENT_DEBUG, "create_setattr_comp_fn\n"); @@ -622,7 +659,6 @@ static int create_crdirent_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("create_crdirent_comp_fn\n"); gossip_debug(GOSSIP_CLIENT_DEBUG, "create_crdirent_comp_fn\n"); assert(resp_p->op == PVFS_SERV_CRDIRENT); @@ -642,11 +678,24 @@ static PINT_sm_action create_create_setup_msgpair( * Check whether we're storing file metadata on a dedicated OSD MDS * or as attributes of the first datafile. */ + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); - + int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: " "dspace_create_setup_msgpair\n"); + if (is_osd_md) { + /* + * OK, so osd_type == OSD_MDFILE which means that we're storing the + * file metadata as attributes of the first datafile. There is nothing + * else to be done here so we'll go ahead and skip the + * dspace_create_xfer state and jump directly to the + * datafiles_setup_msgpair_array state. + */ + js_p->error_code = OSD_MDFILE_MSGPAIR; + return SM_ACTION_COMPLETE; + } + gossip_debug(GOSSIP_CLIENT_DEBUG," create: posting create req\n"); /* reset the attributes to what got passed in to the sysint call. the retry @@ -682,12 +731,10 @@ static PINT_sm_action create_create_setup_msgpair( } if (is_osd_meta) { - /* * Create a zero-length datafile on the metadata OSD. We'll fill in * the required attributes in create_setattr_setup_msgpair(). */ - struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG, CCAP_OID, NULL, CCAP_OID_LEN }; @@ -708,7 +755,9 @@ static PINT_sm_action create_create_setup_msgpair( return SM_ACTION_COMPLETE; } js_p->error_code = OSD_MSGPAIR; + } else { + /* vanilla PVFS */ PINT_SERVREQ_CREATE_FILL( msg_p->req, *sm_p->cred_p, @@ -718,6 +767,8 @@ static PINT_sm_action create_create_setup_msgpair( sm_p->u.create.layout, sm_p->hints); js_p->error_code = 0; + if (is_osd) + js_p->error_code = OSD_CREATE_MSGPAIR; } msg_p->fs_id = sm_p->object_ref.fs_id; @@ -730,16 +781,13 @@ static PINT_sm_action create_create_setup_msgpair( msg_p->req.u.create.attr.u.meta.dist_size = PINT_DIST_PACK_SIZE(sm_p->u.create.dist); - if (!(is_osd_meta)) { - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - } + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } static PINT_sm_action create_datafiles_setup_msgpair_array( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("create_datafiles_setup_msgpair_array\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; struct server_configuration_s *server_config; @@ -810,10 +858,14 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( #define CURRENT_COMMAND_PAGE 0xfffffffeUL #define CURRENT_COMMAND_PAGE_OID 4 - if (is_osd) { + if (is_osd) { struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; - struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG, - CCAP_OID, NULL, CCAP_OID_LEN }; + uint64_t attrval; + struct attribute_list attrs[] = {{ ATTR_GET, CUR_CMD_ATTR_PG, + CCAP_OID, NULL, CCAP_OID_LEN }, + { ATTR_SET, USER_COLL_PG, 1, &attrval, 8}}; + + set_htonll(&attrval, COLLECTION_OID_LB); /* * A hack in PVFS_util_init_defaults has set up an object an the @@ -828,7 +880,7 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( return 1; } - ret = osd_command_attr_build(command, &attr, 1); + ret = osd_command_attr_build(command, attrs, 2); if (ret) { osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); @@ -844,20 +896,72 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( msg_p->comp_fn = create_datafiles_comp_fn; msg_p->svr_addr = sm_p->u.create.data_server_addrs[0]; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action test_fetch_attrs_func( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + gossip_err("test_fetch_attrs_func\n"); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + int ret = -PVFS_EINVAL; + PINT_sm_msgpair_state *msg_p = NULL; + int i; + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->u.create.io_handle_extent_array[0]. + extent_array[0].first; + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = test_fetch_attrs_comp_fn; + msg_p->svr_addr = sm_p->u.create.data_server_addrs[0]; + + struct attribute_list attr[sm_p->msgarray_op.count]; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + /* Set attr type, page and number */ + for (i = 0; i < sm_p->msgarray_op.count; i++) { + attr[i].type = ATTR_GET; + attr[i].page = 0x1; + attr[i].number = 0x82; /* logical length (not used capacity) */ + attr[i].len = sizeof(uint64_t); + } + + ret = osd_command_set_get_member_attributes(command, PVFS_OSD_DATA_PID, COLLECTION_OID_LB); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_member_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, attr, sm_p->msgarray_op.count); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } + + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } static PINT_sm_action create_setattr_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("create_setattr_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PINT_sm_msgpair_state *msg_p = NULL; - + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); - + gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: setattr_setup_msgpair\n"); @@ -972,7 +1076,6 @@ static PINT_sm_action create_setattr_setup_msgpair( msg_p->req.u.setattr.attr.u.meta.dist_size = PINT_DIST_PACK_SIZE(sm_p->u.create.dist); } - msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = sm_p->u.create.metafile_handle; msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; @@ -987,34 +1090,22 @@ static PINT_sm_action create_setattr_setup_msgpair( js_p->error_code = ret; } - if(!(is_osd_meta)) - { - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - } + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } static PINT_sm_action create_crdirent_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("create_crdirent_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -1; PINT_sm_msgpair_state *msg_p = NULL; - server_configuration_s* server_config = NULL; - server_config = PINT_get_server_config_struct(sm_p->object_ref.fs_id); - - PINT_put_server_config_struct(server_config); - - if(server_config->osd_type && osd_create_flag){ - js_p->error_code = OSD_MSGPAIR; - osd_create_flag = 0; - return SM_ACTION_COMPLETE; - } gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: crdirent_setup_msgpair\n"); + js_p->error_code = 0; + gossip_debug(GOSSIP_CLIENT_DEBUG, "create: %s: posting crdirent req: parent handle: %llu, " "name: %s, handle: %llu\n", @@ -1025,6 +1116,20 @@ static PINT_sm_action create_crdirent_setup_msgpair( PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; + PINT_SERVREQ_CRDIRENT_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->u.create.object_name, + sm_p->u.create.metafile_handle, + sm_p->object_ref.handle, + sm_p->object_ref.fs_id, + sm_p->hints); + + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = create_crdirent_comp_fn; + ret = PINT_cached_config_map_to_server( &msg_p->svr_addr, sm_p->object_ref.handle, sm_p->object_ref.fs_id); @@ -1035,30 +1140,6 @@ static PINT_sm_action create_crdirent_setup_msgpair( js_p->error_code = ret; } - if (server_is_osd(msg_p->svr_addr)) { - /* - * Directory operations for metafile and mdfile. We don't do anything - * here because we'll handle the individual directory operations in - * different states. - */ - js_p->error_code = OSD_META_MSGPAIR; - } else { - PINT_SERVREQ_CRDIRENT_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->u.create.object_name, - sm_p->u.create.metafile_handle, - sm_p->object_ref.handle, - sm_p->object_ref.fs_id, - sm_p->hints); - js_p->error_code = 0; - } - - msg_p->fs_id = sm_p->object_ref.fs_id; - msg_p->handle = sm_p->object_ref.handle; - msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; - msg_p->comp_fn = create_crdirent_comp_fn; - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -1082,7 +1163,6 @@ static PINT_sm_action create_crdirent_failure( static PINT_sm_action create_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("create_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_ref metafile_ref; PVFS_size tmp_size = 0; diff --git a/src/client/sysint/sys-getattr.sm b/src/client/sysint/sys-getattr.sm index 6559da8..70757e5 100644 --- a/src/client/sysint/sys-getattr.sm +++ b/src/client/sysint/sys-getattr.sm @@ -25,6 +25,7 @@ #include "pvfs2-internal.h" #include "pvfs2-types-debug.h" #include "osd-util/osd-util.h" +#include "src/common/misc/extent-utils.h" /* pvfs2_client_getattr_sm * @@ -66,9 +67,16 @@ nested machine pvfs2_client_datafile_getattr_sizes_sm state datafile_getattr_setup_msgpairarray { run getattr_datafile_getattr_setup_msgpairarray; + OSD_MSGPAIR => datafile_getattr_xfer_osd_msgpairarray; success => datafile_getattr_xfer_msgpairarray; default => datafile_getattr_cleanup; } + + state datafile_getattr_xfer_osd_msgpairarray + { + jump pvfs2_osd_msgpairarray_sm; + default => datafile_getattr_cleanup; + } state datafile_getattr_xfer_msgpairarray { @@ -310,7 +318,6 @@ PVFS_error PVFS_sys_getattr( static PINT_sm_action getattr_acache_lookup( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("getattr_acache_lookup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); uint32_t trimmed_mask = 0; int missing_attrs; @@ -487,7 +494,6 @@ static PINT_sm_action getattr_acache_lookup( static PINT_sm_action getattr_object_getattr_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("getattr_object_getattr_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PVFS_object_ref object_ref; @@ -505,11 +511,12 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( msg_p->fs_id = object_ref.fs_id; msg_p->handle = object_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_RETRY; + msg_p->comp_fn = getattr_object_getattr_comp_fn; ret = PINT_cached_config_map_to_server( &msg_p->svr_addr, msg_p->handle, msg_p->fs_id); - if (ret) { gossip_err("Failed to map meta server address\n"); @@ -521,6 +528,7 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( int i, numattrs = 10; struct attribute_list attrl[numattrs]; struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + int is_osd_md = fsid_is_osd_md(object_ref.fs_id); /* Set attr type, page and number */ for (i = 0; i < numattrs - 3; i++) { @@ -576,6 +584,35 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( pid = PVFS_OSD_META_PID; /* datafile, metafile */ oid = object_ref.handle; + if (is_osd_md) { + struct server_configuration_s *config; + struct filesystem_configuration_s *fs; + PINT_llist *l; + int i; + + pid = PVFS_OSD_DATA_PID; /* mdfile: assume data first */ + config = PINT_get_server_config_struct(object_ref.fs_id); + fs = PINT_config_find_fs_id(config, object_ref.fs_id); + PINT_put_server_config_struct(config); + + for (l = fs->meta_handle_ranges; l; l = PINT_llist_next(l)) { + /* + * Could do BMI_addr_rev_lookup and compare rather than + * iterating all metadata extents, but that may be slower, + * depending on the number of md servers. + */ + struct host_handle_mapping_s *hhm = PINT_llist_head(l); + if (!hhm) + break; + PVFS_handle_extent_array *hea = &hhm->handle_extent_array; + for (i=0; i < hea->extent_count; i++) + if (PINT_handle_in_extent(&hea->extent_array[i], oid)) { + pid = PVFS_OSD_META_PID; /* aha, is meta */ + goto out; + } + } + } +out: ret = osd_command_set_get_attributes(command, pid, oid); if (ret) { osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", @@ -607,17 +644,9 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( sm_p->getattr.req_attrmask, sm_p->hints); js_p->error_code = 0; - } - - msg_p->fs_id = object_ref.fs_id; - msg_p->handle = object_ref.handle; - msg_p->retry_flag = PVFS_MSGPAIR_RETRY; - msg_p->comp_fn = getattr_object_getattr_comp_fn; - - if (!(js_p->error_code)) - { - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - } + } + + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -631,20 +660,10 @@ static int getattr_object_getattr_comp_fn( struct PVFS_server_resp *resp_p, int index) { - gossip_err("getattr_object_getattr_comp_fn\n"); PVFS_object_attr *attr = NULL; int ret, status; PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p; - if (smcb->base_frame) - { - sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); - } else { - sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - } - - gossip_debug(GOSSIP_GETATTR_DEBUG, - "getattr_object_getattr_comp_fn called\n"); + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); attr = &sm_p->getattr.attr; @@ -681,16 +700,19 @@ static int getattr_object_getattr_comp_fn( } else { assert(resp_p->op == PVFS_SERV_GETATTR); + gossip_debug(GOSSIP_GETATTR_DEBUG, + "getattr_object_getattr_comp_fn called\n"); + if (resp_p->status != 0) { return resp_p->status; } /* - * If we've reached the callback for the getattr msgpair tranfer, - * then we can make a copy of the retrieved attribute for later - * caching. - */ + * If we've reached the callback for the getattr msgpair tranfer, + * then we can make a copy of the retrieved attribute for later + * caching. + */ PINT_copy_object_attr(&sm_p->getattr.attr, &resp_p->u.getattr.attr); } @@ -859,6 +881,7 @@ static PINT_sm_action getattr_datafile_getattr_setup_msgpairarray( PVFS_object_attr *attr = NULL; PINT_sm_msgpair_state *msg_p = NULL; uint64_t mirror_retry = (sm_p->getattr.attr.mask & PVFS_ATTR_META_MIRROR_DFILES); + int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); gossip_debug(GOSSIP_MIRROR_DEBUG,"Executing getattr_datafile_getattr_setup_msgpairarray...\n"); gossip_debug(GOSSIP_MIRROR_DEBUG,"%s: Are we mirroring? %s\n",__func__ @@ -868,11 +891,11 @@ static PINT_sm_action getattr_datafile_getattr_setup_msgpairarray( ,sm_p->getattr.attr.mask ,(unsigned int)mirror_retry); - js_p->error_code = 0; + js_p->error_code = is_osd ? OSD_MSGPAIR : 0; attr = &sm_p->getattr.attr; assert(attr); - + PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; @@ -924,16 +947,48 @@ static PINT_sm_action getattr_datafile_getattr_setup_msgpairarray( getattr->index_to_server[i] = (uint32_t)i; } + if (is_osd) + { + /* for each datafile, post a send/recv pair to obtain the size */ + for(i = 0; i < attr->u.meta.dfile_count; i++) + { + struct osd_command *command = &sm_p->msgarray_op.msgarray[i].osd_command; + struct attribute_list id = { + .type = ATTR_GET, + .page = 0x1, + .number = 0x82, /* logical length (not used capacity) */ + .len = sizeof(uint64_t), + }; + + ret = osd_command_set_get_attributes(command, PVFS_OSD_DATA_PID, + attr->u.meta.dfile_array[i]); + if (ret) { + osd_error_xerrno(ret, + "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + ret = osd_command_attr_build(command, &id, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } + } + } else { //START_PROFILER(getattr_prof); - PINT_SERVREQ_TREE_GET_FILE_SIZE_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->getattr.object_ref.fs_id, - 0, - attr->u.meta.dfile_count, - handles, - (mirror_retry ? 1 : 0), - sm_p->hints); + PINT_SERVREQ_TREE_GET_FILE_SIZE_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->getattr.object_ref.fs_id, + 0, + attr->u.meta.dfile_count, + handles, + (mirror_retry ? 1 : 0), + sm_p->hints); + } msg_p->fs_id = sm_p->getattr.object_ref.fs_id; msg_p->handle = handles[0]; @@ -963,7 +1018,6 @@ static PINT_sm_action getattr_datafile_getattr_setup_msgpairarray( msg_p->retry_flag = PVFS_MSGPAIR_RETRY; } - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -976,79 +1030,103 @@ static int getattr_datafile_getattr_comp_fn( PINT_sm_msgpair_state *msg = &(sm_p->msgarray_op.msgarray[index]); struct PVFS_servreq_tree_get_file_size *tree = &(msg->req.u.tree_get_file_size); + int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); + PVFS_error status; + int64_t size; PINT_sm_getattr_state *getattr = &(sm_p->getattr); PINT_client_getattr_mirror_ctx *ctx = NULL; uint32_t server_nr = 0; int i = 0; - if (resp_p->status) - { /* tree request had a problem */ - return resp_p->status; + if (is_osd) { + status = osd_errno_from_status( + sm_p->msgarray_op.msgarray[index].osd_command.status); + } else { + assert(resp_p->op == PVFS_SERV_TREE_GET_FILE_SIZE); + status = resp_p->status; } - assert(resp_p->op == PVFS_SERV_TREE_GET_FILE_SIZE); - - - /* if we are mirroring, then we need to check the error code returned from - * each server. If an error is found, mirroring will try to get the size - * from a different server. Below, we are marking which handles completed - * successfully, which tells mirroring NOT to retry them. - */ - if ( getattr->attr.mask & PVFS_ATTR_META_MIRROR_DFILES) + if (status != 0) { - for (i=0; iu.tree_get_file_size.handle_count; i++) - { - if (resp_p->u.tree_get_file_size.error[i] != 0) - { /* error retrieving size for this handle..we will retry it. */ - continue; - } - server_nr = getattr->index_to_server[i]; - sm_p->getattr.size_array[server_nr] = resp_p->u.tree_get_file_size.size[i]; - ctx = &(getattr->mir_ctx_array[server_nr]); - ctx->msg_completed = 1; - - /*For completed messages, update the size array with the file size - *just retrieved. - */ - getattr->size_array[server_nr] = resp_p->u.tree_get_file_size.size[i]; - gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: size[%d]:%lld \thandle:%llu\n" - ,__func__ - ,i - ,llu(getattr->size_array[i]) - ,llu(tree->handle_array[i])); - }/*end for*/ + return status; } - else - { - /* if we are NOT mirroring and an error is found for an individual handle, - * then we must invalidate the size array and return an error code. - */ - for (i=0; iu.tree_get_file_size.handle_count; i++) - { - gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: error[%d]:%d" - "\tsize[%d]:%d\n" - ,__func__ - ,i - ,(int)resp_p->u.tree_get_file_size.error[i] - ,i - ,(int)resp_p->u.tree_get_file_size.size[i]); - - if (resp_p->u.tree_get_file_size.error[i] != 0) - { - gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: error[%d] is %d\n" + + if (is_osd) { + struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; + int ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", + __func__); + return ret; + } + if (command->attr->outlen != sizeof(uint64_t)) { + gossip_err("%s: expecting 8 bytes returned, got %u.\n", __func__, + command->attr->outlen); + return ret; + } + size = get_ntohll(command->attr->val); + osd_command_attr_free(command); + } else { + /* if we are mirroring, then we need to check the error code returned from + * each server. If an error is found, mirroring will try to get the size + * from a different server. Below, we are marking which handles completed + * successfully, which tells mirroring NOT to retry them. + */ + if ( getattr->attr.mask & PVFS_ATTR_META_MIRROR_DFILES) + { + for (i=0; iu.tree_get_file_size.handle_count; i++) + { + if (resp_p->u.tree_get_file_size.error[i] != 0) + { /* error retrieving size for this handle..we will retry it. */ + continue; + } + server_nr = getattr->index_to_server[i]; + sm_p->getattr.size_array[server_nr] = resp_p->u.tree_get_file_size.size[i]; + ctx = &(getattr->mir_ctx_array[server_nr]); + ctx->msg_completed = 1; + + /*For completed messages, update the size array with the file size + *just retrieved. + */ + getattr->size_array[server_nr] = resp_p->u.tree_get_file_size.size[i]; + gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: size[%d]:%lld \thandle:%llu\n" + ,__func__ + ,i + ,llu(getattr->size_array[i]) + ,llu(tree->handle_array[i])); + }/*end for*/ + } + else + { + /* if we are NOT mirroring and an error is found for an individual handle, + * then we must invalidate the size array and return an error code. + */ + for (i=0; iu.tree_get_file_size.handle_count; i++) + { + gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: error[%d]:%d" + "\tsize[%d]:%d\n" ,__func__ ,i - ,resp_p->u.tree_get_file_size.error[i]); - memset(getattr->size_array,0,sizeof(*getattr->size_array)); - return (resp_p->u.tree_get_file_size.error[i]); - } - - getattr->size_array[i] = resp_p->u.tree_get_file_size.size[i]; - }/*end for*/ - - }/*end if*/ - + ,(int)resp_p->u.tree_get_file_size.error[i] + ,i + ,(int)resp_p->u.tree_get_file_size.size[i]); + + if (resp_p->u.tree_get_file_size.error[i] != 0) + { + gossip_debug(GOSSIP_GETATTR_DEBUG,"%s: error[%d] is %d\n" + ,__func__ + ,i + ,resp_p->u.tree_get_file_size.error[i]); + memset(getattr->size_array,0,sizeof(*getattr->size_array)); + return (resp_p->u.tree_get_file_size.error[i]); + } + + getattr->size_array[i] = resp_p->u.tree_get_file_size.size[i]; + }/*end for*/ + + }/*end if*/ + } return(0); }/*end getattr_datafile_getattr_comp_fn*/ @@ -1278,7 +1356,6 @@ static PINT_sm_action getattr_datafile_getattr_cleanup( static PINT_sm_action getattr_acache_insert( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("getattr_acache_insert\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_size* tmp_size = NULL; @@ -1332,7 +1409,6 @@ static PINT_sm_action getattr_acache_insert( static PINT_sm_action getattr_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("getattr_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_getattr_state *getattr = &(sm_p->getattr); diff --git a/src/client/sysint/sys-io.sm b/src/client/sysint/sys-io.sm index 10bf2b6..b542afb 100644 --- a/src/client/sysint/sys-io.sm +++ b/src/client/sysint/sys-io.sm @@ -154,12 +154,6 @@ static int io_datafile_index_array_init( static void io_datafile_index_array_destroy( PINT_client_sm *sm_p); -int osd_io_flag = 1; - -enum { - OSD_MSGPAIR = 2001, -}; - %% machine pvfs2_client_io_sm @@ -181,7 +175,6 @@ machine pvfs2_client_io_sm state inspect_attr { run io_inspect_attr; - OSD_MSGPAIR => io_datafile_setup_msgpairs; IO_UNSTUFF => unstuff_setup_msgpair; IO_GETATTR_SERVER => unstuff_setup_msgpair; success => io_datafile_setup_msgpairs; @@ -484,18 +477,9 @@ static PINT_sm_action io_init( static PINT_sm_action io_inspect_attr( struct PINT_smcb *smcb, job_status_s *js_p) { - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - server_configuration_s* server_config = NULL; - server_config = PINT_get_server_config_struct(sm_p->object_ref.fs_id); - - PINT_put_server_config_struct(server_config); - - if(server_config->osd_type && osd_io_flag){ - js_p->error_code = OSD_MSGPAIR; - osd_io_flag = 0; - return SM_ACTION_COMPLETE; - } + int is_osd_meta = fsid_is_osd_meta(sm_p->getattr.object_ref.fs_id); + int is_osd_md = fsid_is_osd_md(sm_p->getattr.object_ref.fs_id); if (PINT_smcb_cancelled(smcb)) { @@ -510,6 +494,13 @@ static PINT_sm_action io_inspect_attr( sm_p->getattr.attr.u.meta.dist, sm_p->getattr.attr.mask, sm_p->u.io.io_type); + + + if (is_osd_meta || is_osd_md) + { + js_p->error_code = 0; + } + return(SM_ACTION_COMPLETE); } diff --git a/src/client/sysint/sys-osd-dir.sm b/src/client/sysint/sys-osd-dir.sm index 9b214d8..ab93b9c 100755 --- a/src/client/sysint/sys-osd-dir.sm +++ b/src/client/sysint/sys-osd-dir.sm @@ -232,7 +232,6 @@ nested machine pvfs2_client_osd_dirops_attr4_sm static PINT_sm_action osd_dirops_init( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("osd_dirops_init\n"); struct PINT_client_sm *sm_p; sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); @@ -250,7 +249,6 @@ static PINT_sm_action osd_dirops_init( static PINT_sm_action osd_dirops_attr1_init( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("osd_dirops_attr1_init\n"); job_id_t tmp_id; struct PINT_client_sm *sm_p; @@ -624,7 +622,6 @@ static PINT_sm_action osd_dirops_attr1_failure( static PINT_sm_action osd_dirops_attr4_init( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("osd_dirops_attr4_init\n"); job_id_t tmp_id; struct PINT_client_sm *sm_p; diff --git a/src/client/sysint/sys-osd-io.sm b/src/client/sysint/sys-osd-io.sm index 9abb588..167ec65 100755 --- a/src/client/sysint/sys-osd-io.sm +++ b/src/client/sysint/sys-osd-io.sm @@ -597,6 +597,7 @@ static int osd_io_setup_msgpairs(struct PINT_smcb *smcb, job_status_s *js_p) out: js_p->error_code = ret; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return 1; } @@ -611,7 +612,7 @@ static int osd_io_completion_fn( int index) { struct PINT_smcb *smcb = user_args; - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); struct PINT_client_io_sm *io = &sm_p->u.io; struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; PVFS_size result_size; diff --git a/src/client/sysint/sys-readdirplus.sm b/src/client/sysint/sys-readdirplus.sm index 295b3d4..16585c9 100644 --- a/src/client/sysint/sys-readdirplus.sm +++ b/src/client/sysint/sys-readdirplus.sm @@ -29,6 +29,7 @@ #include "pint-util.h" #include "pvfs2-internal.h" #include "osd-util/osd-util.h" +#define MAXNAMELEN (256UL) enum { NO_WORK = 1, @@ -45,10 +46,6 @@ static int readdirplus_fetch_sizes_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index); -static int readdirplus_fetch_sizes_osd_comp_fn(void *v_p, - struct PVFS_server_resp *resp_p, - int index); - %% machine pvfs2_client_readdirplus_sm @@ -489,7 +486,6 @@ static int list_of_meta_servers(PINT_client_sm *sm_p) static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("readdirplus_fetch_attrs_setup_msgpair\n"); int i, ret; struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_msgpair_state *msg_p = NULL; @@ -538,7 +534,6 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm if (server_is_osd(msg_p->svr_addr)) { - gossip_err("osd\n"); uint64_t oid; struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; @@ -562,7 +557,6 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm js_p->error_code = OSD_MSGPAIR; } else { - gossip_err("not_osd\n"); PINT_SERVREQ_LISTATTR_FILL( msg_p->req, *sm_p->cred_p, @@ -575,6 +569,7 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm js_p->error_code = 0; } } + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -584,7 +579,6 @@ static int readdirplus_fetch_attrs_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("readdirplus_fetch_attrs_comp_fn\n"); PINT_smcb *smcb = v_p; PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); @@ -745,7 +739,6 @@ static int list_of_data_servers(PINT_client_sm *sm_p) static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("readdirplus_fetch_sizes_setup_msgpair\n"); int i, ret; struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_msgpair_state *msg_p; @@ -782,44 +775,56 @@ static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( return SM_ACTION_COMPLETE; } - foreach_msgpair(&sm_p->msgarray_op, msg_p, i) - { + if (is_osd) + { + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->retry_flag = PVFS_MSGPAIR_RETRY; msg_p->svr_addr = sm_p->u.readdirplus.server_addresses[0]; + msg_p->comp_fn = readdirplus_fetch_sizes_comp_fn; + msg_p->handle = sm_p->u.readdirplus.input_handle_array[0].handle; + + struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; + + struct attribute_list attr[sm_p->u.readdirplus.nhandles]; + /* Set attr type, page and number */ + for (i = 0; i < sm_p->u.readdirplus.nhandles; i++) { + if (sm_p->u.readdirplus.nhandles == 1) { + attr[i].type = ATTR_GET; + } else { + attr[i].type = ATTR_GET_MULTI; + } + attr[i].page = 0x1; + attr[i].number = 0x82; /* logical length (not used capacity) */ + attr[i].len = sizeof(uint64_t); + } - if (is_osd) - { - struct osd_command *command = &sm_p->msgarray_op.msgarray[i].osd_command; - struct attribute_list id = { - .type = ATTR_GET, - .page = 0x1, - .number = 0x82, /* logical length (not used capacity) */ - .len = sizeof(uint64_t), - }; - - ret = osd_command_set_get_attributes(command, PVFS_OSD_DATA_PID, - sm_p->u.readdirplus.input_handle_array[i].handle); + ret = osd_command_set_get_member_attributes(command, PVFS_OSD_DATA_PID, COLLECTION_OID_LB); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_member_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } - msg_p->handle = sm_p->u.readdirplus.input_handle_array[i].handle; - msg_p->comp_fn = readdirplus_fetch_sizes_osd_comp_fn; + js_p->error_code = OSD_MSGPAIR; - if (ret) { - osd_error_xerrno(ret, - "%s: osd_command_set_get_attributes failed", - __func__); - js_p->error_code = ret; - return 1; - } - ret = osd_command_attr_build(command, &id, 1); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_attr_build failed", - __func__); - js_p->error_code = ret; - return 1; - } + ret = osd_command_multi_attr_build(command, attr, sm_p->u.readdirplus.nhandles, sm_p->u.readdirplus.nhandles); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } - } else { + } else { + foreach_msgpair(&sm_p->msgarray_op, msg_p, i) + { + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->retry_flag = PVFS_MSGPAIR_RETRY; + msg_p->svr_addr = sm_p->u.readdirplus.server_addresses[0]; msg_p->comp_fn = readdirplus_fetch_sizes_comp_fn; msg_p->handle = PVFS_HANDLE_NULL; PINT_SERVREQ_LISTATTR_FILL( @@ -831,17 +836,10 @@ static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( sm_p->u.readdirplus.handles[i], sm_p->hints); } + /* immediate return. next state jumps to msgpairarray machine */ + js_p->error_code = 0; } - - if(is_osd) - { - js_p->error_code = OSD_MSGPAIR; - } else { - /* immediate return. next state jumps to msgpairarray machine */ - js_p->error_code = 0; - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - } - + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -852,10 +850,25 @@ static int readdirplus_fetch_sizes_comp_fn(void *v_p, { PINT_smcb *smcb = v_p; PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + PVFS_error status; + int64_t size[sm_p->u.readdirplus.nhandles]; + int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); gossip_debug(GOSSIP_LISTATTR_DEBUG, "readdirplus_fetch_sizes_comp_fn called\n"); - assert(resp_p->op == PVFS_SERV_LISTATTR); + + if (is_osd) { + status = osd_errno_from_status( + sm_p->msgarray_op.msgarray[index].osd_command.status); + } else { + assert(resp_p->op == PVFS_SERV_LISTATTR); + status = resp_p->status; + } + + if (status != 0) + { + return status; + } /* Mark all handles in this server range as having failed a stat */ if (sm_p->msgarray_op.msgarray[index].op_status != 0) { @@ -876,87 +889,87 @@ static int readdirplus_fetch_sizes_comp_fn(void *v_p, /* fetch all errors from the servresp structure and copy the object attributes */ int i, handle_index, aux_index; - /* make sure that we get back responses for all handles that we sent out */ - assert(resp_p->u.listattr.nhandles == sm_p->u.readdirplus.handle_count[index]); - for (i = 0; i < sm_p->u.readdirplus.handle_count[index]; i++) - { - get_handle_index(sm_p->u.readdirplus.input_handle_array, - sm_p->u.readdirplus.nhandles, - sm_p->u.readdirplus.handles[index][i], - &handle_index, - &aux_index); - /* Copy any errors */ - sm_p->u.readdirplus.readdirplus_resp->stat_err_array[handle_index] = - resp_p->u.listattr.error[i]; - if (resp_p->u.listattr.error[i] == 0) - { - /* if no errors, stash the object sizes */ - assert(resp_p->u.listattr.attr[i].objtype == PVFS_TYPE_DATAFILE); - sm_p->u.readdirplus.size_array[handle_index][aux_index] = - resp_p->u.listattr.attr[i].u.data.size; + if (is_osd) { + struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; + + int ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", + __func__); + return ret; } - } - } - /* If this is the last server response, check all the status values - * and stash any error codes if any of them failed - */ - if (index == (sm_p->msgarray_op.count - 1)) - { - return PINT_msgarray_status(&sm_p->msgarray_op); - } - return 0; -} - -static int readdirplus_fetch_sizes_osd_comp_fn(void *v_p, - struct PVFS_server_resp *resp_p, - int index) -{ - PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p; - sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - PVFS_error status; - int64_t size; + struct attribute_list *attr = command->attr; + + for (i=0; iu.readdirplus.nhandles; i++) { + if(sm_p->u.readdirplus.nhandles == 1) { + if (command->attr->outlen != sizeof(uint64_t)) { + gossip_err("%s: expecting 8 bytes returned, got %u.\n", __func__, + command->attr->outlen); + return ret; + } + } else { + if (((struct attribute_get_multi_results *)attr[0].val)->outlen[((struct attribute_get_multi_results *)attr[0].val)->numoid-(sm_p->u.readdirplus.nhandles-i)] != sizeof(uint64_t)) { + gossip_err("%s: expecting 8 bytes returned, got %u.\n", __func__, + command->attr->outlen); + return ret; + } + } + } - gossip_debug(GOSSIP_LISTATTR_DEBUG, - "readdirplus_fetch_sizes_osd_comp_fn called\n"); + for (i=0; iu.readdirplus.nhandles; i++) { + if(sm_p->u.readdirplus.nhandles == 1) { + size[i] = get_ntohll(command->attr[i].val); + } else { + size[i] = get_ntohll(((struct attribute_get_multi_results *)attr[0].val)->val[((struct attribute_get_multi_results *)attr[0].val)->numoid-(sm_p->u.readdirplus.nhandles-i)]); + } + } - status = osd_errno_from_status( - sm_p->msgarray_op.msgarray[index].osd_command.status); + for (i = 0; i < sm_p->u.readdirplus.nhandles; i++) + { + if(sm_p->u.readdirplus.nhandles == 1) { + if(sm_p->msgarray_op.msgarray[index].handle == sm_p->u.readdirplus.input_handle_array[i].handle) + { + handle_index = sm_p->u.readdirplus.input_handle_array[0].handle_index; + aux_index = sm_p->u.readdirplus.input_handle_array[0].aux_index; + sm_p->u.readdirplus.size_array[handle_index][aux_index] = size[0]; + } + } else { + //osd_debug("%d\n", ((struct attribute_get_multi_results *)attr[0].val)->oid[((struct attribute_get_multi_results *)attr[0].val)->numoid-(sm_p->u.readdirplus.nhandles-i)]); + osd_debug("%d\n", sm_p->u.readdirplus.input_handle_array[i].handle); + if(((struct attribute_get_multi_results *)attr[0].val)->oid[((struct attribute_get_multi_results *)attr[0].val)->numoid-(sm_p->u.readdirplus.nhandles-i)] == sm_p->u.readdirplus.input_handle_array[i].handle) + { + handle_index = sm_p->u.readdirplus.input_handle_array[i].handle_index; + aux_index = sm_p->u.readdirplus.input_handle_array[i].aux_index; + sm_p->u.readdirplus.size_array[handle_index][aux_index] = size[i]; + } + } + } - if (status != 0) - { - return status; - } else { - /* fetch all errors from the servresp structure and copy the object attributes */ - int j, handle_index, aux_index; - struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; - int ret = osd_command_attr_resolve(command); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", - __func__); - return ret; - } - if (command->attr->outlen != sizeof(uint64_t)) { - gossip_err("%s: expecting 8 bytes returned, got %u.\n", __func__, - command->attr->outlen); - return ret; - } - size = get_ntohll(command->attr->val); - - for (j = 0; j < sm_p->u.readdirplus.nhandles; j++) - { - if(sm_p->msgarray_op.msgarray[index].handle == sm_p->u.readdirplus.input_handle_array[j].handle) + osd_command_attr_free(command); + } else { + /* make sure that we get back responses for all handles that we sent out */ + assert(resp_p->u.listattr.nhandles == sm_p->u.readdirplus.handle_count[index]); + for (i = 0; i < sm_p->u.readdirplus.handle_count[index]; i++) { - handle_index = sm_p->u.readdirplus.input_handle_array[j].handle_index; - aux_index = sm_p->u.readdirplus.input_handle_array[j].aux_index; - sm_p->u.readdirplus.size_array[handle_index][aux_index] = size; + get_handle_index(sm_p->u.readdirplus.input_handle_array, + sm_p->u.readdirplus.nhandles, + sm_p->u.readdirplus.handles[index][i], + &handle_index, + &aux_index); + /* Copy any errors */ + sm_p->u.readdirplus.readdirplus_resp->stat_err_array[handle_index] = + resp_p->u.listattr.error[i]; + if (resp_p->u.listattr.error[i] == 0) + { + /* if no errors, stash the object sizes */ + assert(resp_p->u.listattr.attr[i].objtype == PVFS_TYPE_DATAFILE); + sm_p->u.readdirplus.size_array[handle_index][aux_index] = + resp_p->u.listattr.attr[i].u.data.size; + } } - } - - osd_command_attr_free(command); + } } - /* If this is the last server response, check all the status values * and stash any error codes if any of them failed */ @@ -967,7 +980,6 @@ static int readdirplus_fetch_sizes_osd_comp_fn(void *v_p, return 0; } - static PINT_sm_action readdirplus_msg_failure( struct PINT_smcb *smcb, job_status_s *js_p) { diff --git a/src/client/sysint/sys-setattr.sm b/src/client/sysint/sys-setattr.sm index 2eee002..b369bd5 100644 --- a/src/client/sysint/sys-setattr.sm +++ b/src/client/sysint/sys-setattr.sm @@ -23,12 +23,17 @@ #include "PINT-reqproto-encode.h" #include "pint-util.h" #include "pvfs2-internal.h" +#include "osd-util/osd-util.h" extern job_context_id pint_client_sm_context; static int setattr_msg_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); +enum { + OSD_MSGPAIR = 2001 +}; + %% machine pvfs2_client_setattr_sm @@ -42,10 +47,18 @@ machine pvfs2_client_setattr_sm state setattr_msg_setup_msgpair { run setattr_msg_setup_msgpair; + OSD_MSGPAIR => setattr_msg_xfer_osd_msgpair; success => setattr_msg_xfer_msgpair; default => cleanup; } + state setattr_msg_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => cleanup; + default => setattr_msg_failure; + } + state setattr_msg_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -189,10 +202,23 @@ static int setattr_msg_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { + int status; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); + int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); + gossip_debug(GOSSIP_CLIENT_DEBUG, "setattr_msg_comp_fn\n"); - assert(resp_p->op == PVFS_SERV_SETATTR); - return resp_p->status; + if (is_osd_meta || is_osd_md) { + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + return status; + } else { + assert(resp_p->op == PVFS_SERV_SETATTR); + return resp_p->status; + } } static PINT_sm_action setattr_msg_setup_msgpair( @@ -203,7 +229,10 @@ static PINT_sm_action setattr_msg_setup_msgpair( PINT_sm_msgpair_state *msg_p = NULL; PVFS_ds_type objtype; - js_p->error_code = 0; + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); + int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); + + js_p->error_code = (is_osd_meta || is_osd_md) ? OSD_MSGPAIR : 0; gossip_debug(GOSSIP_CLIENT_DEBUG," setattr: posting setattr req\n"); @@ -213,15 +242,80 @@ static PINT_sm_action setattr_msg_setup_msgpair( objtype = ((sm_p->u.setattr.sys_attr.mask & PVFS_ATTR_SYS_TYPE) ? sm_p->u.setattr.sys_attr.objtype : PVFS_TYPE_NONE); - PINT_SERVREQ_SETATTR_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->object_ref.fs_id, - sm_p->object_ref.handle, - objtype, - sm_p->u.setattr.sys_attr, - 0, - sm_p->hints); + if (is_osd_meta || is_osd_md) { + /* + * Fill in the metadata as attributes of the datafile. A list of + * attributes that PVFS2 currently stores in the MDS can be found here: + * osd/pvfs/src/io/trove/pvfs2-storage:62. + */ + uint64_t oid; + int i, numattrs = 5; + struct attribute_list attr[numattrs]; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + /* Set attr type, page and number */ + for (i = 0; i < numattrs; i++) { + attr[i].type = ATTR_SET; + attr[i].page = PVFS_USEROBJECT_ATTR_PG; + attr[i].number = i; + } + + /* uid */ + attr[0].val = &sm_p->u.setattr.sys_attr.owner; + attr[0].len = sizeof(PVFS_uid); + + /* gid */ + attr[1].val = &sm_p->u.setattr.sys_attr.group; + attr[1].len = sizeof(PVFS_gid); + + /* perms */ + attr[2].val = &sm_p->u.setattr.sys_attr.perms; + attr[2].len = sizeof(PVFS_permissions); + + /* mask */ + sm_p->u.setattr.sys_attr.mask = PVFS_ATTR_COMMON_UID | + PVFS_ATTR_COMMON_GID | + PVFS_ATTR_COMMON_PERM | + PVFS_ATTR_COMMON_ATIME | + PVFS_ATTR_COMMON_CTIME | + PVFS_ATTR_COMMON_MTIME | + PVFS_ATTR_META_DIST | + PVFS_ATTR_META_DFILES | + PVFS_ATTR_COMMON_TYPE; + attr[3].val = &sm_p->u.setattr.sys_attr.mask; + attr[3].len = sizeof(uint32_t); + + /* object type */ + attr[4].val = &sm_p->u.setattr.sys_attr.objtype; + attr[4].len = sizeof(PVFS_ds_type); + + oid = sm_p->object_ref.handle; + ret = osd_command_set_set_attributes(command, is_osd_md ? PVFS_OSD_DATA_PID : PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_set_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, attr, numattrs); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } + } else { + PINT_SERVREQ_SETATTR_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->object_ref.fs_id, + sm_p->object_ref.handle, + objtype, + sm_p->u.setattr.sys_attr, + 0, + sm_p->hints); + } /* clients should not be able to mess with dfile and distribution * information here. Those parameters should only be set at create time.