diff --git a/src/client/sysint/client-state-machine.h b/src/client/sysint/client-state-machine.h index ca85297..f0afc97 100644 --- a/src/client/sysint/client-state-machine.h +++ b/src/client/sysint/client-state-machine.h @@ -793,6 +793,9 @@ extern struct PINT_state_machine_s pvfs2_client_datafile_getattr_sizes_sm; extern struct PINT_state_machine_s pvfs2_client_setattr_sm; extern struct PINT_state_machine_s pvfs2_client_io_sm; extern struct PINT_state_machine_s pvfs2_client_osd_io_sm; +extern struct PINT_state_machine_s pvfs2_client_osd_dirops_sm; +extern struct PINT_state_machine_s pvfs2_client_osd_dirops_attr1_sm; +extern struct PINT_state_machine_s pvfs2_client_osd_dirops_attr4_sm; extern struct PINT_state_machine_s pvfs2_client_small_io_sm; extern struct PINT_state_machine_s pvfs2_client_flush_sm; extern struct PINT_state_machine_s pvfs2_client_sysint_readdir_sm; diff --git a/src/client/sysint/module.mk.in b/src/client/sysint/module.mk.in index 4f606c9..a83139d 100644 --- a/src/client/sysint/module.mk.in +++ b/src/client/sysint/module.mk.in @@ -52,7 +52,8 @@ CLIENT_SMCGEN := \ $(DIR)/mgmt-create-dirent.c \ $(DIR)/mgmt-get-dirdata-handle.c \ $(DIR)/osdsm.c \ - $(DIR)/sys-osd-io.c + $(DIR)/sys-osd-io.c \ + $(DIR)/sys-osd-dir.c # track generated .c files that need to be removed during dist clean, etc. SMCGEN += $(CLIENT_SMCGEN) diff --git a/src/client/sysint/remove.sm b/src/client/sysint/remove.sm index 682b733..11a246f 100644 --- a/src/client/sysint/remove.sm +++ b/src/client/sysint/remove.sm @@ -94,10 +94,18 @@ nested machine pvfs2_client_remove_helper_sm state object_remove_setup_msgpair { run remove_object_remove_setup_msgpair; + OSD_MSGPAIR => object_remove_xfer_osd_msgpair; success => object_remove_xfer_msgpair; default => object_remove_failure; } + state object_remove_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => remove_helper_cleanup; + default => object_remove_failure; + } + state object_remove_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -123,6 +131,7 @@ nested machine pvfs2_client_remove_helper_sm static PINT_sm_action remove_getattr_init( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("remove_getattr_init\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_SM_GETATTR_STATE_FILL( sm_p->getattr, @@ -136,6 +145,7 @@ static PINT_sm_action remove_getattr_init( static PINT_sm_action remove_getattr_analyze_results( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("remove_getattr_analyze_results\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_attr *attr = NULL; attr = &sm_p->getattr.attr; @@ -179,6 +189,7 @@ static PINT_sm_action remove_getattr_analyze_results( static PINT_sm_action remove_datafile_remove_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("remove_datafile_remove_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_attr *attr = NULL; int ret = -PVFS_EINVAL; @@ -244,6 +255,7 @@ static PINT_sm_action remove_datafile_remove_setup_msgpair( static PINT_sm_action remove_object_remove_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("remove_object_remove_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PINT_sm_msgpair_state *msg_p = NULL; @@ -251,22 +263,11 @@ static PINT_sm_action remove_object_remove_setup_msgpair( gossip_debug(GOSSIP_CLIENT_DEBUG, "remove state: object_remove_setup_msgpair\n"); - js_p->error_code = 0; - PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; - PINT_SERVREQ_REMOVE_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->object_ref.fs_id, - sm_p->object_ref.handle, - sm_p->hints); - msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = sm_p->object_ref.handle; - msg_p->retry_flag = PVFS_MSGPAIR_RETRY; - msg_p->comp_fn = NULL; ret = PINT_cached_config_map_to_server( &msg_p->svr_addr, msg_p->handle, msg_p->fs_id); @@ -277,7 +278,38 @@ static PINT_sm_action remove_object_remove_setup_msgpair( js_p->error_code = ret; } - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + if (server_is_osd(msg_p->svr_addr)) { + /* XXX: if object to remove is a directory, first check if + * it is empty, then do the actual remove. And, worse yet, + * this must be done atomically: check for attrs, meaning + * is the dir empty, then if empty, remove. Hairy. + */ + /* Remove the metafile object */ + uint64_t oid = sm_p->object_ref.handle; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + ret = osd_command_set_remove(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_remove failed", + __func__); + js_p->error_code = ret; + return 1; + } + js_p->error_code = OSD_MSGPAIR; + } else { + PINT_SERVREQ_REMOVE_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->object_ref.fs_id, + sm_p->object_ref.handle, + sm_p->hints); + js_p->error_code = 0; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + } + + msg_p->retry_flag = PVFS_MSGPAIR_RETRY; + msg_p->comp_fn = NULL; + return SM_ACTION_COMPLETE; } @@ -316,6 +348,7 @@ static PINT_sm_action remove_object_remove_failure( static PINT_sm_action remove_helper_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("remove_helper_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); gossip_debug(GOSSIP_CLIENT_DEBUG, "remove state: helper cleanup\n"); diff --git a/src/client/sysint/sys-create.sm b/src/client/sysint/sys-create.sm index 3ad12d0..6276f72 100644 --- a/src/client/sysint/sys-create.sm +++ b/src/client/sysint/sys-create.sm @@ -49,13 +49,14 @@ static int create_crdirent_comp_fn( static int create_delete_handles_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); -int osd_create_flag = 1; +int osd_create_flag = 0; /* misc helper functions */ static PINT_dist* get_default_distribution(PVFS_fs_id fs_id); enum { - OSD_MSGPAIR = 2001 + OSD_MSGPAIR = 2001, + OSD_META_MSGPAIR = 2002 }; %% @@ -85,10 +86,18 @@ machine pvfs2_client_create_sm state create_setup_msgpair { run create_create_setup_msgpair; + OSD_MSGPAIR => create_xfer_osd_msgpair; success => create_xfer_msgpair; default => cleanup; } + state create_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => datafiles_setup_msgpair_array; + default => cleanup; + } + state create_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -100,10 +109,18 @@ machine pvfs2_client_create_sm { run create_crdirent_setup_msgpair; OSD_MSGPAIR => datafiles_setup_msgpair_array; + OSD_META_MSGPAIR => crdirent_osd_msgpair; success => crdirent_xfer_msgpair; default => crdirent_failure; } + state crdirent_osd_msgpair + { + jump pvfs2_client_osd_dirops_sm; + success => cleanup; + default => crdirent_failure; + } + state datafiles_setup_msgpair_array { run create_datafiles_setup_msgpair_array; @@ -121,10 +138,18 @@ machine pvfs2_client_create_sm state create_setattr_setup_msgpair { run create_setattr_setup_msgpair; + OSD_MSGPAIR => create_setattr_xfer_osd_msgpair; success => create_setattr_xfer_msgpair; default => cleanup; } + state create_setattr_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => crdirent_setup_msgpair; + default => crdirent_failure; + } + state create_setattr_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -400,34 +425,67 @@ static int create_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { + gossip_err("create_comp_fn\n"); PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + PINT_client_sm *sm_p; + if (smcb->base_frame) { + sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + } else { + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + } + + int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); gossip_debug(GOSSIP_CLIENT_DEBUG, "create_create_comp_fn\n"); - assert(resp_p->op == PVFS_SERV_CREATE); + if (is_osd_meta) { + int ret; + uint64_t oid; + PVFS_error status; + + status = osd_errno_from_status( + sm_p->msgarray_op.msgpair.osd_command.status); + + if (status != 0) { + return status; + } + + /* otherwise, just stash the newly created meta handle */ + ret = osd_command_attr_resolve(&sm_p->msgarray_op.msgpair.osd_command); + if (ret) { + osd_error_xerrno(ret, "%s: attr_resolve failed", __func__); + } + oid = get_ntohll(sm_p->msgarray_op.msgpair.osd_command.attr[0].val); + sm_p->u.create.metafile_handle = oid; + sm_p->u.create.create_resp->ref.handle = oid; + sm_p->u.create.create_resp->ref.fs_id = sm_p->object_ref.fs_id; + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + + } else { + assert(resp_p->op == PVFS_SERV_CREATE); - if (resp_p->status != 0) - { - return resp_p->status; - } + if (resp_p->status != 0) + { + return resp_p->status; + } - /* otherwise, just stash the newly created meta handle */ - sm_p->u.create.metafile_handle = - resp_p->u.create.metafile_handle; - sm_p->u.create.datafile_count = resp_p->u.create.datafile_count; - sm_p->u.create.datafile_handles = malloc( - sizeof(*sm_p->u.create.datafile_handles) * - sm_p->u.create.datafile_count); - if(!sm_p->u.create.datafile_handles) - { - return -PVFS_ENOMEM; + /* otherwise, just stash the newly created meta handle */ + sm_p->u.create.metafile_handle = + resp_p->u.create.metafile_handle; + sm_p->u.create.datafile_count = resp_p->u.create.datafile_count; + sm_p->u.create.datafile_handles = malloc( + sizeof(*sm_p->u.create.datafile_handles) * + sm_p->u.create.datafile_count); + if(!sm_p->u.create.datafile_handles) + { + return -PVFS_ENOMEM; + } + memcpy(sm_p->u.create.datafile_handles, + resp_p->u.create.datafile_handles, + (sizeof(*sm_p->u.create.datafile_handles) * + resp_p->u.create.datafile_count)); + sm_p->u.create.stuffed = resp_p->u.create.stuffed; } - memcpy(sm_p->u.create.datafile_handles, - resp_p->u.create.datafile_handles, - (sizeof(*sm_p->u.create.datafile_handles) * - resp_p->u.create.datafile_count)); - sm_p->u.create.stuffed = resp_p->u.create.stuffed; gossip_debug( GOSSIP_CLIENT_DEBUG, "*** Got newly created handle %llu\n", @@ -523,10 +581,17 @@ static int create_setattr_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { + gossip_err("create_setattr_comp_fn\n"); int res, status; PVFS_object_ref refn; PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + PINT_client_sm *sm_p; + if (smcb->base_frame) + { + sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + } else { + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + } gossip_debug(GOSSIP_CLIENT_DEBUG, "create_setattr_comp_fn\n"); @@ -557,6 +622,7 @@ static int create_crdirent_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { + gossip_err("create_crdirent_comp_fn\n"); gossip_debug(GOSSIP_CLIENT_DEBUG, "create_crdirent_comp_fn\n"); assert(resp_p->op == PVFS_SERV_CRDIRENT); @@ -572,11 +638,15 @@ static PINT_sm_action create_create_setup_msgpair( PINT_sm_msgpair_state *msg_p = NULL; int server_type; + /* + * Check whether we're storing file metadata on a dedicated OSD MDS + * or as attributes of the first datafile. + */ + int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); + gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: " "dspace_create_setup_msgpair\n"); - js_p->error_code = 0; - gossip_debug(GOSSIP_CLIENT_DEBUG," create: posting create req\n"); /* reset the attributes to what got passed in to the sysint call. the retry @@ -611,14 +681,44 @@ static PINT_sm_action create_create_setup_msgpair( &server_type)); } - PINT_SERVREQ_CREATE_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->object_ref.fs_id, - sm_p->u.create.attr, - sm_p->u.create.num_data_files, - sm_p->u.create.layout, - sm_p->hints); + if (is_osd_meta) { + + /* + * Create a zero-length datafile on the metadata OSD. We'll fill in + * the required attributes in create_setattr_setup_msgpair(). + */ + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG, + CCAP_OID, NULL, CCAP_OID_LEN }; + + ret = osd_command_set_create(command, PVFS_OSD_META_PID, 0, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_create failed", + __func__); + js_p->error_code = ret; + return SM_ACTION_COMPLETE; + } + + ret = osd_command_attr_build(command, &attr, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return SM_ACTION_COMPLETE; + } + js_p->error_code = OSD_MSGPAIR; + } else { + PINT_SERVREQ_CREATE_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->object_ref.fs_id, + sm_p->u.create.attr, + sm_p->u.create.num_data_files, + sm_p->u.create.layout, + sm_p->hints); + js_p->error_code = 0; + } msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = meta_handle_extent_array.extent_array[0].first; @@ -630,13 +730,16 @@ static PINT_sm_action create_create_setup_msgpair( msg_p->req.u.create.attr.u.meta.dist_size = PINT_DIST_PACK_SIZE(sm_p->u.create.dist); - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + if (!(is_osd_meta)) { + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + } return SM_ACTION_COMPLETE; } static PINT_sm_action create_datafiles_setup_msgpair_array( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("create_datafiles_setup_msgpair_array\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; struct server_configuration_s *server_config; @@ -707,7 +810,7 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( #define CURRENT_COMMAND_PAGE 0xfffffffeUL #define CURRENT_COMMAND_PAGE_OID 4 - if (is_osd) { + if (is_osd) { struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG, CCAP_OID, NULL, CCAP_OID_LEN }; @@ -747,6 +850,7 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( static PINT_sm_action create_setattr_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("create_setattr_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PINT_sm_msgpair_state *msg_p = NULL; @@ -764,24 +868,110 @@ static PINT_sm_action create_setattr_setup_msgpair( PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; - PINT_SERVREQ_SETATTR_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->object_ref.fs_id, - sm_p->u.create.metafile_handle, - PVFS_TYPE_METAFILE, - sm_p->u.create.attr, - PVFS_ATTR_META_ALL, - sm_p->hints); - - msg_p->req.u.setattr.attr.u.meta.dfile_array = - sm_p->u.create.datafile_handles; - msg_p->req.u.setattr.attr.u.meta.dfile_count = - sm_p->u.create.num_data_files; - msg_p->req.u.setattr.attr.u.meta.dist = - sm_p->u.create.dist; - msg_p->req.u.setattr.attr.u.meta.dist_size = - PINT_DIST_PACK_SIZE(sm_p->u.create.dist); + if (is_osd_meta || is_osd_md) { + /* + * Fill in the metadata as attributes of the datafile. A list of + * attributes that PVFS2 currently stores in the MDS can be found here: + * osd/pvfs/src/io/trove/pvfs2-storage:62. + */ + uint64_t oid; + char *dist_buf; + int i, numattrs = 7; + struct attribute_list attr[numattrs]; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + /* Set attr type, page and number */ + for (i = 0; i < numattrs; i++) { + attr[i].type = ATTR_SET; + attr[i].page = PVFS_USEROBJECT_ATTR_PG; + attr[i].number = i; + } + + /* uid */ + attr[0].val = &sm_p->u.create.attr.owner; + attr[0].len = sizeof(PVFS_uid); + + /* gid */ + attr[1].val = &sm_p->u.create.attr.group; + attr[1].len = sizeof(PVFS_gid); + + /* XXX Default to PVFS_PERM_VALID till we can figure out the umask */ + sm_p->u.create.attr.perms = PVFS_PERM_VALID; + attr[2].val = &sm_p->u.create.attr.perms; + attr[2].len = sizeof(PVFS_permissions); + + /* mask */ + sm_p->u.create.attr.mask = PVFS_ATTR_COMMON_UID | + PVFS_ATTR_COMMON_GID | + PVFS_ATTR_COMMON_PERM | + PVFS_ATTR_COMMON_ATIME | + PVFS_ATTR_COMMON_CTIME | + PVFS_ATTR_COMMON_MTIME | + PVFS_ATTR_META_DIST | + PVFS_ATTR_META_DFILES | + PVFS_ATTR_COMMON_TYPE; + attr[3].val = &sm_p->u.create.attr.mask; + attr[3].len = sizeof(uint32_t); + + /* object type */ + sm_p->u.create.attr.objtype = PVFS_TYPE_METAFILE; + attr[4].val = &sm_p->u.create.attr.objtype; + attr[4].len = sizeof(PVFS_ds_type); + + /* + * Encode the distribution for storage. XXX Where do we free this + * memory? + */ + dist_buf = (char *)malloc(PINT_DIST_PACK_SIZE(sm_p->u.create.dist)); + if (!dist_buf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + PINT_dist_encode(dist_buf, sm_p->u.create.dist); + attr[5].val = dist_buf; + attr[5].len = PINT_DIST_PACK_SIZE(sm_p->u.create.dist); + + /* dfile_array */ + attr[6].val = sm_p->u.create.datafile_handles; + attr[6].len = sizeof(PVFS_handle) * sm_p->u.create.num_data_files; + + oid = sm_p->u.create.metafile_handle; + ret = osd_command_set_set_attributes(command, + is_osd_md ? PVFS_OSD_DATA_PID : PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_set_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, attr, numattrs); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } + } else { + PINT_SERVREQ_SETATTR_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->object_ref.fs_id, + sm_p->u.create.metafile_handle, + PVFS_TYPE_METAFILE, + sm_p->u.create.attr, + PVFS_ATTR_META_ALL, + sm_p->hints); + + msg_p->req.u.setattr.attr.u.meta.dfile_array = + sm_p->u.create.datafile_handles; + msg_p->req.u.setattr.attr.u.meta.dfile_count = + sm_p->u.create.num_data_files; + msg_p->req.u.setattr.attr.u.meta.dist = + sm_p->u.create.dist; + msg_p->req.u.setattr.attr.u.meta.dist_size = + PINT_DIST_PACK_SIZE(sm_p->u.create.dist); + } msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = sm_p->u.create.metafile_handle; @@ -797,13 +987,17 @@ static PINT_sm_action create_setattr_setup_msgpair( js_p->error_code = ret; } - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + if(!(is_osd_meta)) + { + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + } return SM_ACTION_COMPLETE; } static PINT_sm_action create_crdirent_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("create_crdirent_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -1; PINT_sm_msgpair_state *msg_p = NULL; @@ -821,8 +1015,6 @@ static PINT_sm_action create_crdirent_setup_msgpair( gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: crdirent_setup_msgpair\n"); - js_p->error_code = 0; - gossip_debug(GOSSIP_CLIENT_DEBUG, "create: %s: posting crdirent req: parent handle: %llu, " "name: %s, handle: %llu\n", @@ -833,20 +1025,6 @@ static PINT_sm_action create_crdirent_setup_msgpair( PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; - PINT_SERVREQ_CRDIRENT_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->u.create.object_name, - sm_p->u.create.metafile_handle, - sm_p->object_ref.handle, - sm_p->object_ref.fs_id, - sm_p->hints); - - msg_p->fs_id = sm_p->object_ref.fs_id; - msg_p->handle = sm_p->object_ref.handle; - msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; - msg_p->comp_fn = create_crdirent_comp_fn; - ret = PINT_cached_config_map_to_server( &msg_p->svr_addr, sm_p->object_ref.handle, sm_p->object_ref.fs_id); @@ -857,6 +1035,30 @@ static PINT_sm_action create_crdirent_setup_msgpair( js_p->error_code = ret; } + if (server_is_osd(msg_p->svr_addr)) { + /* + * Directory operations for metafile and mdfile. We don't do anything + * here because we'll handle the individual directory operations in + * different states. + */ + js_p->error_code = OSD_META_MSGPAIR; + } else { + PINT_SERVREQ_CRDIRENT_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->u.create.object_name, + sm_p->u.create.metafile_handle, + sm_p->object_ref.handle, + sm_p->object_ref.fs_id, + sm_p->hints); + js_p->error_code = 0; + } + + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = create_crdirent_comp_fn; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; } @@ -880,6 +1082,7 @@ static PINT_sm_action create_crdirent_failure( static PINT_sm_action create_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("create_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_object_ref metafile_ref; PVFS_size tmp_size = 0; diff --git a/src/client/sysint/sys-getattr.sm b/src/client/sysint/sys-getattr.sm index 4e39ef8..6559da8 100644 --- a/src/client/sysint/sys-getattr.sm +++ b/src/client/sysint/sys-getattr.sm @@ -24,6 +24,7 @@ #include "PINT-reqproto-encode.h" #include "pvfs2-internal.h" #include "pvfs2-types-debug.h" +#include "osd-util/osd-util.h" /* pvfs2_client_getattr_sm * @@ -48,7 +49,8 @@ enum { GETATTR_ACACHE_MISS = 1, GETATTR_NEED_DATAFILE_SIZES = 2, - GETATTR_IO_RETRY = 3 + GETATTR_IO_RETRY = 3, + OSD_MSGPAIR = 2001 }; /* completion function prototypes */ @@ -102,10 +104,19 @@ nested machine pvfs2_client_getattr_sm state object_getattr_setup_msgpair { run getattr_object_getattr_setup_msgpair; + OSD_MSGPAIR => object_getattr_xfer_osd_msgpair; success => object_getattr_xfer_msgpair; default => cleanup; } + state object_getattr_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => acache_insert; + GETATTR_NEED_DATAFILE_SIZES => datafile_get_sizes; + default => object_getattr_failure; + } + state object_getattr_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -299,6 +310,7 @@ PVFS_error PVFS_sys_getattr( static PINT_sm_action getattr_acache_lookup( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("getattr_acache_lookup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); uint32_t trimmed_mask = 0; int missing_attrs; @@ -475,6 +487,7 @@ static PINT_sm_action getattr_acache_lookup( static PINT_sm_action getattr_object_getattr_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("getattr_object_getattr_setup_msgpair\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; PVFS_object_ref object_ref; @@ -482,8 +495,6 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( gossip_debug(GOSSIP_CLIENT_DEBUG, "(%p) %s\n", sm_p, __func__); - js_p->error_code = 0; - PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; @@ -492,30 +503,121 @@ static PINT_sm_action getattr_object_getattr_setup_msgpair( assert(object_ref.fs_id != PVFS_FS_ID_NULL); assert(object_ref.handle != PVFS_HANDLE_NULL); - /* setup the msgpair to do a getattr operation */ - PINT_SERVREQ_GETATTR_FILL( - msg_p->req, - *sm_p->cred_p, - object_ref.fs_id, - object_ref.handle, - sm_p->getattr.req_attrmask, - sm_p->hints); - msg_p->fs_id = object_ref.fs_id; msg_p->handle = object_ref.handle; - msg_p->retry_flag = PVFS_MSGPAIR_RETRY; - msg_p->comp_fn = getattr_object_getattr_comp_fn; ret = PINT_cached_config_map_to_server( &msg_p->svr_addr, msg_p->handle, msg_p->fs_id); + if (ret) { gossip_err("Failed to map meta server address\n"); js_p->error_code = ret; } - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + if (server_is_osd(msg_p->svr_addr)) { + uint64_t oid, pid; + int i, numattrs = 10; + struct attribute_list attrl[numattrs]; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + /* Set attr type, page and number */ + for (i = 0; i < numattrs - 3; i++) { + attrl[i].type = ATTR_GET; + attrl[i].page = PVFS_USEROBJECT_ATTR_PG; + attrl[i].number = i; + } + + /* uid, gid & perms */ + attrl[0].len = sizeof(PVFS_uid); + attrl[1].len = sizeof(PVFS_gid); + attrl[2].len = sizeof(PVFS_permissions); + + /* mask, object type & dist */ + attrl[3].len = sizeof(uint32_t); + attrl[4].len = sizeof(PVFS_ds_type); + attrl[5].len = 1024; + + /* dfile_array */ + attrl[6].len = 1024; + + /* ctime, atime & mtime */ + attrl[7].type = ATTR_GET; + attrl[7].page = USER_TMSTMP_PG; + attrl[7].number = UTSAP_CTIME; + attrl[7].len = UTSAP_CTIME_LEN; + + attrl[8].type = ATTR_GET; + attrl[8].page = USER_TMSTMP_PG; + attrl[8].number = UTSAP_DATA_ATIME; + attrl[8].len = UTSAP_DATA_ATIME_LEN; + + attrl[9].type = ATTR_GET; + attrl[9].page = USER_TMSTMP_PG; + attrl[9].number = UTSAP_DATA_MTIME; + attrl[9].len = UTSAP_DATA_MTIME_LEN; + + /* + * XXX: need to know if we're looking at a dir object or + * at a metafile. In the is_osd_md case, for a dir, we must + * do the meta pid. As in all other cases where we do the meta + * pid. This ?: below is just to handle the way we keep metadata + * on the 1st data strip in mdfile. Must override it for + * looking up a directory. + */ + /* + * Could be looking at either a dir object or a metafile. For + * datafile, metafile, both will live in the metadata pid space. + * But for mdfile, dir objcts are in metadata while metafile + * lives on the first data stripe. So query the handle extents + * for the filesystem to see if it is metadata or not. + */ + pid = PVFS_OSD_META_PID; /* datafile, metafile */ + oid = object_ref.handle; + + ret = osd_command_set_get_attributes(command, pid, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return SM_ACTION_COMPLETE; + } + + /* + * XXX Right now, I'll go ahead and retrieve ALL the attributes from + * the OSD MDS. In the future, I'll look at the attrmask for the + * requested attributes. + */ + ret = osd_command_attr_build(command, attrl, numattrs); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return SM_ACTION_COMPLETE; + } + js_p->error_code = OSD_MSGPAIR; + } else { + /* setup the msgpair to do a getattr operation */ + PINT_SERVREQ_GETATTR_FILL( + msg_p->req, + *sm_p->cred_p, + object_ref.fs_id, + object_ref.handle, + sm_p->getattr.req_attrmask, + sm_p->hints); + js_p->error_code = 0; + } + + msg_p->fs_id = object_ref.fs_id; + msg_p->handle = object_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_RETRY; + msg_p->comp_fn = getattr_object_getattr_comp_fn; + + if (!(js_p->error_code)) + { + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + } return SM_ACTION_COMPLETE; } @@ -529,29 +631,69 @@ static int getattr_object_getattr_comp_fn( struct PVFS_server_resp *resp_p, int index) { + gossip_err("getattr_object_getattr_comp_fn\n"); PVFS_object_attr *attr = NULL; + int ret, status; PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); - - assert(resp_p->op == PVFS_SERV_GETATTR); + PINT_client_sm *sm_p; + if (smcb->base_frame) + { + sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + } else { + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + } gossip_debug(GOSSIP_GETATTR_DEBUG, "getattr_object_getattr_comp_fn called\n"); - if (resp_p->status != 0) - { - return resp_p->status; - } + attr = &sm_p->getattr.attr; - /* - * If we've reached the callback for the getattr msgpair tranfer, - * then we can make a copy of the retrieved attribute for later - * caching. - */ - PINT_copy_object_attr(&sm_p->getattr.attr, - &resp_p->u.getattr.attr); + if (server_is_osd(sm_p->msgarray_op.msgpair.svr_addr)) { + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + status = osd_errno_from_status(command->status); + if (status != 0) return status; + + ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", + __func__); + return ret; + } - attr = &sm_p->getattr.attr; + /* memcpy the attributes from osd_command attr to sm_p attr object */ + if (*(PVFS_ds_type*)command->attr[4].val == PVFS_TYPE_DIRECTORY) { + + if (sm_p->getattr.req_attrmask & PVFS_ATTR_DIR_HINT) { + /* we do not provide dir hints, always come back zero, but + * lie that the server provided a hint so that acache works + */ + *(uint32_t *) command->attr[3].val |= PVFS_ATTR_DIR_HINT; + } + + PINT_copy_osd_dir_attr(attr, command); + } else { + PINT_copy_osd_object_attr(attr, command); + } + + sm_p->msgarray_op.msgpair.req.u.getattr.attrmask = attr->mask; + osd_command_attr_free(command); + } else { + assert(resp_p->op == PVFS_SERV_GETATTR); + + if (resp_p->status != 0) + { + return resp_p->status; + } + + /* + * If we've reached the callback for the getattr msgpair tranfer, + * then we can make a copy of the retrieved attribute for later + * caching. + */ + PINT_copy_object_attr(&sm_p->getattr.attr, + &resp_p->u.getattr.attr); + } /* if the ref_type mask is set to a non-zero value (!PVFS_TYPE_NONE) * a -PVFS_error will be triggered if the @@ -1108,11 +1250,6 @@ static PINT_sm_action getattr_datafile_getattr_retry( return SM_ACTION_COMPLETE; } /*end datafile_getattr_retry*/ - - - - - static PINT_sm_action getattr_datafile_getattr_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { @@ -1141,6 +1278,7 @@ static PINT_sm_action getattr_datafile_getattr_cleanup( static PINT_sm_action getattr_acache_insert( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("getattr_acache_insert\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PVFS_size* tmp_size = NULL; @@ -1194,6 +1332,7 @@ static PINT_sm_action getattr_acache_insert( static PINT_sm_action getattr_cleanup( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("getattr_cleanup\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_getattr_state *getattr = &(sm_p->getattr); diff --git a/src/client/sysint/sys-osd-dir.sm b/src/client/sysint/sys-osd-dir.sm new file mode 100755 index 0000000..9b214d8 --- /dev/null +++ b/src/client/sysint/sys-osd-dir.sm @@ -0,0 +1,1280 @@ +/* + * State machine to handle OSD directory operations. + * + * Copyright (C) 2007 OSC PVFS-OSD Team + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include +#include + +#include "client-state-machine.h" +#include "pvfs2-debug.h" +#include "job.h" +#include "gossip.h" +#include "str-utils.h" +#include "pint-cached-config.h" +#include "PINT-reqproto-encode.h" +#include "pint-util.h" +#include "ncache.h" +#include "pvfs2-internal.h" +#include "osd-util/osd-util.h" + +extern job_context_id pint_client_sm_context; + +enum { + OSD_DIROPS_RETRY = 191, + OSD_PVFS_REMOVE = 192, + OSD_PVFS_LOCK_FAILED = 193, + OSD_PVFS_ATOMIC_FAILED = 194 +}; + +/* OSD directory completion function prototypes */ +static int osd_dirops_attr1_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr1_remove_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr4_lock_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr4_remove_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr4_lookup_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr4_insert_remove_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); +static int osd_dirops_attr4_unlock_comp_fn( + void *v_p, struct PVFS_server_resp *resp_p, int index); + +%% + +nested machine pvfs2_client_osd_dirops_sm +{ + state osd_dirops_init + { + run osd_dirops_init; + OSD_DIR_ATTR1 => osd_dirops_attr1; + OSD_DIR_ATTR4 => osd_dirops_attr4; + default => return; + } + + state osd_dirops_attr1 + { + jump pvfs2_client_osd_dirops_attr1_sm; + default => return; + } + + state osd_dirops_attr4 + { + jump pvfs2_client_osd_dirops_attr4_sm; + default => return; + } +} + +nested machine pvfs2_client_osd_dirops_attr1_sm +{ + state osd_dirops_attr1_init + { + run osd_dirops_attr1_init; + OSD_PVFS_REMOVE => osd_dirops_attr1_remove_setup_msgpair; + default => osd_dirops_attr1_setup_msgpair; + } + + state osd_dirops_attr1_remove_setup_msgpair + { + run osd_dirops_attr1_remove_setup_msgpair; + success => osd_dirops_attr1_remove_xfer_msgpair; + default => osd_dirops_attr1_failure; + } + + state osd_dirops_attr1_remove_xfer_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr1_setup_msgpair; + default => osd_dirops_attr1_failure; + } + + state osd_dirops_attr1_setup_msgpair + { + run osd_dirops_attr1_setup_msgpair; + success => osd_dirops_attr1_xfer_msgpair; + default => osd_dirops_attr1_failure; + } + + state osd_dirops_attr1_xfer_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr1_cleanup; + OSD_PVFS_ATOMIC_FAILED => osd_dirops_attr1_init; + default => osd_dirops_attr1_failure; + } + + state osd_dirops_attr1_failure + { + run osd_dirops_attr1_failure; + default => osd_dirops_attr1_cleanup; + } + + state osd_dirops_attr1_cleanup + { + run osd_dirops_attr1_cleanup; + OSD_DIROPS_RETRY => osd_dirops_attr1_init; + default => return; + } +} + +nested machine pvfs2_client_osd_dirops_attr4_sm +{ + state osd_dirops_attr4_init + { + run osd_dirops_attr4_init; + default => osd_dirops_attr4_setup_lock_msgpair; + } + + state osd_dirops_attr4_setup_lock_msgpair + { + run osd_dirops_attr4_setup_lock_msgpair; + success => osd_dirops_attr4_xfer_lock_msgpair; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_xfer_lock_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + OSD_PVFS_REMOVE => osd_dirops_attr4_remove_setup_msgpair; + OSD_PVFS_LOCK_FAILED => osd_dirops_attr4_init; + success => osd_dirops_attr4_setup_lookup_msgpair; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_remove_setup_msgpair + { + run osd_dirops_attr4_remove_setup_msgpair; + success => osd_dirops_attr4_remove_xfer_msgpair; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_remove_xfer_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr4_setup_lookup_msgpair; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_setup_lookup_msgpair + { + run osd_dirops_attr4_setup_lookup_msgpair; + success => osd_dirops_attr4_xfer_lookup_msgpair; + default => osd_dirops_attr4_setup_unlock_msgpair; + } + + state osd_dirops_attr4_xfer_lookup_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr4_setup_insert_remove_msgpair; + default => osd_dirops_attr4_setup_unlock_msgpair; + } + + state osd_dirops_attr4_setup_insert_remove_msgpair + { + run osd_dirops_attr4_setup_insert_remove_msgpair; + success => osd_dirops_attr4_xfer_insert_remove_msgpair; + default => osd_dirops_attr4_setup_unlock_msgpair; + } + + state osd_dirops_attr4_xfer_insert_remove_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr4_setup_unlock_msgpair; + default => osd_dirops_attr4_setup_unlock_msgpair; + } + + state osd_dirops_attr4_setup_unlock_msgpair + { + run osd_dirops_attr4_setup_unlock_msgpair; + success => osd_dirops_attr4_xfer_unlock_msgpair; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_xfer_unlock_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => osd_dirops_attr4_cleanup; + default => osd_dirops_attr4_failure; + } + + state osd_dirops_attr4_failure + { + run osd_dirops_attr4_failure; + default => osd_dirops_attr4_cleanup; + } + + state osd_dirops_attr4_cleanup + { + run osd_dirops_attr4_cleanup; + OSD_DIROPS_RETRY => osd_dirops_attr4_init; + default => return; + } +} + +%% + +static PINT_sm_action osd_dirops_init( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + gossip_err("osd_dirops_init\n"); + struct PINT_client_sm *sm_p; + + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops state: init\n"); + + if (smcb->op == PVFS_SYS_REMOVE) { + js_p->error_code = fsid_dir_switch(sm_p->parent_ref.fs_id); + } else { + js_p->error_code = fsid_dir_switch(sm_p->object_ref.fs_id); + } + + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr1_init( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + gossip_err("osd_dirops_attr1_init\n"); + job_id_t tmp_id; + struct PINT_client_sm *sm_p; + + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr1 state: init\n"); + + js_p->error_code = (smcb->op == PVFS_SYS_REMOVE) ? OSD_PVFS_REMOVE : 0; + + if (js_p->error_code == OSD_DIROPS_RETRY) { + return job_req_sched_post_timer( + sm_p->msgarray_op.params.retry_delay, smcb, 0, js_p, &tmp_id, + pint_client_sm_context); + } + + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr1_remove_setup_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + char *object_name = NULL; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct attribute_list attr; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr1: osd_dirops_attr1_remove_setup_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + oid = sm_p->parent_ref.handle; + ret = osd_command_set_get_attributes(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + object_name = sm_p->u.remove.object_name; + attr.type = ATTR_GET; + attr.page = PVFS_USEROBJECT_DIR_PG; + attr.number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr.len = 1024; + + ret = osd_command_attr_build(command, &attr, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); + js_p->error_code = ret; + return 1; + } + + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr1_remove_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr1_remove_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret; + PVFS_error status = 0; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr1_remove_comp_fn\n"); + + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (status != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_get_attributes failed %d\n", index); + return status; + } + + ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", __func__); + return ret; + } + + if (command->attr->outlen == 0) { + status = -ENOENT; + } else { + sm_p->object_ref.handle = get_ntohll(command->attr->val); + sm_p->object_ref.fs_id = sm_p->parent_ref.fs_id; + } + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + + return status; +} + +static PINT_sm_action osd_dirops_attr1_setup_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + uint8_t *dirent = NULL; + char *object_name = NULL; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct attribute_list attr[3]; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr1: osd_dirops_attr1_setup_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + if (smcb->op == PVFS_SYS_CREATE) { + /* insert dirent */ + attr[1].len = 8 + strlen(sm_p->u.create.object_name) + 1; + dirent = malloc(attr[1].len); + if (!dirent) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + set_htonll(dirent, sm_p->u.create.metafile_handle); + strcpy((char *)&dirent[8], sm_p->u.create.object_name); + + /* For a create operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.create.object_name; + + /* cmp */ + attr[0].type = ATTR_SET; + attr[0].page = PVFS_USEROBJECT_DIR_PG; + attr[0].number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr[0].val = NULL; + attr[0].len = 0; + + /* swp */ + attr[1].type = ATTR_SET; + attr[1].page = PVFS_USEROBJECT_DIR_PG; + attr[1].number = attr[0].number; + attr[1].val = dirent; + + /* result */ + /* XXX: this should not be NULL. In the case that the attr already + * existed (like there already was a file /pvfs/x), then the CAS + * will return that original value, not empty. Sufficient to + * grab the first 8 bytes of the result, though, to make sure + * that the handles are the same, like the lock 0/1 case. + */ + attr[2].type = ATTR_RESULT; + attr[2].page = PVFS_USEROBJECT_DIR_PG; + attr[2].number = attr[0].number; + attr[2].val = NULL; + attr[2].len = 0; + } else if (smcb->op == PVFS_SYS_MKDIR) { + /* insert dirent */ + attr[1].len = 8 + strlen(sm_p->u.mkdir.object_name) + 1; + dirent = malloc(attr[1].len); + if (!dirent) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + set_htonll(dirent, sm_p->u.mkdir.metafile_handle); + strcpy((char *)&dirent[8], sm_p->u.mkdir.object_name); + + /* For a mkdir operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.mkdir.object_name; + + /* cmp */ + attr[0].type = ATTR_SET; + attr[0].page = PVFS_USEROBJECT_DIR_PG; + attr[0].number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr[0].val = NULL; + attr[0].len = 0; + + /* swp */ + attr[1].type = ATTR_SET; + attr[1].page = PVFS_USEROBJECT_DIR_PG; + attr[1].number = attr[0].number; + attr[1].val = dirent; + + /* result */ + attr[2].type = ATTR_RESULT; + attr[2].page = PVFS_USEROBJECT_DIR_PG; + attr[2].number = attr[0].number; + attr[2].val = NULL; + attr[2].len = 0; + } else if (smcb->op == PVFS_SYS_REMOVE) { + /* For a remove operation, parent_ref refers to the dir. object */ + oid = sm_p->parent_ref.handle; + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + object_name = sm_p->u.remove.object_name; + + /* cmp */ + attr[0].type = ATTR_SET; + attr[0].page = PVFS_USEROBJECT_DIR_PG; + attr[0].number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr[0].len = 8 + strlen(sm_p->u.remove.object_name) + 1; + dirent = malloc(attr[0].len); + if (!dirent) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + set_htonll(dirent, sm_p->object_ref.handle); + strcpy((char *)&dirent[8], sm_p->u.remove.object_name); + attr[0].val = dirent; + + /* swp */ + attr[1].type = ATTR_SET; + attr[1].page = PVFS_USEROBJECT_DIR_PG; + attr[1].number = attr[0].number; + attr[1].val = NULL; + attr[1].len = 0; + + /* result */ + attr[2].type = ATTR_RESULT; + attr[2].page = PVFS_USEROBJECT_DIR_PG; + attr[2].number = attr[0].number; + attr[2].val = dirent; + attr[2].len = attr[0].len; + } else { + return 1; + } + + ret = osd_command_set_gen_cas(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_gen_cas failed", __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, attr, 3); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); + js_p->error_code = ret; + return 1; + } + + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr1_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr1_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret = 0; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr1_comp_fn\n"); + + ret = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (ret != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_gen_cas failed %d\n", index); + goto out; + } + + ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", __func__); + goto out; + } + + if (smcb->op == PVFS_SYS_CREATE || smcb->op == PVFS_SYS_MKDIR) { + /* insert dirent operation */ + if (command->attr[2].val == NULL) { + /* successful insert operation */ + ret = 0; + goto out; + } else { + /* + * insert operation failed. either the name is already being + * used or we have a hash collision. + */ + gossip_debug(GOSSIP_CLIENT_DEBUG, "Atomic CAS: insert failed\n"); + ret = 1; + goto out; + } + } else { + /* remove dirent operation */ + assert(command->attr[0].val && command->attr[2].val); + + if (get_ntohll(command->attr[0].val) == + get_ntohll(command->attr[2].val)) { + /* successful remove operation */ + ret = 0; + goto out; + } else { + /* remove operation failed */ + gossip_debug(GOSSIP_CLIENT_DEBUG, "Atomic CAS: remove failed\n"); + ret = 1; + goto out; + } + } + +out: + /* free data alloced for the create, if this was an insert */ + if (sm_p->msgarray_op.msgpair.osd_command.attr->len) { + free(sm_p->msgarray_op.msgpair.osd_command.attr->val); + } + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + + return ret; +} + +static PINT_sm_action osd_dirops_attr1_cleanup( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + struct PINT_client_sm *sm_p; + + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr1 state: cleanup\n"); + + /* cleanup code goes here */ + + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr1_failure( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + sm_p->u.mkdir.stored_error_code = js_p->error_code; + + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr1 state: failure\n"); + + /* failure code goes here */ + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr4_init( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + gossip_err("osd_dirops_attr4_init\n"); + job_id_t tmp_id; + struct PINT_client_sm *sm_p; + + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4 state: init\n"); + + js_p->error_code = 0; + + if (js_p->error_code == OSD_DIROPS_RETRY) { + return job_req_sched_post_timer( + sm_p->msgarray_op.params.retry_delay, smcb, 0, js_p, &tmp_id, + pint_client_sm_context); + } + + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr4_setup_lock_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + char *object_name = NULL; + uint64_t *outbuf, *inbuf; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4: osd_dirops_attr4_setup_lock_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + /* + * Lock directory. A directory is represented by a single object. The + * directory entries are attributes on that object. object_ref or + * parent_ref here is the parent directory (i.e. OSD object) that will own + * the newly created entry (or that does own the one to be removed). We + * lock entries in it. + */ + if (smcb->op == PVFS_SYS_CREATE) { + /* For a create operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.create.object_name; + } else if (smcb->op == PVFS_SYS_MKDIR) { + /* For a mkdir operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.mkdir.object_name; + } else if (smcb->op == PVFS_SYS_REMOVE) { + /* For a remove operation, parent_ref refers to the dir. object */ + oid = sm_p->parent_ref.handle; + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + object_name = sm_p->u.remove.object_name; + } else { + return 1; + } + + ret = osd_command_set_cas(command, PVFS_OSD_META_PID, oid, 8, 0); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_cas failed", __func__); + js_p->error_code = ret; + return 1; + } + + command->outlen = 2 * sizeof(uint64_t); + outbuf = malloc(command->outlen); + if (!outbuf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + memset(outbuf, 0, command->outlen); + command->outdata = outbuf; + + command->inlen_alloc = sizeof(uint64_t); + inbuf = malloc(command->inlen_alloc); + if (!inbuf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + memset(inbuf, 0, command->inlen_alloc); + command->indata = inbuf; + + set_htonll((uint8_t *)&outbuf[0], 0); /* cmp */ + set_htonll((uint8_t *)&outbuf[1], 1); /* swp */ + + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr4_lock_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr4_lock_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret = 0; + PVFS_error status = 0; + PINT_smcb *smcb = v_p; + uint8_t *inbuf, *outbuf; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4_lock_comp_fn\n"); + + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (status != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_cas failed %d\n", index); + return status; + } + + /* osd_command.indata should contain the previous value of the lock */ + inbuf = (uint8_t *)sm_p->msgarray_op.msgpair.osd_command.indata; + outbuf = (uint8_t *)sm_p->msgarray_op.msgpair.osd_command.outdata; + assert(inbuf); + assert(outbuf); + + if (get_ntohll(&inbuf[0]) != get_ntohll(&outbuf[0])) { + /* lock failed */ + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4_lock_comp_fn: lock failed\n"); + ret = OSD_PVFS_LOCK_FAILED; + goto out; + } + + /* In case its a REMOVE operation we need to jump to a new state */ + if (smcb->op == PVFS_SYS_REMOVE) { + ret = OSD_PVFS_REMOVE; + } + +out: + /* free the 16 bytes alloced for the CAS operation above */ + free((void *)sm_p->msgarray_op.msgpair.osd_command.indata); + free((void *)sm_p->msgarray_op.msgpair.osd_command.outdata); + + return ret; +} + +static PINT_sm_action osd_dirops_attr4_remove_setup_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + char *object_name = NULL; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct attribute_list attr; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4: osd_dirops_attr4_remove_setup_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + oid = sm_p->parent_ref.handle; + ret = osd_command_set_get_attributes(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + object_name = sm_p->u.remove.object_name; + attr.type = ATTR_GET; + attr.page = PVFS_USEROBJECT_DIR_PG; + attr.number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr.len = 1024; + + ret = osd_command_attr_build(command, &attr, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); + js_p->error_code = ret; + return 1; + } + + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr4_remove_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr4_remove_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret; + PVFS_error status = 0; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4_remove_comp_fn\n"); + + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (status != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_get_attributes failed %d\n", index); + return status; + } + + ret = osd_command_attr_resolve(command); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_resolve failed", __func__); + return ret; + } + + sm_p->object_ref.handle = get_ntohll(command->attr->val); + sm_p->object_ref.fs_id = sm_p->parent_ref.fs_id; + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + + return status; +} + +static PINT_sm_action osd_dirops_attr4_setup_lookup_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + char *object_name = NULL; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct attribute_list attr; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4: osd_dirops_attr4_setup_lookup_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + if (smcb->op == PVFS_SYS_CREATE) { + /* For a create operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.create.object_name; + } else if (smcb->op == PVFS_SYS_MKDIR) { + /* For a mkdir operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.mkdir.object_name; + } else if (smcb->op == PVFS_SYS_REMOVE) { + /* For a remove operation, parent_ref refers to the dir. object */ + oid = sm_p->parent_ref.handle; + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + object_name = sm_p->u.remove.object_name; + } else { + return 1; + } + + ret = osd_command_set_get_attributes(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + /* Perform lookup */ + attr.type = ATTR_GET; + attr.page = PVFS_USEROBJECT_DIR_PG; + attr.number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + attr.len = 1024; + + ret = osd_command_attr_build(command, &attr, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); + js_p->error_code = ret; + return 1; + } + + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr4_lookup_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr4_lookup_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret = 0; + PVFS_error status = 0; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4_lookup_comp_fn\n"); + + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (status != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_get_attributes failed %d\n", index); + return status; + } + + ret = osd_command_attr_resolve(&sm_p->msgarray_op.msgpair.osd_command); + if (ret) { + osd_error_xerrno(ret, "%s: attr_resolve failed", __func__); + goto out; + } + + ret = 0; + + /* + * If the operation is either PVFS_SYS_CREATE or PVFS_SYS_MKDIR and the + * outlen contains a positive value, then the dirent already exists: + * return 1. + */ + if ((smcb->op == PVFS_SYS_CREATE || smcb->op == PVFS_SYS_MKDIR) && + sm_p->msgarray_op.msgpair.osd_command.attr->outlen) { + ret = 1; + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4_lookup_comp_fn: dirent exists\n"); + } + +out: + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + return ret; +} + +static PINT_sm_action osd_dirops_attr4_setup_insert_remove_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = -1; + uint64_t oid; + uint8_t *dirent = NULL; + char *object_name = NULL; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + struct attribute_list attr; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4: osd_dirops_attr4_setup_insert_remove_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + /* Insert dirent as attribute of directory object */ + attr.type = ATTR_SET; + attr.page = PVFS_USEROBJECT_DIR_PG; + + if (smcb->op == PVFS_SYS_CREATE) { + /* insert dirent */ + attr.len = 8 + strlen(sm_p->u.create.object_name) + 1; + dirent = malloc(attr.len); + if (!dirent) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + set_htonll(dirent, sm_p->u.create.metafile_handle); + strcpy((char *)&dirent[8], sm_p->u.create.object_name); + attr.val = dirent; + + /* For a create operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.create.object_name; + } else if (smcb->op == PVFS_SYS_MKDIR) { + /* insert dirent */ + attr.len = 8 + strlen(sm_p->u.mkdir.object_name) + 1; + dirent = malloc(attr.len); + if (!dirent) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + set_htonll(dirent, sm_p->u.mkdir.metafile_handle); + strcpy((char *)&dirent[8], sm_p->u.mkdir.object_name); + attr.val = dirent; + + /* For a mkdir operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + object_name = sm_p->u.mkdir.object_name; + } else if (smcb->op == PVFS_SYS_REMOVE) { + /* remove dirent */ + attr.len = 0; + attr.val = NULL; + + /* For a remove operation, parent_ref refers to the dir. object */ + oid = sm_p->parent_ref.handle; + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + object_name = sm_p->u.remove.object_name; + } else { + return 1; + } + + attr.number = jenkins_one_at_a_time_hash((uint8_t *)object_name, + strlen(object_name) + 1); + + ret = osd_command_set_set_attributes(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_set_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, &attr, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", __func__); + js_p->error_code = ret; + return 1; + } + + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr4_insert_remove_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr4_insert_remove_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + PVFS_error status = 0; + PINT_smcb *smcb = v_p; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + gossip_debug(GOSSIP_CLIENT_DEBUG, "%s\n", __func__); + + status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (status != 0) + gossip_debug(GOSSIP_CLIENT_DEBUG, + "%s: failed to insert/remove dirent %d\n", + __func__, index); + + /* free data alloced for the create, if this was an insert */ + if (sm_p->msgarray_op.msgpair.osd_command.attr->len) + free(sm_p->msgarray_op.msgpair.osd_command.attr->val); + + osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); + + return status; +} + +static PINT_sm_action osd_dirops_attr4_setup_unlock_msgpair( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + int ret = 0; + uint64_t oid; + uint64_t *outbuf, *inbuf; + PINT_sm_msgpair_state *msg_p = NULL; + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4: osd_dirops_attr4_setup_unlock_msgpair\n"); + + js_p->error_code = 0; + + PINT_msgpair_init(&sm_p->msgarray_op); + msg_p = &sm_p->msgarray_op.msgpair; + + /* + * Unlock dir. oid should correspond to the directory we're trying to + * unlock. + */ + if (smcb->op == PVFS_SYS_CREATE || smcb->op == PVFS_SYS_MKDIR) { + /* For a create operation, object_ref refers to the dir. object */ + oid = sm_p->object_ref.handle; + msg_p->fs_id = sm_p->object_ref.fs_id; + msg_p->handle = sm_p->object_ref.handle; + } else if (smcb->op == PVFS_SYS_REMOVE) { + /* For a remove operation, parent_ref refers to the dir. object */ + oid = sm_p->parent_ref.handle; + msg_p->fs_id = sm_p->parent_ref.fs_id; + msg_p->handle = sm_p->parent_ref.handle; + } else { + return 1; + } + + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + ret = osd_command_set_cas(command, PVFS_OSD_META_PID, oid, 8, 0); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_cas failed", __func__); + js_p->error_code = ret; + return 1; + } + + command->outlen = 2 * sizeof(uint64_t); + outbuf = malloc(command->outlen); + if (!outbuf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + memset(outbuf, 0, command->outlen); + command->outdata = outbuf; + + command->inlen_alloc = sizeof(uint64_t); + inbuf = malloc(command->inlen_alloc); + if (!inbuf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + + memset(inbuf, 0, command->inlen_alloc); + command->indata = inbuf; + + set_htonll((uint8_t *)&outbuf[0], 1); /* cmp */ + set_htonll((uint8_t *)&outbuf[1], 0); /* swp */ + + msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; + msg_p->comp_fn = osd_dirops_attr4_unlock_comp_fn; + + ret = PINT_cached_config_map_to_server(&msg_p->svr_addr, msg_p->handle, + msg_p->fs_id); + if (ret) { + gossip_err("Failed to map meta server address\n"); + js_p->error_code = ret; + } + + return SM_ACTION_COMPLETE; +} + +static int osd_dirops_attr4_unlock_comp_fn(void *v_p, + struct PVFS_server_resp *resp_p, + int index) +{ + int ret = 0; + PINT_smcb *smcb = v_p; + uint8_t *inbuf, *outbuf; + PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + gossip_debug(GOSSIP_CLIENT_DEBUG, "%s\n", __func__); + + ret = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); + if (ret != 0) { + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_command_set_cas failed %d\n", index); + } + + /* osd_command.indata should contain the previous value of the lock */ + inbuf = (uint8_t *)sm_p->msgarray_op.msgpair.osd_command.indata; + outbuf = (uint8_t *)sm_p->msgarray_op.msgpair.osd_command.outdata; + assert(inbuf); + assert(outbuf); + + if (get_ntohll(&inbuf[0]) == get_ntohll(&outbuf[0])) { + /* unlock successful */ + ret = 0; + goto out; + } else { + /* unlock failed */ + gossip_debug(GOSSIP_CLIENT_DEBUG, + "osd_dirops_attr4_unlock_comp_fn: unlock failed\n"); + ret = 1; + goto out; + } + +out: + /* free the 16 bytes alloced for the CAS operation above */ + free((void *)sm_p->msgarray_op.msgpair.osd_command.indata); + free((void *)sm_p->msgarray_op.msgpair.osd_command.outdata); + + return ret; +} + +static PINT_sm_action osd_dirops_attr4_cleanup( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + struct PINT_client_sm *sm_p; + + sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4 state: cleanup\n"); + + /* cleanup code goes here */ + + return SM_ACTION_COMPLETE; +} + +static PINT_sm_action osd_dirops_attr4_failure( + struct PINT_smcb *smcb, job_status_s *js_p) +{ + struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + sm_p->u.mkdir.stored_error_code = js_p->error_code; + + gossip_debug(GOSSIP_CLIENT_DEBUG, "osd_dirops_attr4 state: failure\n"); + + /* failure code goes here */ + return SM_ACTION_COMPLETE; +} + +/* + * Local variables: + * mode: c + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ft=c ts=8 sts=4 sw=4 expandtab + */ diff --git a/src/client/sysint/sys-readdirplus.sm b/src/client/sysint/sys-readdirplus.sm index b2aee65..295b3d4 100644 --- a/src/client/sysint/sys-readdirplus.sm +++ b/src/client/sysint/sys-readdirplus.sm @@ -63,11 +63,19 @@ machine pvfs2_client_readdirplus_sm state readdirplus_fetch_attrs_setup_msgpair { run readdirplus_fetch_attrs_setup_msgpair; + OSD_MSGPAIR => readdirplus_fetch_attrs_xfer_osd_msgpair; NO_WORK => cleanup; success => readdirplus_fetch_attrs_xfer_msgpair; default => readdirplus_msg_failure; } + state readdirplus_fetch_attrs_xfer_osd_msgpair + { + jump pvfs2_osd_msgpairarray_sm; + success => readdirplus_fetch_sizes_setup_msgpair; + default => readdirplus_msg_failure; + } + state readdirplus_fetch_attrs_xfer_msgpair { jump pvfs2_msgpairarray_sm; @@ -481,6 +489,7 @@ static int list_of_meta_servers(PINT_client_sm *sm_p) static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("readdirplus_fetch_attrs_setup_msgpair\n"); int i, ret; struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_msgpair_state *msg_p = NULL; @@ -521,24 +530,53 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm foreach_msgpair(&sm_p->msgarray_op, msg_p, i) { - PINT_SERVREQ_LISTATTR_FILL( - msg_p->req, - *sm_p->cred_p, - sm_p->object_ref.fs_id, - sm_p->u.readdirplus.attrmask, - sm_p->u.readdirplus.handle_count[i], - sm_p->u.readdirplus.handles[i], - sm_p->hints); msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = PVFS_HANDLE_NULL; msg_p->retry_flag = PVFS_MSGPAIR_RETRY; msg_p->comp_fn = readdirplus_fetch_attrs_comp_fn; msg_p->svr_addr = sm_p->u.readdirplus.server_addresses[i]; + + if (server_is_osd(msg_p->svr_addr)) + { + gossip_err("osd\n"); + uint64_t oid; + struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; + + /* Retrieve all the directory entries */ + oid = sm_p->object_ref.handle; + ret = osd_command_set_get_attributes(command, PVFS_OSD_META_PID, oid); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_get_attributes failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_all_build(command, PVFS_USEROBJECT_DIR_PG); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_all_build failed", + __func__); + js_p->error_code = ret; + return 1; + } + js_p->error_code = OSD_MSGPAIR; + + } else { + gossip_err("not_osd\n"); + PINT_SERVREQ_LISTATTR_FILL( + msg_p->req, + *sm_p->cred_p, + sm_p->object_ref.fs_id, + sm_p->u.readdirplus.attrmask, + sm_p->u.readdirplus.handle_count[i], + sm_p->u.readdirplus.handles[i], + sm_p->hints); + /* immediate return. next state jumps to msgpairarray machine */ + js_p->error_code = 0; + } } - /* immediate return. next state jumps to msgpairarray machine */ - js_p->error_code = 0; - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - return SM_ACTION_COMPLETE; + PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); + return SM_ACTION_COMPLETE; } /* Phase 1 completion callback */ @@ -546,6 +584,7 @@ static int readdirplus_fetch_attrs_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { + gossip_err("readdirplus_fetch_attrs_comp_fn\n"); PINT_smcb *smcb = v_p; PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); @@ -706,6 +745,7 @@ static int list_of_data_servers(PINT_client_sm *sm_p) static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( struct PINT_smcb *smcb, job_status_s *js_p) { + gossip_err("readdirplus_fetch_sizes_setup_msgpair\n"); int i, ret; struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); PINT_sm_msgpair_state *msg_p; @@ -795,7 +835,7 @@ static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( if(is_osd) { - js_p->error_code = OSD_MSGPAIR; + js_p->error_code = OSD_MSGPAIR; } else { /* immediate return. next state jumps to msgpairarray machine */ js_p->error_code = 0; @@ -889,7 +929,7 @@ static int readdirplus_fetch_sizes_osd_comp_fn(void *v_p, return status; } else { /* fetch all errors from the servresp structure and copy the object attributes */ - int i, j, handle_index, aux_index; + int j, handle_index, aux_index; struct osd_command *command = &sm_p->msgarray_op.msgarray[index].osd_command; int ret = osd_command_attr_resolve(command); if (ret) {