From 581b7e897ea9a356367e3e075fa279a526782207 Mon Sep 17 00:00:00 2001 From: cek10006 Date: Sun, 22 Jul 2012 02:50:30 -0400 Subject: [PATCH] datafile is done, mdfile is almost done Details: * All functions work for -datafile case for both post create and member attr options. * For -mdfile, readdirplus do not work yet; a msgpair needs to be deployed to osd and pvfs simultaneously. * sys-osd-dir.sm can be fixed for this purpose --- config.save | 22 +- src/apps/admin/pvfs2-chmod.c | 4 +- src/client/sysint/remove.sm | 16 +- src/client/sysint/sys-create.sm | 341 +++++++-------------------- src/client/sysint/sys-io.sm | 12 +- src/client/sysint/sys-mkdir.sm | 1 + src/client/sysint/sys-osd-io.sm | 76 ++++-- src/client/sysint/sys-readdirplus.sm | 6 +- src/client/sysint/sys-setattr.sm | 5 +- src/io/description/pint-request.c | 3 +- src/server/create.sm | 27 ++- src/server/final-response.sm | 1 - src/server/get-attr.sm | 1 + src/server/mkdir.sm | 2 +- 14 files changed, 210 insertions(+), 307 deletions(-) diff --git a/config.save b/config.save index c8eb321..dc18b21 100755 --- a/config.save +++ b/config.save @@ -525,17 +525,17 @@ s,@ECHO_C@,,;t t s,@ECHO_N@,-n,;t t s,@ECHO_T@,,;t t s,@LIBS@, -lcrypto -lssl -ldl,;t t -s,@PVFS2_VERSION@,2.8.6-orangefs-2012-07-13-043849,;t t +s,@PVFS2_VERSION@,2.8.6-orangefs-2012-07-19-212817,;t t s,@PVFS2_VERSION_MAJOR@,2,;t t s,@PVFS2_VERSION_MINOR@,8,;t t s,@PVFS2_VERSION_SUB@,6,;t t -s,@build@,i686-pc-linux-gnu,;t t -s,@build_cpu@,i686,;t t -s,@build_vendor@,pc,;t t +s,@build@,x86_64-unknown-linux-gnu,;t t +s,@build_cpu@,x86_64,;t t +s,@build_vendor@,unknown,;t t s,@build_os@,linux-gnu,;t t -s,@host@,i686-pc-linux-gnu,;t t -s,@host_cpu@,i686,;t t -s,@host_vendor@,pc,;t t +s,@host@,x86_64-unknown-linux-gnu,;t t +s,@host_cpu@,x86_64,;t t +s,@host_vendor@,unknown,;t t s,@host_os@,linux-gnu,;t t s,@CC@,gcc,;t t s,@CFLAGS@, -g -O2,;t t @@ -983,14 +983,14 @@ CEOF t clr : clr ${ac_dA}HAVE_ARPA_INET_H${ac_dB}HAVE_ARPA_INET_H${ac_dC}1${ac_dD} -${ac_dA}SIZEOF_LONG_INT${ac_dB}SIZEOF_LONG_INT${ac_dC}4${ac_dD} +${ac_dA}SIZEOF_LONG_INT${ac_dB}SIZEOF_LONG_INT${ac_dC}8${ac_dD} ${ac_dA}WITH_OPENSSL${ac_dB}WITH_OPENSSL${ac_dC}1${ac_dD} ${ac_dA}HAVE_OPENSSL_EVP_H${ac_dB}HAVE_OPENSSL_EVP_H${ac_dC}1${ac_dD} ${ac_dA}HAVE_OPENSSL_CRYPTO_H${ac_dB}HAVE_OPENSSL_CRYPTO_H${ac_dC}1${ac_dD} ${ac_dA}HAVE_OPENSSL_SHA_H${ac_dB}HAVE_OPENSSL_SHA_H${ac_dC}1${ac_dD} ${ac_dA}HAVE_AIOCB_ERROR_CODE${ac_dB}HAVE_AIOCB_ERROR_CODE${ac_dC}1${ac_dD} ${ac_dA}HAVE_AIOCB_RETURN_VALUE${ac_dB}HAVE_AIOCB_RETURN_VALUE${ac_dC}1${ac_dD} -${ac_dA}SIZEOF_VOID_P${ac_dB}SIZEOF_VOID_P${ac_dC}4${ac_dD} +${ac_dA}SIZEOF_VOID_P${ac_dB}SIZEOF_VOID_P${ac_dC}8${ac_dD} ${ac_dA}HAVE_SYS_EPOLL_H${ac_dB}HAVE_SYS_EPOLL_H${ac_dC}1${ac_dD} ${ac_dA}PVFS_USRINT_BUILD${ac_dB}PVFS_USRINT_BUILD${ac_dC}1${ac_dD} ${ac_dA}PVFS_USRINT_KMOUNT${ac_dB}PVFS_USRINT_KMOUNT${ac_dC}0${ac_dD} @@ -1075,14 +1075,14 @@ CEOF t clr : clr ${ac_uA}HAVE_ARPA_INET_H${ac_uB}HAVE_ARPA_INET_H${ac_uC}1${ac_uD} -${ac_uA}SIZEOF_LONG_INT${ac_uB}SIZEOF_LONG_INT${ac_uC}4${ac_uD} +${ac_uA}SIZEOF_LONG_INT${ac_uB}SIZEOF_LONG_INT${ac_uC}8${ac_uD} ${ac_uA}WITH_OPENSSL${ac_uB}WITH_OPENSSL${ac_uC}1${ac_uD} ${ac_uA}HAVE_OPENSSL_EVP_H${ac_uB}HAVE_OPENSSL_EVP_H${ac_uC}1${ac_uD} ${ac_uA}HAVE_OPENSSL_CRYPTO_H${ac_uB}HAVE_OPENSSL_CRYPTO_H${ac_uC}1${ac_uD} ${ac_uA}HAVE_OPENSSL_SHA_H${ac_uB}HAVE_OPENSSL_SHA_H${ac_uC}1${ac_uD} ${ac_uA}HAVE_AIOCB_ERROR_CODE${ac_uB}HAVE_AIOCB_ERROR_CODE${ac_uC}1${ac_uD} ${ac_uA}HAVE_AIOCB_RETURN_VALUE${ac_uB}HAVE_AIOCB_RETURN_VALUE${ac_uC}1${ac_uD} -${ac_uA}SIZEOF_VOID_P${ac_uB}SIZEOF_VOID_P${ac_uC}4${ac_uD} +${ac_uA}SIZEOF_VOID_P${ac_uB}SIZEOF_VOID_P${ac_uC}8${ac_uD} ${ac_uA}HAVE_SYS_EPOLL_H${ac_uB}HAVE_SYS_EPOLL_H${ac_uC}1${ac_uD} ${ac_uA}PVFS_USRINT_BUILD${ac_uB}PVFS_USRINT_BUILD${ac_uC}1${ac_uD} ${ac_uA}PVFS_USRINT_KMOUNT${ac_uB}PVFS_USRINT_KMOUNT${ac_uC}0${ac_uD} diff --git a/src/apps/admin/pvfs2-chmod.c b/src/apps/admin/pvfs2-chmod.c index f0a2a01..e2f74fe 100644 --- a/src/apps/admin/pvfs2-chmod.c +++ b/src/apps/admin/pvfs2-chmod.c @@ -155,7 +155,7 @@ int pvfs2_chmod (PVFS_permissions perms, char *destfile) { } memset(&resp_getattr,0,sizeof(PVFS_sysresp_getattr)); attrmask = (PVFS_ATTR_SYS_ALL_SETABLE); - + ret = PVFS_sys_getattr(resp_lookup.ref,attrmask,&credentials,&resp_getattr, NULL); if (ret < 0) { @@ -167,7 +167,7 @@ int pvfs2_chmod (PVFS_permissions perms, char *destfile) { new_attr.perms = perms; new_attr.mask = PVFS_ATTR_SYS_PERM; - + ret = PVFS_sys_setattr(resp_lookup.ref,new_attr,&credentials, NULL); if (ret < 0) { diff --git a/src/client/sysint/remove.sm b/src/client/sysint/remove.sm index a92102e..74d56e6 100644 --- a/src/client/sysint/remove.sm +++ b/src/client/sysint/remove.sm @@ -96,7 +96,7 @@ nested machine pvfs2_client_remove_helper_sm state object_remove_setup_msgpair { run remove_object_remove_setup_msgpair; - OSD_MDFILE_MSGPAIR => remove_helper_cleanup; + OSD_MDFILE_MSGPAIR => remove_collection; OSD_MSGPAIR => object_remove_xfer_osd_msgpair; success => object_remove_xfer_msgpair; default => object_remove_failure; @@ -150,12 +150,14 @@ static PINT_sm_action remove_getattr_init( struct PINT_smcb *smcb, job_status_s *js_p) { struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + PINT_SM_GETATTR_STATE_FILL( sm_p->getattr, sm_p->object_ref, PVFS_ATTR_META_ALL|PVFS_ATTR_COMMON_TYPE, PVFS_TYPE_NONE, 0); + return SM_ACTION_COMPLETE; } @@ -167,11 +169,17 @@ static PINT_sm_action remove_getattr_analyze_results( attr = &sm_p->getattr.attr; assert(attr); + struct server_configuration_s *server_config; + + server_config = PINT_get_server_config_struct( + sm_p->object_ref.fs_id); + PINT_put_server_config_struct(server_config); + switch(attr->objtype) { case PVFS_TYPE_METAFILE: - assert(attr->mask & PVFS_ATTR_META_DFILES); - assert(attr->u.meta.dfile_count > 0); + assert(attr->mask & PVFS_ATTR_META_DFILES); + assert(attr->u.meta.dfile_count > 0); gossip_debug(GOSSIP_CLIENT_DEBUG, "%s: must remove %d datafiles\n", __func__, attr->u.meta.dfile_count); @@ -219,7 +227,7 @@ static PINT_sm_action remove_datafile_remove_setup_msgpair( gossip_debug(GOSSIP_CLIENT_DEBUG, "remove state: datafile_remove_setup_msgpair\n"); - + attr = &sm_p->getattr.attr; assert(attr); diff --git a/src/client/sysint/sys-create.sm b/src/client/sysint/sys-create.sm index 78c27af..4a1b7fa 100644 --- a/src/client/sysint/sys-create.sm +++ b/src/client/sysint/sys-create.sm @@ -41,8 +41,6 @@ static int create_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); static int create_datafiles_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); -static int create_setattr_comp_fn( - void *v_p, struct PVFS_server_resp *resp_p, int index); static int create_crdirent_comp_fn( void *v_p, struct PVFS_server_resp *resp_p, int index); static int create_delete_handles_comp_fn( @@ -87,7 +85,6 @@ machine pvfs2_client_create_sm run create_create_setup_msgpair; OSD_MSGPAIR => create_xfer_osd_msgpair; OSD_CREATE_MSGPAIR => create_xfer_datafile_msgpair; - OSD_MDFILE_MSGPAIR => datafiles_setup_msgpair_array; success => create_xfer_msgpair; default => cleanup; } @@ -128,20 +125,6 @@ machine pvfs2_client_create_sm } state datafiles_xfer_osd_msgpair_array - { - jump pvfs2_osd_msgpairarray_sm; - success => create_setattr_setup_msgpair; - default => crdirent_failure; - } - - state create_setattr_setup_msgpair - { - run create_setattr_setup_msgpair; - OSD_MSGPAIR => create_setattr_xfer_osd_msgpair; - default => crdirent_setup_msgpair; - } - - state create_setattr_xfer_osd_msgpair { jump pvfs2_osd_msgpairarray_sm; success => crdirent_setup_msgpair; @@ -440,15 +423,20 @@ static int create_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("create_comp_fn\n"); PINT_smcb *smcb = v_p; PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); + struct server_configuration_s *server_config; + + server_config = PINT_get_server_config_struct( + sm_p->object_ref.fs_id); + PINT_put_server_config_struct(server_config); + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); gossip_debug(GOSSIP_CLIENT_DEBUG, "create_create_comp_fn\n"); - if (is_osd_meta) { + if (is_osd_meta) { int ret; uint64_t oid; PVFS_error status; @@ -473,15 +461,12 @@ static int create_comp_fn(void *v_p, osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); } else { assert(resp_p->op == PVFS_SERV_CREATE); - if (resp_p->status != 0) { return resp_p->status; } /* otherwise, just stash the newly created meta handle */ - sm_p->u.create.metafile_handle = - resp_p->u.create.metafile_handle; sm_p->u.create.datafile_count = resp_p->u.create.datafile_count; sm_p->u.create.datafile_handles = malloc( sizeof(*sm_p->u.create.datafile_handles) * @@ -494,7 +479,12 @@ static int create_comp_fn(void *v_p, resp_p->u.create.datafile_handles, (sizeof(*sm_p->u.create.datafile_handles) * resp_p->u.create.datafile_count)); - + + if (is_osd_md && server_config->post_create) + sm_p->u.create.metafile_handle = sm_p->u.create.datafile_handles[0]; + else + sm_p->u.create.metafile_handle = resp_p->u.create.metafile_handle; + sm_p->u.create.stuffed = resp_p->u.create.stuffed; } @@ -509,7 +499,6 @@ static int create_datafiles_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) { - gossip_err("create_datafiles_comp_fn\n"); PVFS_error status; PINT_smcb *smcb = v_p; PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); @@ -591,35 +580,6 @@ static int create_datafiles_comp_fn(void *v_p, return 0; } -static int create_setattr_comp_fn(void *v_p, - struct PVFS_server_resp *resp_p, - int index) -{ - int res, status; - PINT_smcb *smcb = v_p; - PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM); - - gossip_debug(GOSSIP_CLIENT_DEBUG, "create_setattr_comp_fn\n"); - - if (server_is_osd(sm_p->msgarray_op.msgpair.svr_addr)) { - osd_command_attr_free(&sm_p->msgarray_op.msgpair.osd_command); - status = osd_errno_from_status(sm_p->msgarray_op.msgpair.osd_command.status); - } else { - assert(resp_p->op == PVFS_SERV_SETATTR); - status = resp_p->status; - } - - res = PINT_copy_object_attr(&sm_p->u.create.cache_attr, - &sm_p->msgarray_op.msgpair.req.u.setattr.attr); - - if(res != 0) - { - return res; - } - - return status; -} - static int create_crdirent_comp_fn(void *v_p, struct PVFS_server_resp *resp_p, int index) @@ -651,22 +611,10 @@ static PINT_sm_action create_create_setup_msgpair( int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); int is_osd = fsid_is_osd(sm_p->object_ref.fs_id); - + gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: " "dspace_create_setup_msgpair\n"); - //if (is_osd_md) { - /* - * OK, so osd_type == OSD_MDFILE which means that we're storing the - * file metadata as attributes of the first datafile. There is nothing - * else to be done here so we'll go ahead and skip the - * create_xfer state and jump directly to the - * datafiles_setup_msgpair_array state. - */ - //js_p->error_code = OSD_MDFILE_MSGPAIR; - //return SM_ACTION_COMPLETE; - //} - gossip_debug(GOSSIP_CLIENT_DEBUG," create: posting create req\n"); /* reset the attributes to what got passed in to the sysint call. the retry @@ -684,7 +632,6 @@ static PINT_sm_action create_create_setup_msgpair( ret = PINT_cached_config_get_next_meta( sm_p->object_ref.fs_id, &msg_p->svr_addr, &meta_handle_extent_array, 0); - gossip_err("server: %s\n", BMI_addr_rev_lookup(msg_p->svr_addr)); if(ret != 0) { gossip_err("Failed to map meta server address\n"); @@ -703,10 +650,6 @@ static PINT_sm_action create_create_setup_msgpair( } if (is_osd_meta) { - /* - * Create a zero-length datafile on the metadata OSD. We'll fill in - * the required attributes in create_setattr_setup_msgpair(). - */ struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG, CCAP_OID, NULL, CCAP_OID_LEN }; @@ -729,6 +672,7 @@ static PINT_sm_action create_create_setup_msgpair( js_p->error_code = OSD_MSGPAIR; } else { + PINT_SERVREQ_CREATE_FILL( msg_p->req, *sm_p->cred_p, @@ -745,7 +689,6 @@ static PINT_sm_action create_create_setup_msgpair( msg_p->fs_id = sm_p->object_ref.fs_id; msg_p->handle = meta_handle_extent_array.extent_array[0].first; - gossip_err("msg_p->handle: %d\n", msg_p->handle); msg_p->retry_flag = PVFS_MSGPAIR_RETRY; msg_p->comp_fn = create_comp_fn; msg_p->req.u.create.attr.u.meta.dfile_count = 0; @@ -763,11 +706,11 @@ static PINT_sm_action create_create_setup_msgpair( static PINT_sm_action create_datafiles_setup_msgpair_array( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("create_datafiles_setup_msgpair_array\n"); struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int ret = -PVFS_EINVAL; struct server_configuration_s *server_config; int is_osd; + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); PINT_sm_msgpair_state *msg_p = NULL; gossip_debug(GOSSIP_CLIENT_DEBUG, "create state: " @@ -831,120 +774,17 @@ static PINT_sm_action create_datafiles_setup_msgpair_array( PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; -#define CURRENT_COMMAND_PAGE 0xfffffffeUL -#define CURRENT_COMMAND_PAGE_OID 4 - if (is_osd) { - struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; uint64_t attrval; -/* struct attribute_list attrs[] = {{ ATTR_GET, CUR_CMD_ATTR_PG,*/ -/* CCAP_OID, NULL, CCAP_OID_LEN },*/ -/* { ATTR_SET, USER_COLL_PG, 1, &attrval, 8}};*/ - - struct attribute_list attr = {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}; - - if(!sm_p->getattr.attr.cid) { - sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ - } - set_htonll(&attrval, sm_p->getattr.attr.cid); - - /* - * A hack in PVFS_util_init_defaults has set up an object an the - * low bound of the extent array. So as we create handles, we get - * them in increasing order that satisfies our extent bounds. - */ - ret = osd_command_set_create(command, PVFS_OSD_DATA_PID, sm_p->u.create.datafile_handles[0], 1); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_set_create failed", - __func__); - js_p->error_code = ret; - return 1; - } - -/* ret = osd_command_attr_build(command, attrs, 2);*/ - ret = osd_command_attr_build(command, &attr, 1); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_attr_build failed", - __func__); - js_p->error_code = ret; - return 1; - } - } - - msg_p->fs_id = sm_p->object_ref.fs_id; - msg_p->handle = sm_p->u.create.io_handle_extent_array[0]. - extent_array[0].first; - msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; - msg_p->comp_fn = create_datafiles_comp_fn; - msg_p->svr_addr = sm_p->u.create.data_server_addrs[0]; - - PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); - - return SM_ACTION_COMPLETE; -} - -static PINT_sm_action create_setattr_setup_msgpair( - struct PINT_smcb *smcb, job_status_s *js_p) -{ - struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); - int ret = -PVFS_EINVAL; - PINT_sm_msgpair_state *msg_p = NULL; - struct server_configuration_s *server_config; - - server_config = PINT_get_server_config_struct( - sm_p->object_ref.fs_id); - PINT_put_server_config_struct(server_config); - - int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); - int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); - - gossip_debug(GOSSIP_CLIENT_DEBUG, - "create state: setattr_setup_msgpair\n"); - - js_p->error_code = 0; - - if (!is_osd_md && !is_osd_meta) - return SM_ACTION_COMPLETE; - - gossip_debug(GOSSIP_CLIENT_DEBUG," create: posting setattr req\n"); - - PINT_msgpair_init(&sm_p->msgarray_op); - msg_p = &sm_p->msgarray_op.msgpair; - - if (is_osd_meta || is_osd_md) { - /* - * Fill in the metadata as attributes of the datafile. A list of - * attributes that PVFS2 currently stores in the MDS can be found here: - * osd/pvfs/src/io/trove/pvfs2-storage:62. - */ - uint64_t oid; - char *dist_buf; - int i, numattrs = 7; - struct attribute_list attr[numattrs]; - struct osd_command *command = &sm_p->msgarray_op.msgpair.osd_command; - - /* Set attr type, page and number */ - for (i = 0; i < numattrs; i++) { - attr[i].type = ATTR_SET; - attr[i].page = PVFS_USEROBJECT_ATTR_PG; - attr[i].number = i; - } - - /* uid */ - attr[0].val = &sm_p->u.create.attr.owner; - attr[0].len = sizeof(PVFS_uid); - - /* gid */ - attr[1].val = &sm_p->u.create.attr.group; - attr[1].len = sizeof(PVFS_gid); + struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; - /* XXX Default to PVFS_PERM_VALID till we can figure out the umask */ - sm_p->u.create.attr.perms = PVFS_PERM_VALID; - attr[2].val = &sm_p->u.create.attr.perms; - attr[2].len = sizeof(PVFS_permissions); + if (is_osd_md) + { + /* needed for osd_md */ + char *dist_buf; + sm_p->u.create.attr.perms = PVFS_PERM_VALID; - /* mask */ - sm_p->u.create.attr.mask = PVFS_ATTR_COMMON_UID | + sm_p->u.create.attr.mask = PVFS_ATTR_COMMON_UID | PVFS_ATTR_COMMON_GID | PVFS_ATTR_COMMON_PERM | PVFS_ATTR_COMMON_ATIME | @@ -953,83 +793,81 @@ static PINT_sm_action create_setattr_setup_msgpair( PVFS_ATTR_META_DIST | PVFS_ATTR_META_DFILES | PVFS_ATTR_COMMON_TYPE; - attr[3].val = &sm_p->u.create.attr.mask; - attr[3].len = sizeof(uint32_t); - - /* object type */ - sm_p->u.create.attr.objtype = PVFS_TYPE_METAFILE; - attr[4].val = &sm_p->u.create.attr.objtype; - attr[4].len = sizeof(PVFS_ds_type); - - /* - * Encode the distribution for storage. XXX Where do we free this - * memory? - */ - dist_buf = (char *)malloc(PINT_DIST_PACK_SIZE(sm_p->u.create.dist)); - if (!dist_buf) { - js_p->error_code = -PVFS_ENOMEM; - return 1; - } - PINT_dist_encode(dist_buf, sm_p->u.create.dist); - attr[5].val = dist_buf; - attr[5].len = PINT_DIST_PACK_SIZE(sm_p->u.create.dist); - /* dfile_array */ - attr[6].val = sm_p->u.create.datafile_handles; - attr[6].len = sizeof(PVFS_handle) * sm_p->u.create.num_data_files; + sm_p->u.create.attr.objtype = PVFS_TYPE_METAFILE; + + dist_buf = (char *)malloc(PINT_DIST_PACK_SIZE(sm_p->u.create.dist)); + if (!dist_buf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + PINT_dist_encode(dist_buf, sm_p->u.create.dist); + + struct attribute_list attrs[] = { + {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 0, &sm_p->u.create.attr.owner, sizeof(PVFS_uid)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 1, &sm_p->u.create.attr.group, sizeof(PVFS_gid)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 2, &sm_p->u.create.attr.perms, sizeof(PVFS_permissions)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 3, &sm_p->u.create.attr.mask, sizeof(uint32_t)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 4, &sm_p->u.create.attr.objtype, sizeof(PVFS_ds_type)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 5, dist_buf, PINT_DIST_PACK_SIZE(sm_p->u.create.dist)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 6, sm_p->u.create.datafile_handles, sizeof(PVFS_handle) * sm_p->u.create.num_data_files}}; + + if(!sm_p->getattr.attr.cid) { + sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ + } + set_htonll(&attrval, sm_p->getattr.attr.cid); + + ret = osd_command_set_create(command, PVFS_OSD_DATA_PID, sm_p->u.create.datafile_handles[0], 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_create failed", + __func__); + js_p->error_code = ret; + return 1; + } - oid = sm_p->u.create.metafile_handle; - ret = osd_command_set_set_attributes(command, - is_osd_md ? PVFS_OSD_DATA_PID : PVFS_OSD_META_PID, oid); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_set_set_attributes failed", - __func__); - js_p->error_code = ret; - return 1; + ret = osd_command_attr_build(command, attrs, 8); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } } + else + { + struct attribute_list attrs = {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}; - ret = osd_command_attr_build(command, attr, numattrs); - if (ret) { - osd_error_xerrno(ret, "%s: osd_command_attr_build failed", - __func__); - js_p->error_code = ret; - return 1; + if(!sm_p->getattr.attr.cid) { + sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ + } + set_htonll(&attrval, sm_p->getattr.attr.cid); + + ret = osd_command_set_create(command, PVFS_OSD_DATA_PID, sm_p->u.create.datafile_handles[0], 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_set_create failed", + __func__); + js_p->error_code = ret; + return 1; + } + + ret = osd_command_attr_build(command, &attrs, 1); + if (ret) { + osd_error_xerrno(ret, "%s: osd_command_attr_build failed", + __func__); + js_p->error_code = ret; + return 1; + } } - } else { -/* PINT_SERVREQ_SETATTR_FILL(*/ -/* msg_p->req,*/ -/* *sm_p->cred_p,*/ -/* sm_p->object_ref.fs_id,*/ -/* sm_p->u.create.metafile_handle,*/ -/* PVFS_TYPE_METAFILE,*/ -/* sm_p->u.create.attr,*/ -/* PVFS_ATTR_META_ALL,*/ -/* sm_p->hints);*/ - -/* msg_p->req.u.setattr.attr.u.meta.dfile_array =*/ -/* sm_p->u.create.datafile_handles;*/ -/* msg_p->req.u.setattr.attr.u.meta.dfile_count =*/ -/* sm_p->u.create.num_data_files;*/ -/* msg_p->req.u.setattr.attr.u.meta.dist =*/ -/* sm_p->u.create.dist;*/ -/* msg_p->req.u.setattr.attr.u.meta.dist_size =*/ -/* PINT_DIST_PACK_SIZE(sm_p->u.create.dist);*/ } msg_p->fs_id = sm_p->object_ref.fs_id; - msg_p->handle = sm_p->u.create.metafile_handle; + msg_p->handle = sm_p->u.create.io_handle_extent_array[0]. + extent_array[0].first; msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY; - msg_p->comp_fn = create_setattr_comp_fn; - - ret = PINT_cached_config_map_to_server( - &msg_p->svr_addr, msg_p->handle, msg_p->fs_id); + msg_p->comp_fn = create_datafiles_comp_fn; + msg_p->svr_addr = sm_p->u.create.data_server_addrs[0]; - if (ret) - { - gossip_err("Failed to map meta server address\n"); - js_p->error_code = ret; - } - js_p->error_code = (is_osd_meta || is_osd_md) ? OSD_MSGPAIR : 0; PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); return SM_ACTION_COMPLETE; @@ -1057,7 +895,6 @@ static PINT_sm_action create_crdirent_setup_msgpair( PINT_msgpair_init(&sm_p->msgarray_op); msg_p = &sm_p->msgarray_op.msgpair; - gossip_err("is_osd: %d\n", server_is_osd(msg_p->svr_addr)); PINT_SERVREQ_CRDIRENT_FILL( msg_p->req, *sm_p->cred_p, @@ -1130,7 +967,7 @@ static PINT_sm_action create_cleanup( metafile_ref.handle = sm_p->u.create.metafile_handle; metafile_ref.fs_id = sm_p->object_ref.fs_id; metafile_ref.cid = sm_p->getattr.attr.cid; - + /* fill in outgoing response fields */ sm_p->u.create.create_resp->ref = metafile_ref; diff --git a/src/client/sysint/sys-io.sm b/src/client/sysint/sys-io.sm index c6132da..bcffab6 100644 --- a/src/client/sysint/sys-io.sm +++ b/src/client/sysint/sys-io.sm @@ -34,7 +34,8 @@ extern job_context_id pint_client_sm_context; enum { - IO_DO_OSD_IO = 2001 + IO_DO_OSD_IO = 2001, + OSD_MDFILE_MSGPAIR = 2002 }; enum @@ -437,6 +438,10 @@ static PINT_sm_action io_init( { struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); job_id_t tmp_id; + struct server_configuration_s *server_config = NULL; + + server_config = PINT_get_server_config_struct(sm_p->object_ref.fs_id); + PINT_put_server_config_struct(server_config); gossip_debug(GOSSIP_CLIENT_DEBUG, "(%p) io state: io_init\n", sm_p); @@ -449,7 +454,7 @@ static PINT_sm_action io_init( IO_ATTR_MASKS, PVFS_TYPE_METAFILE, 0); - + if (js_p->error_code == IO_RETRY || (js_p->error_code == IO_RETRY_NODELAY)) { @@ -475,6 +480,7 @@ static PINT_sm_action io_init( sm_p->msgarray_op.params.retry_delay, smcb, 0, js_p, &tmp_id, pint_client_sm_context); } + return SM_ACTION_COMPLETE; } @@ -596,7 +602,6 @@ static PINT_sm_action io_datafile_setup_msgpairs( switch(attr->objtype) { case PVFS_TYPE_METAFILE: - assert(attr->mask & PVFS_ATTR_META_DFILES); assert(attr->mask & PVFS_ATTR_META_DIST); assert(attr->u.meta.dist_size > 0); @@ -615,6 +620,7 @@ static PINT_sm_action io_datafile_setup_msgpairs( js_p->error_code = -PVFS_EBADF; goto exit; } + /* cannot write to an immutable file */ if (sm_p->u.io.io_type == PVFS_IO_WRITE && (attr->u.meta.hint.flags & PVFS_IMMUTABLE_FL)) diff --git a/src/client/sysint/sys-mkdir.sm b/src/client/sysint/sys-mkdir.sm index 4dad827..d93559b 100644 --- a/src/client/sysint/sys-mkdir.sm +++ b/src/client/sysint/sys-mkdir.sm @@ -469,6 +469,7 @@ static PINT_sm_action create_collection_setup_msgpair( /* struct attribute_list attr = { ATTR_GET, CUR_CMD_ATTR_PG,*/ /* CCAP_OID, NULL, CCAP_OID_LEN };*/ + ret = osd_command_set_create_collection(command, PVFS_OSD_DATA_PID, sm_p->u.mkdir.cid); if (ret) { diff --git a/src/client/sysint/sys-osd-io.sm b/src/client/sysint/sys-osd-io.sm index 1109221..a470a86 100644 --- a/src/client/sysint/sys-osd-io.sm +++ b/src/client/sysint/sys-osd-io.sm @@ -95,7 +95,7 @@ static int osd_io_init(struct PINT_smcb *smcb, job_status_s *js_p) struct PINT_client_io_sm *io = &sm_p->u.io; PVFS_object_attr *attr = &sm_p->getattr.attr; int i, ret; - + ret = PINT_msgpairarray_init(&sm_p->msgarray_op, io->datafile_count); if (ret) goto out; @@ -181,6 +181,7 @@ static int osd_io_setup_msgpairs(struct PINT_smcb *smcb, job_status_s *js_p) struct osd_command *command; struct bsg_iovec *iov; int dfile_count = sm_p->u.io.datafile_count; + int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); struct server_configuration_s *server_config; server_config = PINT_get_server_config_struct( @@ -341,7 +342,6 @@ static int osd_io_setup_msgpairs(struct PINT_smcb *smcb, job_status_s *js_p) sm_p->msgarray_op.msgarray[i].osd_sgl = NULL; p = io->buffer; - if (sresult.segs == 1) { /* contiguous server buff to write to */ if (csegs_count == 1) { p += offseta[0]; @@ -381,28 +381,74 @@ static int osd_io_setup_msgpairs(struct PINT_smcb *smcb, job_status_s *js_p) } else if (io->io_type == PVFS_IO_WRITE) { if (server_config->post_create && !target_offset[0]) { - uint64_t attrval; - struct attribute_list attr = {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}; - - if(!sm_p->getattr.attr.cid) { - sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ - } - set_htonll(&attrval, sm_p->getattr.attr.cid); - - osd_command_set_create_and_write(command, PVFS_OSD_DATA_PID, datafile_handle, len, target_offset[0]); - - osd_command_attr_build(command, &attr, 1); + uint64_t attrval; + + if (is_osd_md) + { + /* needed for osd_md */ + char *dist_buf; + attr->perms = PVFS_PERM_VALID; + attr->mask = PVFS_ATTR_COMMON_UID | + PVFS_ATTR_COMMON_GID | + PVFS_ATTR_COMMON_PERM | + PVFS_ATTR_COMMON_ATIME | + PVFS_ATTR_COMMON_CTIME | + PVFS_ATTR_COMMON_MTIME | + PVFS_ATTR_META_DIST | + PVFS_ATTR_META_DFILES | + PVFS_ATTR_COMMON_TYPE; + + attr->objtype = PVFS_TYPE_METAFILE; + + dist_buf = (char *)malloc(PINT_DIST_PACK_SIZE(sm_p->getattr.attr.u.meta.dist)); + if (!dist_buf) { + js_p->error_code = -PVFS_ENOMEM; + return 1; + } + PINT_dist_encode(dist_buf, sm_p->getattr.attr.u.meta.dist); + + struct attribute_list attrs[] = { + {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 0, &attr->owner, sizeof(PVFS_uid)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 1, &attr->group, sizeof(PVFS_gid)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 2, &attr->perms, sizeof(PVFS_permissions)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 3, &attr->mask, sizeof(uint32_t)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 4, &attr->objtype, sizeof(PVFS_ds_type)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 5, dist_buf, PINT_DIST_PACK_SIZE(sm_p->getattr.attr.u.meta.dist)}, + {ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 6, attr->u.meta.dfile_array, sizeof(PVFS_handle) * sm_p->getattr.attr.u.meta.dfile_count }}; + + if(!sm_p->getattr.attr.cid) { + sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ + } + set_htonll(&attrval, sm_p->getattr.attr.cid); + + osd_command_set_create_and_write(command, PVFS_OSD_DATA_PID, datafile_handle, len, target_offset[0]); + osd_command_attr_build(command, attrs, 8); + } + else + { + struct attribute_list attrs = {ATTR_SET, USER_COLL_PG, 1, &attrval, 8}; + + if(!sm_p->getattr.attr.cid) { + sm_p->getattr.attr.cid = COLLECTION_OID_LB; /* root directory */ + } + set_htonll(&attrval, sm_p->getattr.attr.cid); + + osd_command_set_create_and_write(command, PVFS_OSD_DATA_PID, datafile_handle, len, target_offset[0]); + osd_command_attr_build(command, &attrs, 1); + } + } else { osd_command_set_write(command, PVFS_OSD_DATA_PID, datafile_handle, len, target_offset[0]); + command->outdata = p; } - command->outdata = p; command->outlen = len; command->iov_outlen = csegs_count; //~ printf("Contig WRITE %d\n", len); - } + } } else if (sresult.segs > 0) { /* either need a SGL or optimized SGL */ int j, flag, stride, segl, total_len; diff --git a/src/client/sysint/sys-readdirplus.sm b/src/client/sysint/sys-readdirplus.sm index 1fee7e0..55edd73 100644 --- a/src/client/sysint/sys-readdirplus.sm +++ b/src/client/sysint/sys-readdirplus.sm @@ -524,6 +524,7 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm js_p->error_code = -PVFS_EINVAL; return SM_ACTION_COMPLETE; } + ret = PINT_msgpairarray_init( &sm_p->msgarray_op, sm_p->u.readdirplus.svr_count); if(ret != 0) @@ -541,7 +542,7 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm msg_p->retry_flag = PVFS_MSGPAIR_RETRY; msg_p->comp_fn = readdirplus_fetch_attrs_comp_fn; msg_p->svr_addr = sm_p->u.readdirplus.server_addresses[i]; - + if (server_is_osd(msg_p->svr_addr)) { uint64_t oid; @@ -577,7 +578,7 @@ static PINT_sm_action readdirplus_fetch_attrs_setup_msgpair(struct PINT_smcb *sm sm_p->hints); /* immediate return. next state jumps to msgpairarray machine */ js_p->error_code = 0; - } + } } PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op); @@ -810,6 +811,7 @@ static PINT_sm_action readdirplus_fetch_sizes_setup_msgpair( struct osd_command *command = &sm_p->msgarray_op.msgarray[0].osd_command; + gossip_err("msg_p->handle: %d\n", msg_p->handle); ret = osd_command_set_get_member_attributes(command, PVFS_OSD_DATA_PID, sm_p->object_ref.cid, msg_p->handle); if (ret) { osd_error_xerrno(ret, "%s: osd_command_set_get_member_attributes failed", diff --git a/src/client/sysint/sys-setattr.sm b/src/client/sysint/sys-setattr.sm index b369bd5..a80ef50 100644 --- a/src/client/sysint/sys-setattr.sm +++ b/src/client/sysint/sys-setattr.sm @@ -232,7 +232,7 @@ static PINT_sm_action setattr_msg_setup_msgpair( int is_osd_md = fsid_is_osd_md(sm_p->object_ref.fs_id); int is_osd_meta = fsid_is_osd_meta(sm_p->object_ref.fs_id); - js_p->error_code = (is_osd_meta || is_osd_md) ? OSD_MSGPAIR : 0; + js_p->error_code = (is_osd_meta || (is_osd_md && !(sm_p->u.setattr.sys_attr.objtype & PVFS_TYPE_DIRECTORY))) ? OSD_MSGPAIR : 0; gossip_debug(GOSSIP_CLIENT_DEBUG," setattr: posting setattr req\n"); @@ -242,7 +242,8 @@ static PINT_sm_action setattr_msg_setup_msgpair( objtype = ((sm_p->u.setattr.sys_attr.mask & PVFS_ATTR_SYS_TYPE) ? sm_p->u.setattr.sys_attr.objtype : PVFS_TYPE_NONE); - if (is_osd_meta || is_osd_md) { + /* if we have a directory object and osd_md, the dir object still exists in pvfs mds*/ + if (is_osd_meta || (is_osd_md && !(sm_p->u.setattr.sys_attr.objtype & PVFS_TYPE_DIRECTORY))) { /* * Fill in the metadata as attributes of the datafile. A list of * attributes that PVFS2 currently stores in the MDS can be found here: diff --git a/src/io/description/pint-request.c b/src/io/description/pint-request.c index 13bbba5..6b68eba 100644 --- a/src/io/description/pint-request.c +++ b/src/io/description/pint-request.c @@ -104,6 +104,7 @@ int PINT_process_request(PINT_Request_state *req, gossip_lerr("PINT_process_request: NULL offset or size array!\n"); return -PVFS_EINVAL; } + /* initialize some variables */ retval = 0; if (PINT_EQ_CKSIZE(mode)) /* be must be exact here */ @@ -315,7 +316,7 @@ int PINT_process_request(PINT_Request_state *req, { sz = result->bytemax - result->bytes; } - PINT_ADD_SEGMENT(result, contig_offset, sz, mode); + PINT_ADD_SEGMENT(result, contig_offset, sz, mode); retval = sz; } else diff --git a/src/server/create.sm b/src/server/create.sm index 3689867..4b7147a 100644 --- a/src/server/create.sm +++ b/src/server/create.sm @@ -20,7 +20,8 @@ enum { OSD_MSGPAIR = 2001, - OSD_MDFILE_MSGPAIR = 2002 + OSD_DATAFILE_MSGPAIR = 2002, + OSD_MDFILE_MSGPAIR = 2003 }; %% @@ -37,8 +38,8 @@ machine pvfs2_create_sm state create_metafile { run create_metafile; - success => check_stuffed; OSD_MDFILE_MSGPAIR => setup_local_datafile_handles; + success => check_stuffed; default => setup_final_response; } @@ -46,6 +47,7 @@ machine pvfs2_create_sm { run check_stuffed; success => create_local_datafiles; + OSD_DATAFILE_MSGPAIR => setup_local_datafile_handles; default => setup_final_response; } @@ -60,6 +62,7 @@ machine pvfs2_create_sm { run setup_local_datafile_handles; success => request_datafiles; + OSD_MSGPAIR => write_keyvals; default => remove_local_datafile_handles; } @@ -160,14 +163,14 @@ static int create_metafile( job_id_t i; PVFS_handle_extent_array meta_handle_ext_array; server_configuration_s *config = get_server_config_struct(); - - if (config->osd_type == OSD_MDFILE) + + if(config->osd_type == OSD_MDFILE) { - /* Nothing to do here, go to the setup_local_datafile_handles to get a data handle_array_local */ js_p->error_code = OSD_MDFILE_MSGPAIR; return SM_ACTION_COMPLETE; } + /* first state to check in, make sure the attr mask contains the dist bit. * it's required later (not sure if we have to require it) but if we don't * have it here, return an EINVAL */ @@ -223,7 +226,7 @@ static int check_stuffed( if(config->osd_type == OSD_DATAFILE) { - js_p->error_code = 0; + js_p->error_code = OSD_DATAFILE_MSGPAIR; return SM_ACTION_COMPLETE; } @@ -597,7 +600,6 @@ static PINT_sm_action remove_keyvals( static PINT_sm_action setup_local_datafile_handles( struct PINT_smcb *smcb, job_status_s *js_p) { - gossip_err("setup_local_datafile_handles\n"); struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); int i; int tmp_index = 0; @@ -625,18 +627,14 @@ static PINT_sm_action setup_local_datafile_handles( } cur = PINT_llist_next(cur); } - + s_op->u.create.num_io_servers = s_op->req->u.create.num_dfiles_req; s_op->req->u.create.attr.u.meta.dfile_count = s_op->u.create.num_io_servers; s_op->resp.u.create.datafile_count = s_op->u.create.num_io_servers; s_op->resp.u.create.datafile_handles = malloc(sizeof(PVFS_handle)); s_op->resp.u.create.datafile_handles[0] = trove_handle_alloc_from_range(s_op->req->u.create.fs_id, &data_handle_ext_array); - - if (config->osd_type == OSD_MDFILE) - js_p->error_code = OSD_MDFILE_MSGPAIR; - else - js_p->error_code = 0; + js_p->error_code = OSD_MSGPAIR; } else { if(s_op->resp.u.create.stuffed) { @@ -838,6 +836,9 @@ static int setup_resp( struct PINT_smcb *smcb, job_status_s *js_p) { struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT); + + if (js_p->error_code == OSD_MSGPAIR) + js_p->error_code = 0; if (js_p->error_code == 0) { PINT_ACCESS_DEBUG(s_op, GOSSIP_ACCESS_DEBUG, diff --git a/src/server/final-response.sm b/src/server/final-response.sm index ea48bf2..b82a21e 100644 --- a/src/server/final-response.sm +++ b/src/server/final-response.sm @@ -81,7 +81,6 @@ static PINT_sm_action final_response_release( * shows up as a generic pvfs error on the client side */ s_op->resp.status = -PVFS_ERROR_CODE(-js_p->error_code); - /* catch cases in which the operation has not been scheduled */ if (!s_op->scheduled_id) { diff --git a/src/server/get-attr.sm b/src/server/get-attr.sm index df150e0..4098e57 100644 --- a/src/server/get-attr.sm +++ b/src/server/get-attr.sm @@ -325,6 +325,7 @@ static PINT_sm_action getattr_verify_attribs( have the original client request attr mask (s_op->u.getattr.attrmask). */ + switch(resp_attr->objtype) { case PVFS_TYPE_METAFILE: diff --git a/src/server/mkdir.sm b/src/server/mkdir.sm index 451dce5..7dc5ea1 100644 --- a/src/server/mkdir.sm +++ b/src/server/mkdir.sm @@ -294,7 +294,7 @@ static PINT_sm_action mkdir_prep_sm( s_op->u.mkdir.handle_extent_array = s_op->req->u.mkdir.handle_extent_array; - if(config->osd_type == OSD_DATAFILE) + if(config->osd_type == OSD_DATAFILE || config->osd_type == OSD_MDFILE) { cur = config->host_aliases; while(cur)