From 749e802658527807f050799f758c73f789cd04c9 Mon Sep 17 00:00:00 2001 From: cengizk Date: Fri, 13 Jun 2014 11:32:46 -0400 Subject: [PATCH] Updated osd-util and osd-initiator * Not tested the functionality yet --- osd-initiator/command.c | 58 ++++-- osd-initiator/command.h | 6 +- osd-initiator/drivelist.c | 31 +++ osd-initiator/drivelist.h | 1 + osd-initiator/python/Makefile | 2 +- osd-initiator/python/command.c | 7 +- osd-initiator/python/drivelist.c | 5 + osd-initiator/python/pvfs-init.py | 52 ++++- .../python/pvfs-osd-integrated-init.py | 2 +- osd-initiator/sync.c | 4 +- osd-initiator/tests/iospeed-mpi.c | 4 +- osd-util/initiator | 54 ++++- osd-util/perf.py | 193 +++++++++++------- 13 files changed, 293 insertions(+), 126 deletions(-) diff --git a/osd-initiator/command.c b/osd-initiator/command.c index 71845d5..7df84fb 100644 --- a/osd-initiator/command.c +++ b/osd-initiator/command.c @@ -222,13 +222,14 @@ int osd_command_set_get_attributes(struct osd_command *command, uint64_t pid, return 0; } - int osd_command_set_get_member_attributes(struct osd_command *command, - uint64_t pid, uint64_t cid) + uint64_t pid, uint64_t cid, uint64_t min_oid, uint64_t max_oid) { varlen_cdb_init(command, OSD_GET_MEMBER_ATTRIBUTES); set_htonll(&command->cdb[16], pid); set_htonll(&command->cdb[24], cid); + set_htonll(&command->cdb[32], min_oid); + set_htonll(&command->cdb[40], max_oid); return 0; } @@ -503,18 +504,25 @@ struct attr_malloc_header { * After the command completes, call the resolve function to fill in * the results and free the temp buffers. */ + int osd_command_attr_build(struct osd_command *command, const struct attribute_list *attr, int numattr) +{ + return osd_command_multi_attr_build(command, attr, numattr, 1); +} + +int osd_command_multi_attr_build(struct osd_command *command, + const struct attribute_list *attr, int numattr, int numobj) { int use_getpage, use_set_one_attr = 0; int numget, numgetpage, numgetmulti, numset, numresult; uint32_t getsize, getpagesize, getmultisize, setsize, resultsize; - uint32_t getmulti_num_objects = 0; + uint32_t getmulti_num_objects = 1; struct attr_malloc_header *header; uint8_t *p, *extra_out_buf, *extra_in_buf; struct bsg_iovec *iov; - int i; + int i, j; int neediov; int setattr_index; uint64_t extra_out, extra_in; @@ -549,7 +557,7 @@ int osd_command_attr_build(struct osd_command *command, uint64_t size_pad_outdata_alloc; uint64_t size_pad_indata_alloc; uint64_t extra_out_alloc, extra_in_alloc; - + if (numattr == 0) return 0; @@ -583,7 +591,6 @@ int osd_command_attr_build(struct osd_command *command, return -EINVAL; } } - if (numgetpage) { if (numgetpage > 1) { osd_error("%s: only one ATTR_GET_PAGE at a time", @@ -624,13 +631,13 @@ int osd_command_attr_build(struct osd_command *command, __func__); return -EINVAL; } - if (action != OSD_CREATE) { + if (action != OSD_CREATE && action != OSD_GET_MEMBER_ATTRIBUTES) { osd_error( - "%s: ATTR_GET_MULTI must only be used with CREATE", + "%s: ATTR_GET_MULTI must only be used with CREATE or GET_MEMBER_ATTRIBUTES", __func__); return -EINVAL; } - getmulti_num_objects = get_ntohs(&command->cdb[36]); + getmulti_num_objects = numobj; } use_getpage = 0; if (numget == 0 && numgetmulti == 0 && numset == 1) { @@ -687,7 +694,7 @@ int osd_command_attr_build(struct osd_command *command, size_pad_setlist = start_getlist - prev; if (size_setlist == 0) size_pad_outdata_alloc = roundup8(size_pad_setlist); - size_getlist = 8 + 8 * (numget + numgetmulti); + size_getlist = 8 + 8 * getmulti_num_objects * (numget + numgetmulti); } extra_out = size_pad_outdata + size_setlist @@ -714,7 +721,7 @@ int osd_command_attr_build(struct osd_command *command, if (numgetpage) size_retrieved = getpagesize; /* no rounding */ if (numgetmulti) - size_retrieved = 8 + getmultisize * + size_retrieved = 8 + (getmultisize + 16) * getmulti_num_objects; if (numresult) { if (!size_retrieved) @@ -875,13 +882,15 @@ int osd_command_attr_build(struct osd_command *command, */ if (numget || numgetmulti) { uint8_t *q = p + 8; - for (i=0; icdb[64], start_retrieved); set_htonl(&command->cdb[68], size_setlist); set_htonoffset(&command->cdb[72], start_setlist); + set_htonl(&command->cdb[80], numobj); } /* @@ -1065,14 +1075,14 @@ int osd_command_attr_resolve(struct osd_command *command) __func__); goto unwind; } - } else if ((p[0] & 0xf) == 0xf) { + } else if ((p[0] & 0xf) == 0xe) { if (!numgetmulti) { - osd_error("%s: got list type f, not expecting multi", + osd_error("%s: got list type e, not expecting multi", __func__); goto unwind; } } else { - osd_error("%s: expecting list type 9 or f, got 0x%x", + osd_error("%s: expecting list type 9 or e, got 0x%x", __func__, p[0] & 0xf); goto unwind; } @@ -1100,7 +1110,8 @@ int osd_command_attr_resolve(struct osd_command *command) break; if (numgetmulti) { oid = get_ntohll(&p[0]); - p += 8; + p += 16; + len -= 16; } page = get_ntohl(&p[0]); number = get_ntohl(&p[4]); @@ -1147,7 +1158,10 @@ int osd_command_attr_resolve(struct osd_command *command) mr->oid[mr->numoid] = oid; mr->val[mr->numoid] = avail_len ? p : NULL; + //osd_debug("%d\n", mr->oid[mr->numoid]); + //osd_debug("%d\n", get_ntohll(((struct attribute_get_multi_results *)attr[i].val)->val[((struct attribute_get_multi_results *)attr[i].val)->numoid])); mr->outlen[mr->numoid] = avail_len; + //osd_debug("%d\n", mr->outlen[mr->numoid]); ++mr->numoid; } else { attr[i].val = avail_len ? p : NULL; diff --git a/osd-initiator/command.h b/osd-initiator/command.h index 92db82c..4c763bc 100644 --- a/osd-initiator/command.h +++ b/osd-initiator/command.h @@ -59,7 +59,7 @@ struct attribute_get_multi_results { and just set to point to the scsi/sg.h sg_iovec (which is ignored in the kernel */ -#ifdef __APPLE__ +#if 0 /* * This is copied from a kernel header to avoid including it. */ @@ -129,7 +129,7 @@ int osd_command_set_format_osd(struct osd_command *command, uint64_t capacity); int osd_command_set_get_attributes(struct osd_command *command, uint64_t pid, uint64_t oid); int osd_command_set_get_member_attributes(struct osd_command *command, - uint64_t pid, uint64_t cid); + uint64_t pid, uint64_t cid, uint64_t min_oid, uint64_t max_oid); int osd_command_set_list(struct osd_command *command, uint64_t pid, uint32_t list_id, uint64_t alloc_len, uint64_t initial_oid, int list_attr); @@ -185,6 +185,8 @@ int osd_command_set_cond_setattr(struct osd_command *command, uint64_t pid, /* Attributes */ int osd_command_attr_build(struct osd_command *command, const struct attribute_list *const attrs, int num); +int osd_command_multi_attr_build(struct osd_command *command, + const struct attribute_list *const attrs, int num, int numobj); int osd_command_attr_resolve(struct osd_command *command); void osd_command_attr_free(struct osd_command *command); diff --git a/osd-initiator/drivelist.c b/osd-initiator/drivelist.c index f1813f8..8a15a69 100644 --- a/osd-initiator/drivelist.c +++ b/osd-initiator/drivelist.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "osd-util/osd-util.h" #include "command.h" @@ -151,6 +152,36 @@ int osd_get_drive_list(struct osd_drive_description **drives, int *num_drives) ++count; } + /* grab the hostname from the fs.conf file */ + int i=0, j; + int condition=1; + char *string[200]; + char *line; + FILE *file; + file = fopen("/usr/src/fs.conf", "r"); + line = (char *) malloc(120); + + while(condition != EOF) + { + condition = fscanf(file, "%s", line); + if(condition != EOF) + { + string[i]=(char *) malloc(120); + strcpy(string[i], line); + if(!strncmp(string[i], "osd://", 5)) + { + + for(j = 0; j < count; j++) { + if(!strcmp(&(line[6]), ret[j].targetname)) + ret[j].hostname = strdup(string[i-1]); + } + } + i++; + } + } + + fclose(file); + out: if (toplevel) closedir(toplevel); diff --git a/osd-initiator/drivelist.h b/osd-initiator/drivelist.h index d7f8ac3..658bd2c 100644 --- a/osd-initiator/drivelist.h +++ b/osd-initiator/drivelist.h @@ -21,6 +21,7 @@ struct osd_drive_description { char *targetname; char *chardev; + char *hostname; }; int osd_get_drive_list(struct osd_drive_description **drives, int *num_drives); diff --git a/osd-initiator/python/Makefile b/osd-initiator/python/Makefile index 3c4b292..c3d2f09 100644 --- a/osd-initiator/python/Makefile +++ b/osd-initiator/python/Makefile @@ -15,7 +15,7 @@ else # mad guess INCDIR := -I/usr/include/python2.4 endif -INCDIR += -I../.. +INCDIR += -I../.. -I../ OSDLIBS := ../libosdinit.a ../../osd-util/libosdutil.a -include ../../Makedefs diff --git a/osd-initiator/python/command.c b/osd-initiator/python/command.c index ab0ad7e..f3a2b37 100644 --- a/osd-initiator/python/command.c +++ b/osd-initiator/python/command.c @@ -462,10 +462,7 @@ static PyObject *pyosd_command_set_flush(PyObject *self, PyObject *args) } py_command->set = 1; - -/* FIXME: scope=2 needs a start/length params should read these in too - osd_command_set_flush(command, pid, oid, scope);*/ - osd_command_set_flush(command, pid, 0, 0, oid, 0); + osd_command_set_flush(command, pid,0, 0, oid, scope); Py_IncRef(self); return self; } @@ -586,7 +583,7 @@ static PyObject *pyosd_command_set_get_member_attributes(PyObject *self, } py_command->set = 1; - osd_command_set_get_member_attributes(command, pid, cid); + osd_command_set_get_member_attributes(command, pid, cid, 0, 1); Py_IncRef(self); return self; } diff --git a/osd-initiator/python/drivelist.c b/osd-initiator/python/drivelist.c index 78b4537..2155911 100644 --- a/osd-initiator/python/drivelist.c +++ b/osd-initiator/python/drivelist.c @@ -20,6 +20,7 @@ #include "osd-initiator/command.h" #include "osd-initiator/drivelist.h" #include "pyosd.h" +#include "osd-util/osd-util.h" /* * OSDDrive type. @@ -28,6 +29,7 @@ struct pyosd_drive { PyObject_HEAD; PyObject *targetname; PyObject *chardev; + PyObject *hostname; }; /* @@ -64,6 +66,7 @@ static int pyosd_drivelist_init(PyObject *self, PyObject *args, drive = PyObject_New(typeof(*drive), &pyosd_drive_type); drive->targetname = PyString_FromString(drives[i].targetname); drive->chardev = PyString_FromString(drives[i].chardev); + drive->hostname = PyString_FromString(drives[i].hostname); Py_INCREF(drive); PyList_Append(self, (PyObject *) drive); } @@ -80,6 +83,8 @@ struct PyMemberDef pyosd_drive_members[] = { READONLY, "target name" }, { "chardev", T_OBJECT, offsetof(struct pyosd_drive, chardev), READONLY, "character device" }, + { "hostname", T_OBJECT, offsetof(struct pyosd_drive, hostname), + READONLY, "hostname" }, { NULL } }; diff --git a/osd-initiator/python/pvfs-init.py b/osd-initiator/python/pvfs-init.py index 787d49c..88d47aa 100755 --- a/osd-initiator/python/pvfs-init.py +++ b/osd-initiator/python/pvfs-init.py @@ -68,7 +68,11 @@ def usage(): set_progname(sys.argv[0]) +# defaults root_handle = 0 +fsconf = False +osddirtype = "pvfs" +coll_object = False if len(sys.argv) == 4 or len(sys.argv) == 6: devname = sys.argv[1]; datalb = int(sys.argv[2]); @@ -79,10 +83,33 @@ if len(sys.argv) == 4 or len(sys.argv) == 6: else: usage() +# figure out if we are using a collection object to represent directories +# or not + +if fsconf: + osddirtype = False + coll_object = False + get_next_osddir = False + get_next_coll = False + fh = open(fsconf, "r") + buf = fh.read() + fh.close() + for word in buf.split(): + if get_next_osddir == True: + osddirtype = word + get_next_osddir = False + if get_next_coll == True: + coll_object = word + get_next_coll = False + if "OSDDirType" == word: + get_next_osddir = True + if "CollectionObject" == word: + get_next_coll = True + drives = OSDDriveList() dev = False for d in drives: - if d.targetname == devname: + if d.hostname == devname: dev = OSDDevice(d.chardev) break if not dev: @@ -107,22 +134,27 @@ run(OSDCommand().set_format_osd(1<<30)) # one to the highest allocated handle and return that, and do not # allocate and free so many that it would wrap. run(OSDCommand().set_create_partition(PVFS_OSD_DATA_PID)) -run(OSDCommand().set_create(PVFS_OSD_DATA_PID, datalb)) - -run(OSDCommand().set_create_partition(PVFS_OSD_META_PID)) +if osddirtype == "pvfs": + cid = COLLECTION_OID_LB + run(OSDCommand().set_create_collection(PVFS_OSD_DATA_PID, cid)) + run(OSDCommand().set_create_partition(PVFS_OSD_META_PID)) if metalb != root_handle: run(OSDCommand().set_create(PVFS_OSD_META_PID, metalb)) # create root handle, the top level directory if root_handle: - command = OSDCommand().set_create(PVFS_OSD_META_PID, root_handle) + if coll_object == "yes": + command = OSDCommand().set_create_collection(PVFS_OSD_META_PID, root_handle) + + else: + command = OSDCommand().set_create(PVFS_OSD_META_PID, root_handle) command.attr_build([ \ - OSDAttr(ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 0, 0), \ - OSDAttr(ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 1, 0), \ - OSDAttr(ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 2, 0777), \ - OSDAttr(ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 3, \ + OSDAttr(ATTR_SET, ANY_PG + PVFS_USEROBJECT_ATTR_PG, 0, 0), \ + OSDAttr(ATTR_SET, ANY_PG + PVFS_USEROBJECT_ATTR_PG, 1, 0), \ + OSDAttr(ATTR_SET, ANY_PG + PVFS_USEROBJECT_ATTR_PG, 2, 0777), \ + OSDAttr(ATTR_SET, ANY_PG + PVFS_USEROBJECT_ATTR_PG, 3, \ PVFS_ATTR_COMMON_ALL), \ - OSDAttr(ATTR_SET, PVFS_USEROBJECT_ATTR_PG, 4, \ + OSDAttr(ATTR_SET, ANY_PG + PVFS_USEROBJECT_ATTR_PG, 4, \ PVFS_TYPE_DIRECTORY), \ ]) run(command) diff --git a/osd-initiator/python/pvfs-osd-integrated-init.py b/osd-initiator/python/pvfs-osd-integrated-init.py index 6233548..712c6cf 100755 --- a/osd-initiator/python/pvfs-osd-integrated-init.py +++ b/osd-initiator/python/pvfs-osd-integrated-init.py @@ -40,7 +40,7 @@ pid = int(sys.argv[2]) drives = OSDDriveList() dev = False for d in drives: - if d.targetname == devname: + if d.hostname == devname: dev = OSDDevice(d.chardev) break if not dev: diff --git a/osd-initiator/sync.c b/osd-initiator/sync.c index f1e8f61..bc73e66 100644 --- a/osd-initiator/sync.c +++ b/osd-initiator/sync.c @@ -43,7 +43,7 @@ static int check_response(int ret, struct osd_command *command, osd_error("status: %u sense len: %u inlen: %zu", command->status, command->sense_len, command->inlen); osd_error("%s ", osd_sense_as_string(command->sense, - command->sense_len)); + command->sense_len)); } else if (command->inlen > 0) { osd_debug("Successfully performed task. BUF: <<%s>>", buf); @@ -777,7 +777,7 @@ int get_member_attributes(int fd, uint64_t pid, uint64_t cid) osd_debug("****** GET MEMBER ATTRIBUTES ******"); osd_debug("PID: %llu CID: %llu", llu(pid), llu(cid)); - osd_command_set_get_member_attributes(&command, pid, cid); + osd_command_set_get_member_attributes(&command, pid, cid, 0, 1); command.indata = buf; command.inlen_alloc = sizeof(buf); memset(buf, 0, sizeof(buf)); diff --git a/osd-initiator/tests/iospeed-mpi.c b/osd-initiator/tests/iospeed-mpi.c index bd9661c..070068c 100644 --- a/osd-initiator/tests/iospeed-mpi.c +++ b/osd-initiator/tests/iospeed-mpi.c @@ -120,7 +120,7 @@ static void read_bw(int fd, uint64_t pid, uint64_t oid, delta = end - start; rdtsc(start); - ret = flush_object(fd, pid, oid, 2); + ret = flush_object (fd, pid, oid, 0, 0, 0); rdtsc(end); assert(ret == 0); delta += (end - start); @@ -231,7 +231,7 @@ static void write_bw(int fd, uint64_t pid, uint64_t oid, rdtsc(start); /* XXX: this may be broken, numbers reported * are too big to be reasonable */ - ret = flush_object(fd, pid, oid, 2); + ret = flush_object(fd, pid, oid, 0, 0, 0); rdtsc(end); assert(ret == 0); diff --git a/osd-util/initiator b/osd-util/initiator index e128b39..c1fcd62 100755 --- a/osd-util/initiator +++ b/osd-util/initiator @@ -53,6 +53,10 @@ start() { if (( $# < 1 )) ; then usage ; fi fi + count=$# + last_arg=${@: -1} + complete=0 + # connect to each of the list of target hosts given while (( $# >= 1 )) ; do targethost=$1 @@ -68,8 +72,37 @@ start() { egrep ^$targetip: | sed -n 1p | cut -d\ -f2) else # hope it is the first/only IP we get back - targetname=$(iscsiadm -m discovery -t st -p $targethost |\ - sed -n 1p | cut -d\ -f2) + if [[ $count = 2 ]] ; then + targetname=$last_arg + fi + + if [[ $count = 1 ]] ; then + + ##length of one line + oneline=$(iscsiadm -m discovery -t st -p $targethost |\ + sed -n 1p) + whole=$(iscsiadm -m discovery -t st -p $targethost) + length=$((${#oneline})) + wholelength=$((${#whole})) + numtargets=$(($wholelength/$length)) + var="p" + counter=0 + for (( counter=1; counter<=numtargets; counter++ )) + do + targetname=$(iscsiadm -m discovery -t st -p $targethost |\ + sed -n $counter$var | cut -d\ -f2) + targetip=$(iscsiadm -m discovery -t st -p $targethost |\ + sed -n $counter$var | cut -d\ -f1) + # set transport type; must provide targetname here + iscsiadm -m node -p $targetip -T $targetname --op update \ + -n node.transport_name -v $transport_type + # login + iscsiadm -m node -p $targetip -T $targetname -l + done + complete=1 + fi + + if [[ -n $targetname ]] ; then targetip=$(iscsiadm -m discovery -t st -p $targethost |\ sed -n 1p | cut -d\ -f1) @@ -79,11 +112,18 @@ start() { echo "Could not connect to SCSI target on $targethost." >&2 continue fi - # set transport type; must provide targetname here - iscsiadm -m node -p $targetip -T $targetname --op update \ - -n node.transport_name -v $transport_type - # login - iscsiadm -m node -p $targetip -T $targetname -l + + if [[ $complete = 0 ]] ; then + # set transport type; must provide targetname here + iscsiadm -m node -p $targetip -T $targetname --op update \ + -n node.transport_name -v $transport_type + # login + iscsiadm -m node -p $targetip -T $targetname -l + fi + + if [[ $count = 2 ]] ; then + break + fi done } diff --git a/osd-util/perf.py b/osd-util/perf.py index 44258d8..6a4aed5 100755 --- a/osd-util/perf.py +++ b/osd-util/perf.py @@ -20,6 +20,7 @@ # along with this program. If not, see . import os +import fileinput import errno import sys import pwd @@ -32,6 +33,9 @@ options = { "osdtype": "none", "dirtype": "pvfs", "storage": "disk", + "post_create": "no", + "member_attr": "no", + "coll_object": "no", "meta_on_io": "no", "one_config_file" : "yes", "mirror": "", @@ -53,17 +57,23 @@ elif id == "alin": elif id == "ananth": osd_dir = "/home/ananth/osd" else: - osd_dir = os.environ["HOME"] + "/osd" + osd_dir = os.environ["HOME"] initiator = osd_dir + "/osd-util/initiator" -tgtd = osd_dir + "/stgt/tgtd" +tgtd = osd_dir + "/osc-osd/tgt/usr/otgtd" pvfs_init = osd_dir + "/osd-initiator/python/pvfs-init.py" pvfs_osd_integrated_init = osd_dir + "/osd-initiator/python/pvfs-osd-integrated-init.py" -allify_code = "/home/pw/bin/allify" +nodes_dir = osd_dir + "/node_list.txt" +targets_dir = osd_dir + "/target_list.txt" -# old location in target dir -if not os.access(tgtd, os.X_OK): - tgtd = osd_dir + "/osd-target/tgtd" +devname = [] +fd = open(targets_dir, "r") +while True: + line = fd.readline() + if line == "": + break + devname.append(line[:-1]) +fd.close() # # All the possibilities for -o and -d. @@ -112,6 +122,8 @@ nummeta = 1 numdir = 1 numconfig = 1 +#=====================================================USAGE=================================================================================== + def usage(): print >>sys.stderr, "Usage:", sys.argv[0], "[Options] start " print >>sys.stderr, " ", sys.argv[0], "[Options] restart []" @@ -134,6 +146,7 @@ def usage(): print >>sys.stderr, " -poi : for PVFS_OSD_INTEGRATED, connect to OSDs" print >>sys.stderr, " -rdma : connect to OSDs using iSER" sys.exit(1) +#=====================================================USAGE=================================================================================== # filenames testdir = os.environ["TMPDIR"] @@ -153,6 +166,7 @@ serverconf = testdir + "/server.conf" mirror_metanodes = "" mirror_ionodes = "" +#=================================================ALLBUT====================================================================================== def allbut(nodes, numservers): # keep 1 extra as a compute node if numservers > len(nodes): @@ -163,8 +177,9 @@ def allbut(nodes, numservers): # these are the compnodes, high nodes will be pvfs or osd servers return nodes[0:-numservers] +#=================================================ALLBUT====================================================================================== - +#================================================HANDLE_ALLOC================================================================================= def handle_alloc(): global options, osdnodes, pvfsnodes, metanodes, ionodes global datahandles, metahandles, roothandle, roothandle_node @@ -224,6 +239,9 @@ def handle_alloc(): h += metastep +#===============================================HANDLE_ALLOC============================================================= + +#===============================================BUILDFILES=============================================================== def buildfiles(): global numion, nummeta, numdir, options global nodes, compnodes, osdnodes, pvfsnodes, metanodes, ionodes @@ -251,9 +269,11 @@ def buildfiles(): # list of all nodes from PBS nodes = [] - fd = os.popen("uniq $PBS_NODEFILE") + fd = open(nodes_dir, "r") while True: line = fd.readline() + if line == "#compnodes#\n" or line == "#osdnodes#\n" or line == "#pvfsnodes#\n": + continue; if line == "": break nodes.append(line[:-1]) @@ -430,6 +450,9 @@ def buildfiles(): print >>fd, " ClientJobFlowTimeoutSecs 300" print >>fd, " ClientRetryLimit 5" print >>fd, " ClientRetryDelayMilliSecs 2000" + print >>fd, " PostCreate " + options["post_create"] + print >>fd, " MemberAttr " + options["member_attr"] + print >>fd, " CollectionObject " + options["coll_object"] if options["osdtype"] != "none": print >>fd, " OSDType " + options["osdtype"] if options["dirtype"] != "pvfs": @@ -441,15 +464,17 @@ def buildfiles(): print >>fd print >>fd, "" - for n in osdnodes: - print >>fd, " Alias", n, "osd://" + n for n in pvfsnodes: print >>fd, " Alias", n, options["protocol"] + "://" + n + ":" + \ str(port) + j = 0 + for n in osdnodes: + print >>fd, " Alias", n, "osd://" + devname[j] + j += 1 print >>fd, "" global pid - pid = 424242 + pid = 65536 print >>fd print >>fd, "" print >>fd, " Name pvfs2-fs" @@ -515,6 +540,9 @@ def buildfiles(): % (options["protocol"], roothandle_node, port, mountpoint) +#===================================================BUILDFILES================================================= + +#===================================================READ_ONE_FILE============================================== def read_one_file(filename): l = [] fd = open(filename) @@ -525,8 +553,9 @@ def read_one_file(filename): l.append(line[:-1]) fd.close() return l +#==================================================READ_ONE_FILE=============================================== - +#==================================================READFILES=================================================== def readfiles(): global options options = {} @@ -557,19 +586,9 @@ def readfiles(): numion = len(ionodes) if nummeta == -1: nummeta = len(metanodes) +#==========================================READFILES================================================ - -def allify(n): - (fdto, fdfrom) = os.popen2(allify_code) - for i in n: - print >>fdto, i - fdto.close() - s = fdfrom.read() - fdfrom.close() - s = s[:-1] - return s - - +#========================================START======================================================= def start(): # don't mess with certain classes of nodes if mirror mypvfsnodes = pvfsnodes @@ -581,7 +600,7 @@ def start(): mypvfsnodes = [x for x in mypvfsnodes if x not in metanodes] # sync files to pvfs nodes - for n in mypvfsnodes: + for n in nodes: if options["one_config_file"] == "yes": os.system("rcp " + testdir + "/fs.conf " + n + ":" + testdir) @@ -597,12 +616,13 @@ def start(): else: mountup = "" - os.system("all -p " + allify(osdnodes) + " " - + "cd " + testdir + " \; " - + "rm -rf /tmp/tgt-" + id + " \; " - + "mkdir /tmp/tgt-" + id + " \; " - + mountup - + tgtd + " -d 9 \< /dev/null \&\> tgtd.log") + for n in osdnodes: + os.system("'rsh " + n + " " + + "cd " + testdir + " \; " + + "rm -rf /tmp/tgt-" + id + " \; " + + "mkdir /tmp/tgt-" + id + " \; " + + mountup + + tgtd + " -d 9 \< /dev/null \&\> tgtd.log") if len(mypvfsnodes) > 0: if options["storage"] == "tmpfs": @@ -611,24 +631,26 @@ def start(): mountup = "" if options["one_config_file"] == "yes": - os.system("all -p " + allify(mypvfsnodes) + " " - + "cd " + testdir + " \; " - + "rm -rf " + testdir + "/storage \; " - + "mkdir " + testdir + "/storage \; " - + mountup - + "rm -f pvfs2.log \;" - + "pvfs2-server --mkfs fs.conf \;" - + "TZ=EST5EDT pvfs2-server fs.conf") + for n in mypvfsnodes: + os.system("rsh " + n + " " + + "cd " + testdir + " \; " + + "rm -rf " + testdir + "/storage \; " + + "mkdir " + testdir + "/storage \; " + + mountup + + "rm -f pvfs2.log \;" + + "pvfs2-server --mkfs fs.conf \;" + + "TZ=EST5EDT pvfs2-server fs.conf") else: - os.system("all -p " + allify(mypvfsnodes) + " " - + "cd " + testdir + " \; " - + "rm -rf " + testdir + "/storage \; " - + "mkdir " + testdir + "/storage \; " - + mountup - + "ln -fs server.conf-\$\(hostname\) server.conf \; " - + "rm -f pvfs2.log \;" - + "pvfs2-server --mkfs fs.conf server.conf \;" - + "TZ=EST5EDT pvfs2-server fs.conf server.conf") + for n in mypvfsnodes: + os.system("rsh " + n + " " + + "cd " + testdir + " \; " + + "rm -rf " + testdir + "/storage \; " + + "mkdir " + testdir + "/storage \; " + + mountup + + "ln -fs server.conf-\$\(hostname\) server.conf \; " + + "rm -f pvfs2.log \;" + + "pvfs2-server --mkfs fs.conf server.conf \;" + + "TZ=EST5EDT pvfs2-server fs.conf server.conf") # compnodes myosdnodes = osdnodes @@ -645,18 +667,22 @@ def start(): else: myibosdnodes = myosdnodes startcmd = "start" + stopcmd = "stop" if len(myibosdnodes) > 0: - os.system("all " + allify(compnodes) + " " - + "echo " + tabfile_contents + " \> " + tabfile + " \; " - + "sudo " + initiator + " " + startcmd + " " - + " ".join(myibosdnodes)) + for n in myibosdnodes: + os.system("rsh " + n + " " + + "echo " + tabfile_contents + " \> " + tabfile + " \; " + + "sudo " + initiator + " " + startcmd + " " + + n) + elif len(compnodes) > 0: - os.system("all -p " + allify(compnodes) + " " - + "echo " + tabfile_contents + " \> " + tabfile) + for n in compnodes: + os.system("rsh " + n + " " + + "echo " + tabfile_contents + " \> " + tabfile) else: # leave it on this starting machine at least - os.system("echo " + tabfile_contents + " > " + tabfile) + os.system("rsh " + n + " " + "echo " + tabfile_contents + " > " + tabfile) # format and initial pvfs layout on osd nodes. The machine where # this script runs must be one of the compnodes so it can talk to @@ -685,7 +711,9 @@ def start(): ret = os.system(pvfs_osd_integrated_init + " " + n + " " + str(pid)) if ret: print >>sys.stderr, pvfs_osd_integrated_init + " failed" +#===============================================START==================================== +#===============================================STATUS=================================== def status(): print "Protocol: ", options["protocol"] print "OSDType: ", options["osdtype"] @@ -694,6 +722,9 @@ def status(): print "Meta-on-io: ", options["meta_on_io"] print "One config file: ", options["one_config_file"] print "Mirror: ", options["mirror"] + print "Postcr: ", options["post_create"] + print "MemberAttr: ", options["member_attr"] + print "CollectionObject: ", options["coll_object"] n = filter(lambda x: x in metanodes, osdnodes) if len(n) > 0: @@ -715,8 +746,9 @@ def status(): print "Clients: ", " ".join(compnodes) print "export PVFS2TAB_FILE=" + tabfile +#======================================STATUS================================================= - +#======================================STOP=================================================== def stop(): # don't mess with io nodes if just a mirror mypvfsnodes = pvfsnodes @@ -725,29 +757,33 @@ def stop(): # compnodes if len(osdnodes) > 0: - os.system("all -p " + allify(compnodes) + " " - + "sudo " + initiator + " stop \; " - + "rm " + tabfile) + for n in compnodes: + os.system("rsh " + n + " " + + "sudo " + initiator + " stop \; " + + "rm " + tabfile) elif len(compnodes) > 0: - os.system("all -p " + allify(compnodes) + " " - + "rm " + tabfile) + for n in compnodes: + os.system("rsh " + n + " " + + "rm " + tabfile) else: os.system("rm " + tabfile) if len(osdnodes) > 0: - os.system("all -p " + allify(osdnodes) + " " - + "cd " + testdir + " \; " - + "killall -9 tgtd \; " - + "sleep 1 \; " - + "sudo umount /tmp/tgt-" + id + " 2>/dev/null \; " - + "rm -f tgtd.log") + for n in osdnodes: + os.system("rsh " + n + " " + + "cd " + testdir + " \; " + + "killall -9 tgtd \; " + + "sleep 1 \; " + + "sudo umount /tmp/tgt-" + id + " 2>/dev/null \; " + + "rm -f tgtd.log") if len(mypvfsnodes) > 0: - os.system("all -p " + allify(mypvfsnodes) + " " - + "cd " + testdir + " \; " - + "killall -9 pvfs2-server \; " - + "sleep 1 \; " - + "sudo umount " + testdir + "/storage 2>/dev/null \; " - + "rm -rf pvfs2.log fs.conf server.conf server.conf-\* storage") + for n in mypvfsnodes: + os.system("rsh " + n + " " + + "cd " + testdir + " \; " + + "killall -9 pvfs2-server \; " + + "sleep 1 \; " + + "sudo umount " + testdir + "/storage 2>/dev/null \; " + + "rm -rf pvfs2.log fs.conf server.conf server.conf-\* storage") for f in [fnodes, fcompnodes, fpvfsnodes, fosdnodes, fionodes, fmetanodes, fsconf, foptions]: @@ -763,7 +799,7 @@ def stop(): if not options["one_config_file"] == "yes": for n in mypvfsnodes: os.unlink(serverconf + "-" + n) - +#======================================================STOP========================================== i = 1 while i < len(sys.argv): @@ -822,6 +858,15 @@ while i < len(sys.argv): elif sys.argv[i] == "-poi": options["pvfs_osd_integrated"] = "yes" i += 1 + elif sys.argv[i] == "-poc": + options["post_create"] = "yes" + i += 1 + elif sys.argv[i] == "-gma": + options["member_attr"] = "yes" + i += 1 + elif sys.argv[i] == "-cad": + options["coll_object"] = "yes" + i += 1 elif sys.argv[i] == "-rdma": options["rdma"] = "yes" i += 1