Python testlib.save_output函数代码示例

    def run(self):

        # try reading various ranges (these are (start, end) absolute ranges, not offset/length)
        ranges = [
            (0, 16384),
            (1, 200),
            (0, 4096),
            (0, 8192),
            (0, 1000),
            (0, 6000),
            (100, 4000),
            (5000, 10000),
            (4096, 10000),
            (5000, 8192),
            (4096, 16834),
            (5000, 16384),
            (4096, 16000),
            (5000, 16000)
        ]

        self.exitcode = 0

        for (start, end) in ranges:

            if self.errors:
                break

            # only clear reader's cache
            testlib.clear_cache( self.config_dir, volume_id=self.volume_id, gateway_id=self.read_gateway_id )

            for i in xrange(0, 2):

                if self.errors:
                    break

                exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(self.config_dir, 'syndicate.conf'),
                                            '-u', testconf.SYNDICATE_ADMIN, '-v', self.volume_name, '-g', self.read_gateway_name,
                                            self.output_path, start, end - start, valgrind=True )

                out_name = "uncached"
                if i > 0:
                    out_name = "cached"

                testlib.save_output( self.output_dir, 'syndicate-read-thread-%s-%s-%s-%s' % (self.get_ident(), start, end, out_name), out )
                if exitcode != 0:
                    self.exitcode = exitcode
                    self.errormsg = "syndicate-read exit code %s on %s-%s-%s" % (exitcode, start, end, out_name)
                    break

                # correctness 
                if expected_data[start:end] not in out:
                    self.exitcode = -1
                    self.errormsg = "Thread %s missing data for %s-%s-%s" % (self.get_ident(), start, end, out_name)
                    break

            
            if self.exitcode != 0:
                break

        return

    path = '/setxattr-%s' % random_part
    attr_name_base = 'foo-%s' % random_part
    attr_value_base = 'bar-%s' % random_part

    config_dir, output_dir = testlib.test_setup()
    volume_name = testlib.add_test_volume( config_dir )

    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    random_part = hex(random.randint(0, 2**32-1))[2:]
   
    exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
                                path )

    testlib.save_output( output_dir, "syndicate-touch", out )

    if exitcode != 0:
        raise Exception("Failed to touch %s" % path)

    # do setxattr a few times
    # 1 attr
    exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                      '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,  
                                      path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )

    testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
    if exitcode != 0:
        raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))

    # 5 attrs at once

    for (start, end) in ranges:

        # only clear reader's cache
        testlib.clear_cache( config_dir, volume_id=volume_id, gateway_id=read_gateway_id )

        # do each read twice--once uncached, and one cached 
        for i in xrange(0, 2):
            exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                        '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
                                        output_path, start, end - start, valgrind=True )

            out_name = "uncached"
            if i > 0:
                out_name = "cached"

            testlib.save_output( output_dir, 'syndicate-read-%s-%s-%s' % (start, end, out_name), out )
            if exitcode != 0:
                stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
                raise Exception("%s exited %s" % (READ_PATH, exitcode))

            # correctness 
            if expected_data[start:end] not in out:
                stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
                raise Exception("Missing data for %s-%s" % (start, end))

    ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )

    testlib.save_output( output_dir, "syndicate-ag", ag_out )

    if ag_exitcode != 0:
        raise Exception("%s exited %s" % (AG_PATH, ag_exitcode))

        local_fd = open(local_path, "r")
        expected_data = local_fd.read()
        local_fd.close()

        local_paths.append(local_path)
        expected_datas.append(expected_data)

    # start AG
    AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
    ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
    time.sleep(20)

    if ag_proc.poll() is not None:
        ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
        testlib.save_output(output_dir, "syndicate-ag", ag_out)
        raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))

    # should cause the AG to get updated that there's a new gateway 
    cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    expected_data_error = False
    for i in xrange(0,len(local_paths)):
        # do each file twice 
        for j in xrange(0, 2):
            local_path = local_paths[i]
            expected_data = expected_datas[i]

            output_path = "/" + os.path.basename(local_path)
            exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', cat_gateway_name, output_path, valgrind=True )

    # put a file to publish
    local_paths = []
    for i in xrange(0, 1):
        local_path = testlib.make_random_file(16384, dir=inner_dir)
        local_paths.append(local_path)

    # start AG
    AG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "AG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, AG_gateway_name, "port=31112", "driver=%s" % AG_DRIVER )
    ag_proc, ag_out_path = testlib.start_gateway( config_dir, AG_PATH, testconf.SYNDICATE_ADMIN, volume_name, AG_gateway_name, valgrind=True )
    time.sleep(20)

    if ag_proc.poll() is not None:
        ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
        testlib.save_output(output_dir, "syndicate-ag", ag_out)
        raise Exception("%s exited %s" % (AG_PATH, ag_proc.poll()))

    # should cause the AG to get updated that there's a new gateway 
    ug_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    # remove inner dir, and have the AG refresh
    shutil.rmtree(rmdir)
    exitcode, out = testlib.run( REFRESH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', ug_gateway_name, '/to_remove_1', valgrind=True )
    testlib.save_output( output_dir, "syndicate-refresh", out )

    if exitcode != 0:
        ag_exitcode, ag_out = testlib.stop_gateway( ag_proc, ag_out_path )
        testlib.save_output(output_dir, "syndicate-ag", ag_out)
        raise Exception("syndicate-refresh exit code %s" % exitcode) 

    if not testlib.gateway_ping( 31112, 15 ):
        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # should cause the RG to get updated that there's a new gateway 
    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    cat_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email="ANONYMOUS" )

    random_part = hex(random.randint(0, 2**32-1))[2:]
    output_paths = []

    for i in xrange(0, NUM_FILES):
        output_path = "/put-%s-%s" % (random_part, i)
        output_paths.append(output_path)

        exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path, valgrind=True )
        testlib.save_output( output_dir, "syndicate-put-%s" % i, out )

        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (PUT_PATH, exitcode))

    for i in xrange(0, NUM_FILES):
        path = output_paths[i]
        exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', 'ANONYMOUS', '-v', volume_name, '-g', cat_gateway_name, path, valgrind=True )
        testlib.save_output( output_dir, 'syndicate-cat-%s' % i, out )
        
        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (CAT_PATH, exitcode)) 

        # check for correctnes 

    for (start, end) in ranges:

        # only clear reader's cache
        testlib.clear_cache( config_dir, volume_id=volume_id, gateway_id=read_gateway_id )

        # do each read twice--once uncached, and one cached 
        for i in xrange(0, 2):
            exitcode, out = testlib.run( READ_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                        '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
                                        output_path, start, end - start, valgrind=True )

            out_name = "uncached"
            if i > 0:
                out_name = "cached"

            testlib.save_output( output_dir, 'syndicate-read-%s-%s-%s' % (start, end, out_name), out )
            if exitcode != 0:
                stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
                raise Exception("%s exited %s" % (READ_PATH, exitcode))

            # correctness 
            if expected_data[start:end] not in out:
                stop_and_save( output_dir, ag_proc, ag_out_path, "syndicate-ag")
                raise Exception("Missing data for %s-%s" % (start, end))

    # finally, list it
    for p in ['/', '/dir1', '/dir1/dir2', output_path]:
        exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                     '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', read_gateway_name,
                                     p, valgrind=True )

    rg_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, rg_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )

    # start the RG 
    rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, rg_gateway_name )
    if not testlib.gateway_ping( 31112, 15 ):
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # put the file...
    exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
                                random_file_path, path )

    testlib.save_output( output_dir, "syndicate-put", out )

    if exitcode != 0:
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("Failed to touch %s" % path)

    # finish populating expected built-in xattrs 
    exitcode, out = testlib.run( STAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, "syndicate.conf"),
                                "-u", testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', getxattr_gateway_name,
                                path )

    testlib.save_output( output_dir, "syndicate-stat", out )
    if exitcode != 0:
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("Failed to stat %s" % path)

    attr_name_base = 'foo-%s' % random_part
    attr_value_base = 'bar-%s' % random_part
    expected_listxattr_output = ""

    config_dir, output_dir = testlib.test_setup()
    volume_name = testlib.add_test_volume( config_dir )

    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    random_part = hex(random.randint(0, 2**32-1))[2:]
   
    exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
                                path )

    testlib.save_output( output_dir, "syndicate-touch", out )

    if exitcode != 0:
        raise Exception("Failed to touch %s" % path)

    # do setxattr a few times
    # 1 attr
    exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                      '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,  
                                      path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )

    testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
    expected_listxattr_output += attr_name_base + "-1attr" + '\n'

    if exitcode != 0:
        raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))

    RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", caps="NONE", email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, RG_gateway_name, "port=31112", "driver=%s" % RG_DRIVER )

    rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name )
    time.sleep(1)
    if rg_proc.poll() is not None:
        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # should cause the RG to get updated that there's a new gateway 
    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    random_part = hex(random.randint(0, 2**32-1))[2:]
    output_path = "/put-%s" % random_part
    exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path )

    testlib.save_output( output_dir, "syndicate-put", out )

    if exitcode != 0:
        raise Exception("%s exited %s" % (PUT_PATH, exitcode))

    # read the file, to populate the cache 
    exitcode, out = testlib.run( CAT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                 '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
                                 output_path, valgrind=True )

    if exitcode != 0:
        rg_exitcode, rg_out = testlib.stop_gateway( rg_proc, rg_out_path )
        testlib.save_output( output_dir, "syndicate-rg", rg_out )
        raise Exception("Failed to read %s" % path)

    # check cache 

    expected_data = local_fd.read()
    local_fd.close()

    # start up RG
    RG_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "RG", port=32110, email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, RG_gateway_name, "driver=%s" % RG_DRIVER )

    rg_proc, rg_out_path = testlib.start_gateway( config_dir, RG_PATH, testconf.SYNDICATE_ADMIN, volume_name, RG_gateway_name, valgrind=True )
    time.sleep(5)
    if rg_proc.poll() is not None:
        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # put the file
    put_gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", port=31110, email=testconf.SYNDICATE_ADMIN )
    exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', put_gateway_name, local_path, output_path, valgrind=True )
    testlib.save_output( output_dir, "syndicate-put", out )

    if exitcode != 0:
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("syndicate-put exit code %s" % exitcode)

    # create separate threads for UGs 
    for i in xrange(0, num_threads):
        t = UGThread( config_dir, output_dir, output_path, volume_name, caps="ALL", port=(31111 + i + 1), email=testconf.SYNDICATE_ADMIN )
        threads.append(t)

    # run all threads 
    for i in xrange(0, num_threads):
        threads[i].start()

    # wait for them to finish 

    if not testlib.gateway_ping( 31112, 15 ):
        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # should cause the RG to get updated that there's a new gateway 
    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    # touch $NUM_FILES files 
    random_part = hex(random.randint(0, 2**32-1))[2:]
    expected_paths = []

    for i in xrange(0, NUM_FILES):
        output_path = "/touch-%s-%s" % (random_part, i)
        expected_paths.append(output_path)

        exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, output_path )
        testlib.save_output( output_dir, "syndicate-touch-%s" % i, out )
        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))

    # list them 
    exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, '/' )
    testlib.save_output( output_dir, "syndicate-ls", out )
    if exitcode != 0:
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("%s exited %s" % (LS_PATH, exitcode))

    # verify that they're all there 
    for path in expected_paths:
        if path.strip("/") not in out:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")

    # look up RG 
    rg_gateway_info = testlib.read_gateway( config_dir, RG_gateway_name )
    rg_gateway_id = rg_gateway_info['g_id']

    volume_info = testlib.read_volume( config_dir, volume_name )
    volume_id = volume_info['volume_id']

    random_part = hex(random.randint(0, 2**32-1))[2:]
    output_paths = []

    for i in xrange(0, NUM_FILES):
        output_path = "/put-%s-%s" % (random_part, i)
        output_paths.append(output_path)

        exitcode, out = testlib.run( PUT_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, local_path, output_path, valgrind=True )
        testlib.save_output( output_dir, "syndicate-put-%s" % i, out )

        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (PUT_PATH, exitcode))

    # make target directory
    exitcode, out = testlib.run( MKDIR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, "/newdir", valgrind=True )
    testlib.save_output( output_dir, "syndicate-mkdir", out )

    if exitcode != 0:
        stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
        raise Exception("%s exited %s" % (PUT_PATH, exitcode))

    # do several renames
    for j in xrange(0, NUM_RENAMES):

    attr_value_base = 'bar-%s' % random_part

    config_dir, output_dir = testlib.test_setup()
    volume_name = testlib.add_test_volume( config_dir )

    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )
    gateway_client_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL",  email=testconf.SYNDICATE_ADMIN )
    testlib.update_gateway( config_dir, gateway_client_name, "port=31112" )

    random_part = hex(random.randint(0, 2**32-1))[2:]
   
    exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,
                                path )

    testlib.save_output( output_dir, "syndicate-touch", out )

    if exitcode != 0:
        raise Exception("Failed to touch %s" % path)

    # do setxattr a few times
    # 1 attr
    exitcode, out_1attr = testlib.run( SETXATTR_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'),
                                      '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name,  
                                      path, attr_name_base + "-1attr", attr_value_base + "-1attr", valgrind=True )

    testlib.save_output( output_dir, "syndicate-setxattr-1attr", out_1attr )
    if exitcode != 0:
        raise Exception("%s exited %s" % (SETXATTR_PATH, exitcode))

    # 5 attrs at once

        raise Exception("%s exited %s" % (RG_PATH, rg_proc.poll()))

    # should cause the RG to get updated that there's a new gateway 
    gateway_name = testlib.add_test_gateway( config_dir, volume_name, "UG", caps="ALL", email=testconf.SYNDICATE_ADMIN )

    for rep in xrange(0, NUM_REPS):
        # touch $NUM_FILES files 
        random_part = hex(random.randint(0, 2**32-1))[2:]
        expected_paths = []

        for i in xrange(0, NUM_FILES):
            output_path = "/touch-%s-%s" % (random_part, i)
            expected_paths.append(output_path)

        exitcode, out = testlib.run( TOUCH_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, *expected_paths )
        testlib.save_output( output_dir, "syndicate-touch-%s" % rep, out )
        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (TOUCH_PATH, exitcode))

        # list them 
        exitcode, out = testlib.run( LS_PATH, '-d2', '-f', '-c', os.path.join(config_dir, 'syndicate.conf'), '-u', testconf.SYNDICATE_ADMIN, '-v', volume_name, '-g', gateway_name, '/' )
        testlib.save_output( output_dir, "syndicate-ls-full-%s" % rep, out )
        if exitcode != 0:
            stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")
            raise Exception("%s exited %s" % (LS_PATH, exitcode))

        # verify that they're all there 
        for path in expected_paths:
            if path.strip("/") not in out:
                stop_and_save( output_dir, rg_proc, rg_out_path, "syndicate-rg")