Python ECDriver.encode方法代码示例

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
class TestNullDriver(unittest.TestCase):
    def setUp(self):
        self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode("")
        self.null_driver.decode([])

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
class Eraser(object):
    """A wrapper for pyeclib erasure coding driver (ECDriver)"""
    def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
        self.k = k
        self.m = m
        self.ec_type = ec_type
        if EC_TYPE == "pylonghair":
            self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
        elif EC_TYPE == "striping" or EC_TYPE == "bypass":
            self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
        else:
            self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

    def encode(self, data):
        """Encode a string of bytes in flattened string of byte strips"""
        raw_strips = self.driver.encode(data)
        strips = []
        for raw_strip in raw_strips:
            strip = Strip()
            strip.data = raw_strip
            strips.append(strip)
        return strips

    def decode(self, playcloud_strips):
        """Decode byte strips in a string of bytes"""
        strips = convert_strips_to_bytes_list(playcloud_strips)
        return self.driver.decode(strips)

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
    def check_metadata_formatted(self, k, m, ec_type, chksum_type):

        if ec_type not in VALID_EC_TYPES:
            return

        filesize = 1024 * 1024 * 3
        file_str = ''.join(random.choice(ascii_letters)
                           for i in range(filesize))
        file_bytes = file_str.encode('utf-8')

        pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
                                  chksum_type=chksum_type)

        fragments = pyeclib_driver.encode(file_bytes)

        f = 0
        for fragment in fragments:
            metadata = pyeclib_driver.get_metadata(fragment, 1)
            if 'index' in metadata:
                self.assertEqual(metadata['index'], f)
            else:
                self.assertTrue(False)

            if 'chksum_mismatch' in metadata:
                self.assertEqual(metadata['chksum_mismatch'], 0)
            else:
                self.assertTrue(False)

            if 'backend_id' in metadata:
                self.assertEqual(metadata['backend_id'], ec_type)
            else:
                self.assertTrue(False)

            if 'orig_data_size' in metadata:
                self.assertEqual(metadata['orig_data_size'], 3145728)
            else:
                self.assertTrue(False)

            if 'chksum_type' in metadata:
                self.assertEqual(metadata['chksum_type'], 'crc32')
            else:
                self.assertTrue(False)

            if 'backend_version' not in metadata:
                self.assertTrue(False)

            if 'chksum' not in metadata:
                self.assertTrue(False)

            if 'size' not in metadata:
                self.assertTrue(False)

            f += 1

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
    def check_metadata_formatted(self, k, m, ec_type, chksum_type):

        if ec_type not in VALID_EC_TYPES:
            return

        filesize = 1024 * 1024 * 3
        file_str = "".join(random.choice(ascii_letters) for i in range(filesize))
        file_bytes = file_str.encode("utf-8")

        pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)

        fragments = pyeclib_driver.encode(file_bytes)

        f = 0
        for fragment in fragments:
            metadata = pyeclib_driver.get_metadata(fragment, 1)
            if "index" in metadata:
                self.assertEqual(metadata["index"], f)
            else:
                self.assertTrue(False)

            if "chksum_mismatch" in metadata:
                self.assertEqual(metadata["chksum_mismatch"], 0)
            else:
                self.assertTrue(False)

            if "backend_id" in metadata:
                self.assertEqual(metadata["backend_id"], ec_type)
            else:
                self.assertTrue(False)

            if "orig_data_size" in metadata:
                self.assertEqual(metadata["orig_data_size"], 3145728)
            else:
                self.assertTrue(False)

            if "chksum_type" in metadata:
                self.assertEqual(metadata["chksum_type"], "crc32")
            else:
                self.assertTrue(False)

            if "backend_version" not in metadata:
                self.assertTrue(False)

            if "chksum" not in metadata:
                self.assertTrue(False)

            if "size" not in metadata:
                self.assertTrue(False)

            f += 1

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
    def test_get_metadata_formatted(self):
        pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
        
        filesize = 1024 * 1024 * 3
        file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
        file_bytes = file_str.encode('utf-8')
        
        fragments = pyeclib_driver.encode(file_bytes)

        i = 0
        for fragment in fragments:
          metadata = pyeclib_driver.get_metadata(fragment, 1)
          if metadata.has_key('index'):
            self.assertEqual(metadata['index'], i)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('chksum_mismatch'):
            self.assertEqual(metadata['chksum_mismatch'], 0)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('backend_id'):
            self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
          else:
            self.assertTrue(false)
          
          if metadata.has_key('orig_data_size'):
            self.assertEqual(metadata['orig_data_size'], 3145728)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('chksum_type'):
            self.assertEqual(metadata['chksum_type'], 'crc32')
          else:
            self.assertTrue(false)
          
          if not metadata.has_key('backend_version'):
            self.assertTrue(false)
          
          if not metadata.has_key('chksum'):
            self.assertTrue(false)

          if not metadata.has_key('size'):
            self.assertTrue(false)

          i += 1

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
def encode(k,m,data_to_encode,base_filename_to_store,ec_type) :

	home = expanduser("~")	
	key_list = base_filename_to_store.split('.')
	Disk_base_dir = home + '/Disk'
	jencoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]


	# Create a replicated copy first.
	#replica_dir = Disk_base_dir + "/Disk_0"
	#if not os.path.exists(replica_dir):
	#	os.makedirs(replica_dir)


	#with open(replica_dir + "/" + base_filename_to_store,'wb') as fp :
	#	fp.write(data_to_encode)
	#fp.close()


	if ec_type in jencoder_types :
		
		jencoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jencoder.so')
		w = 8
		key = key_list[0]
		jencoder.encode(key,data_to_encode,Disk_base_dir,k,m,ec_type,w,0)
		

	else :
		ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)


		# encode
		fragments = ec_driver.encode(data_to_encode)

		
		# store
		i = 1
		for fragment in fragments:
			fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i)
			if not os.path.exists(fragment_dir):
				os.makedirs(fragment_dir)
	
		 	with open("%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp:
				fp.write(fragment)
			fp.close()
			i += 1

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
    # Generate a new string for each test
    file_str = ''.join(
        random.choice(
            string.ascii_uppercase + string.digits) for x in range(args.s))

    try:
        ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
    except Exception as e:
        print("Scheme %s is not defined (%s)." % (scheme, e))
        continue

    timer.start()

    for i in range(num_iterations):
        ec_driver.encode(file_str)

    duration = timer.stop_and_return()

    results.append((scheme, duration))

    timer.reset()

print(results)
results.sort(lambda x, y: (int)((1000 * x[1]) - (1000 * y[1])))

for i in range(len(results)):
    if i > return_limit:
        break

    print("\n\nPerf Rank #%d:" % i)

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
    print "Arguments:"
    print "\tsize                        Size of the payload to encode in bytes"

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print_usage()
        sys.exit(0)
    SIZE = int(sys.argv[1])
    EC_K = int(os.environ.get("EC_K", 10))
    EC_M = int(os.environ.get("EC_M", 4))
    EC_TYPE = os.environ.get("EC_TYPE", "liberasurecode_rs_vand")

    DRIVER = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

    DATA = os.urandom(SIZE)
    STRIPS = DRIVER.encode(DATA)
    LENGTH = EC_K + EC_M
    SUPPORTED_DISTANCE = LENGTH - EC_K + 1
    print "About to reconstruct ", REQUESTS, " times a payload of size ", SIZE, " bytes (", \
    (DRIVER.ec_type if hasattr(DRIVER, "ec_type") else EC_TYPE), ", k =", DRIVER.k, \
    ", m =", DRIVER.m, ") from 0 to", SUPPORTED_DISTANCE, "missing blocks"

    random.seed(0)

    for missing_blocks in range(SUPPORTED_DISTANCE):
        for i in range(REQUESTS):
            missing_indices = range(missing_blocks)
            start = time.clock()
            DRIVER.reconstruct(STRIPS[missing_blocks:], missing_indices)
            end = time.clock()
            elapsed_in_milliseconds = (end - start) * 1000

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
from pyeclib.ec_iface import ECDriver
import pickle


if __name__ == '__main__':
    ec_pattern = [(6, 3), (10, 4), (20, 4), (11, 7)]

    results_dict = {}
    for k, m in ec_pattern:
        driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m)
        results_dict[(k, m)] = driver.encode('a' * 100)

    with open('results.pkl', 'wb') as f:
        pickle.dump(results_dict, f)

# 需要导入模块: from pyeclib.ec_iface import ECDriver [as 别名]
# 或者: from pyeclib.ec_iface.ECDriver import encode [as 别名]
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)


start = time.time()

ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)

# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
    whole_file_str = fp.read()

# encode
fragments = ec_driver.encode(whole_file_str)

# store
i = 0
for fragment in fragments:
    with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
        fp.write(fragment)
    i += 1


done = time.time()
elapsed = done - start

print(" encoding time : " + str(elapsed) + "seconds")
print(" DONE!!!\n\n")