Python ec_iface.ECDriver类代码示例

class Eraser(object):
    """A wrapper for pyeclib erasure coding driver (ECDriver)"""
    def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
        self.k = k
        self.m = m
        self.ec_type = ec_type
        if EC_TYPE == "pylonghair":
            self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
        elif EC_TYPE == "striping" or EC_TYPE == "bypass":
            self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
        else:
            self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

    def encode(self, data):
        """Encode a string of bytes in flattened string of byte strips"""
        raw_strips = self.driver.encode(data)
        strips = []
        for raw_strip in raw_strips:
            strip = Strip()
            strip.data = raw_strip
            strips.append(strip)
        return strips

    def decode(self, playcloud_strips):
        """Decode byte strips in a string of bytes"""
        strips = convert_strips_to_bytes_list(playcloud_strips)
        return self.driver.decode(strips)

class ECStorageMethod(StorageMethod):
    def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
                 ec_nb_parity):
        super(ECStorageMethod, self).__init__(name=name, ec=True)

        try:
            self._ec_nb_data = int(ec_nb_data)
        except (TypeError, ValueError):
            raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
                                           ec_nb_data)

        try:
            self._ec_nb_parity = int(ec_nb_parity)
        except (TypeError, ValueError):
            raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
                                           ec_nb_parity)

        self._ec_segment_size = ec_segment_size
        self._ec_type = ec_type
        self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
                               ec_type=ec_type_to_pyeclib_type[ec_type])
        self._ec_quorum_size = \
            self._ec_nb_data + self.driver.min_parity_fragments_needed()

    @property
    def quorum(self):
        return self._ec_quorum_size

    @classmethod
    def build(cls, params):
        ec_nb_data = params.pop('k')
        ec_nb_parity = params.pop('m')
        ec_type = params.pop('algo')
        return cls('ec', ec_segment_size=EC_SEGMENT_SIZE,
                   ec_type=ec_type, ec_nb_data=ec_nb_data,
                   ec_nb_parity=ec_nb_parity)

    @property
    def ec_type(self):
        return self._ec_type

    @property
    def ec_nb_data(self):
        return self._ec_nb_data

    @property
    def ec_nb_parity(self):
        return self._ec_nb_parity

    @property
    def ec_segment_size(self):
        return self._ec_segment_size

    @property
    def ec_fragment_size(self):
        return self.driver.get_segment_info(
            self.ec_segment_size, self.ec_segment_size)['fragment_size']

 def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
     self.k = k
     self.m = m
     self.ec_type = ec_type
     if EC_TYPE == "pylonghair":
         self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
     elif EC_TYPE == "striping" or EC_TYPE == "bypass":
         self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
     else:
         self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

class TestNullDriver(unittest.TestCase):
    def setUp(self):
        self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode("")
        self.null_driver.decode([])

    def check_metadata_formatted(self, k, m, ec_type, chksum_type):

        if ec_type not in VALID_EC_TYPES:
            return

        filesize = 1024 * 1024 * 3
        file_str = ''.join(random.choice(ascii_letters)
                           for i in range(filesize))
        file_bytes = file_str.encode('utf-8')

        pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
                                  chksum_type=chksum_type)

        fragments = pyeclib_driver.encode(file_bytes)

        f = 0
        for fragment in fragments:
            metadata = pyeclib_driver.get_metadata(fragment, 1)
            if 'index' in metadata:
                self.assertEqual(metadata['index'], f)
            else:
                self.assertTrue(False)

            if 'chksum_mismatch' in metadata:
                self.assertEqual(metadata['chksum_mismatch'], 0)
            else:
                self.assertTrue(False)

            if 'backend_id' in metadata:
                self.assertEqual(metadata['backend_id'], ec_type)
            else:
                self.assertTrue(False)

            if 'orig_data_size' in metadata:
                self.assertEqual(metadata['orig_data_size'], 3145728)
            else:
                self.assertTrue(False)

            if 'chksum_type' in metadata:
                self.assertEqual(metadata['chksum_type'], 'crc32')
            else:
                self.assertTrue(False)

            if 'backend_version' not in metadata:
                self.assertTrue(False)

            if 'chksum' not in metadata:
                self.assertTrue(False)

            if 'size' not in metadata:
                self.assertTrue(False)

            f += 1

    def check_metadata_formatted(self, k, m, ec_type, chksum_type):

        if ec_type not in VALID_EC_TYPES:
            return

        filesize = 1024 * 1024 * 3
        file_str = "".join(random.choice(ascii_letters) for i in range(filesize))
        file_bytes = file_str.encode("utf-8")

        pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)

        fragments = pyeclib_driver.encode(file_bytes)

        f = 0
        for fragment in fragments:
            metadata = pyeclib_driver.get_metadata(fragment, 1)
            if "index" in metadata:
                self.assertEqual(metadata["index"], f)
            else:
                self.assertTrue(False)

            if "chksum_mismatch" in metadata:
                self.assertEqual(metadata["chksum_mismatch"], 0)
            else:
                self.assertTrue(False)

            if "backend_id" in metadata:
                self.assertEqual(metadata["backend_id"], ec_type)
            else:
                self.assertTrue(False)

            if "orig_data_size" in metadata:
                self.assertEqual(metadata["orig_data_size"], 3145728)
            else:
                self.assertTrue(False)

            if "chksum_type" in metadata:
                self.assertEqual(metadata["chksum_type"], "crc32")
            else:
                self.assertTrue(False)

            if "backend_version" not in metadata:
                self.assertTrue(False)

            if "chksum" not in metadata:
                self.assertTrue(False)

            if "size" not in metadata:
                self.assertTrue(False)

            f += 1

    def test_get_metadata_formatted(self):
        pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
        
        filesize = 1024 * 1024 * 3
        file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
        file_bytes = file_str.encode('utf-8')
        
        fragments = pyeclib_driver.encode(file_bytes)

        i = 0
        for fragment in fragments:
          metadata = pyeclib_driver.get_metadata(fragment, 1)
          if metadata.has_key('index'):
            self.assertEqual(metadata['index'], i)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('chksum_mismatch'):
            self.assertEqual(metadata['chksum_mismatch'], 0)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('backend_id'):
            self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
          else:
            self.assertTrue(false)
          
          if metadata.has_key('orig_data_size'):
            self.assertEqual(metadata['orig_data_size'], 3145728)
          else:
            self.assertTrue(false)
          
          if metadata.has_key('chksum_type'):
            self.assertEqual(metadata['chksum_type'], 'crc32')
          else:
            self.assertTrue(false)
          
          if not metadata.has_key('backend_version'):
            self.assertTrue(false)
          
          if not metadata.has_key('chksum'):
            self.assertTrue(false)

          if not metadata.has_key('size'):
            self.assertTrue(false)

          i += 1

def encode(k,m,data_to_encode,base_filename_to_store,ec_type) :

	home = expanduser("~")	
	key_list = base_filename_to_store.split('.')
	Disk_base_dir = home + '/Disk'
	jencoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]


	# Create a replicated copy first.
	#replica_dir = Disk_base_dir + "/Disk_0"
	#if not os.path.exists(replica_dir):
	#	os.makedirs(replica_dir)


	#with open(replica_dir + "/" + base_filename_to_store,'wb') as fp :
	#	fp.write(data_to_encode)
	#fp.close()


	if ec_type in jencoder_types :
		
		jencoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jencoder.so')
		w = 8
		key = key_list[0]
		jencoder.encode(key,data_to_encode,Disk_base_dir,k,m,ec_type,w,0)
		

	else :
		ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)


		# encode
		fragments = ec_driver.encode(data_to_encode)

		
		# store
		i = 1
		for fragment in fragments:
			fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i)
			if not os.path.exists(fragment_dir):
				os.makedirs(fragment_dir)
	
		 	with open("%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp:
				fp.write(fragment)
			fp.close()
			i += 1

def decode(k,m,base_filename_to_decode,ec_type) :

	#print("k = %d, m = %d" % (k, m))
	#print("ec_type = %s" % ec_type)
	home = expanduser("~")	
	Disk_base_dir = home + '/Disk'
	key_list = base_filename_to_decode.split('.')
	jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
	if ec_type in jdecoder_types :
		print "Decoding started "
		jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
		key = key_list[0]
		decoded_data = str(jdecoder.decode(key,Disk_base_dir))
		print "decoded data = ", decoded_data
	else :
		ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)

		fragment_list = []
	
	
		i = 0
		while i <=k :
			if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) :
				curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
				if i == 0 :
					if os.path.exists(curr_dir + "/" + base_filename_to_decode) :
						with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp :
							decoded_data = fp.read()					
						fp.close()
						return decoded_data
				else :
					if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) :
						with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp :
							fragment = fp.read()					
						fp.close()
						fragment_list.append(fragment)
					
			i = i + 1
	
		if len(fragment_list) < k :
			return -1 # Not enough fragments to decode
		else :
			
			decoded_data = ec_driver.decode(fragment_list)	
	return decoded_data	

    def test_ec_fragment_size_cached(self):
        policy = ECStoragePolicy(
            0, 'ec2-1', ec_type=DEFAULT_TEST_EC_TYPE,
            ec_ndata=2, ec_nparity=1, object_ring=FakeRing(replicas=3),
            ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, is_default=True)

        ec_driver = ECDriver(ec_type=DEFAULT_TEST_EC_TYPE,
                             k=2, m=1)
        expected_fragment_size = ec_driver.get_segment_info(
            DEFAULT_EC_OBJECT_SEGMENT_SIZE,
            DEFAULT_EC_OBJECT_SEGMENT_SIZE)['fragment_size']

        with mock.patch.object(
                policy.pyeclib_driver, 'get_segment_info') as fake:
            fake.return_value = {
                'fragment_size': expected_fragment_size}

            for x in range(10):
                self.assertEqual(expected_fragment_size,
                                 policy.fragment_size)
                # pyeclib_driver.get_segment_info is called only once
                self.assertEqual(1, fake.call_count)

    def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
                 ec_nb_parity):
        super(ECStorageMethod, self).__init__(name=name, ec=True)

        try:
            self._ec_nb_data = int(ec_nb_data)
        except (TypeError, ValueError):
            raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
                                           ec_nb_data)

        try:
            self._ec_nb_parity = int(ec_nb_parity)
        except (TypeError, ValueError):
            raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
                                           ec_nb_parity)

        self._ec_segment_size = ec_segment_size
        self._ec_type = ec_type
        self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
                               ec_type=ec_type_to_pyeclib_type[ec_type])
        self._ec_quorum_size = \
            self._ec_nb_data + self.driver.min_parity_fragments_needed()

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse

parser = argparse.ArgumentParser(
    description='PyECLib tool to determine fragment indexes needed to '
                'recover missing fragments.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
                    nargs='+', help='missing_fragments')

args = parser.parse_args()

ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)

fragments_needed = ec_driver.fragments_needed(args.missing_fragments)

print(fragments_needed)

parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('file_dir', help='directory with the file')
parser.add_argument('filename', help='file to encode')
parser.add_argument('fragment_dir', help='directory to drop encoded fragments')

args = parser.parse_args()

print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)


start = time.time()

ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)

# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
    whole_file_str = fp.read()

# encode
fragments = ec_driver.encode(whole_file_str)

# store
i = 0
for fragment in fragments:
    with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
        fp.write(fragment)
    i += 1

 def setUp(self):
     self.null_driver = ECDriver(
         library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)

class ECStoragePolicy(BaseStoragePolicy):
    """
    Represents a storage policy of type 'erasure_coding'.

    Not meant to be instantiated directly; use
    :func:`~swift.common.storage_policy.reload_storage_policies` to load
    POLICIES from ``swift.conf``.
    """

    def __init__(self, idx, name='', aliases='', is_default=False,
                 is_deprecated=False, object_ring=None,
                 ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
                 ec_type=None, ec_ndata=None, ec_nparity=None):

        super(ECStoragePolicy, self).__init__(
            idx=idx, name=name, aliases=aliases, is_default=is_default,
            is_deprecated=is_deprecated, object_ring=object_ring)

        # Validate erasure_coding policy specific members
        # ec_type is one of the EC implementations supported by PyEClib
        if ec_type is None:
            raise PolicyError('Missing ec_type')
        if ec_type not in VALID_EC_TYPES:
            raise PolicyError('Wrong ec_type %s for policy %s, should be one'
                              ' of "%s"' % (ec_type, self.name,
                                            ', '.join(VALID_EC_TYPES)))
        self._ec_type = ec_type

        # Define _ec_ndata as the number of EC data fragments
        # Accessible as the property "ec_ndata"
        try:
            value = int(ec_ndata)
            if value <= 0:
                raise ValueError
            self._ec_ndata = value
        except (TypeError, ValueError):
            raise PolicyError('Invalid ec_num_data_fragments %r' %
                              ec_ndata, index=self.idx)

        # Define _ec_nparity as the number of EC parity fragments
        # Accessible as the property "ec_nparity"
        try:
            value = int(ec_nparity)
            if value <= 0:
                raise ValueError
            self._ec_nparity = value
        except (TypeError, ValueError):
            raise PolicyError('Invalid ec_num_parity_fragments %r'
                              % ec_nparity, index=self.idx)

        # Define _ec_segment_size as the encode segment unit size
        # Accessible as the property "ec_segment_size"
        try:
            value = int(ec_segment_size)
            if value <= 0:
                raise ValueError
            self._ec_segment_size = value
        except (TypeError, ValueError):
            raise PolicyError('Invalid ec_object_segment_size %r' %
                              ec_segment_size, index=self.idx)

        # Initialize PyECLib EC backend
        try:
            self.pyeclib_driver = \
                ECDriver(k=self._ec_ndata, m=self._ec_nparity,
                         ec_type=self._ec_type)
        except ECDriverError as e:
            raise PolicyError("Error creating EC policy (%s)" % e,
                              index=self.idx)

        # quorum size in the EC case depends on the choice of EC scheme.
        self._ec_quorum_size = \
            self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()

    @property
    def ec_type(self):
        return self._ec_type

    @property
    def ec_ndata(self):
        return self._ec_ndata

    @property
    def ec_nparity(self):
        return self._ec_nparity

    @property
    def ec_segment_size(self):
        return self._ec_segment_size

    @property
    def fragment_size(self):
        """
        Maximum length of a fragment, including header.

        NB: a fragment archive is a sequence of 0 or more max-length
        fragments followed by one possibly-shorter fragment.
        """
        # Technically pyeclib's get_segment_info signature calls for
        # (data_len, segment_size) but on a ranged GET we don't know the
#.........这里部分代码省略.........