Python compilelock.get_lock函数代码示例

def convnet_compile():
    # Compile .cu files in cuda_convnet
    _logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults))
    import time

    t1 = time.time()
    if should_recompile():
        _logger.debug("should recompile")

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in cuda_convnet_file_sources:
            code.append(open(os.path.join(this_dir, source_file)).read())
        code = "\n".join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug("recompiling")

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(cuda_convnet_loc):
                        os.makedirs(cuda_convnet_loc)
                    compiler.compile_str(
                        "cuda_convnet",
                        code,
                        location=cuda_convnet_loc,
                        include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults
                        + [cuda_convnet_loc]
                        + ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []),
                        libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"],
                        preargs=["-O3"] + args,
                        py_module=False,
                    )
                except Exception, e:
                    _logger.error(
                        "Failed to compile %s %s: %s",
                        os.path.join(cuda_convnet_loc, "mod.cu"),
                        cuda_convnet_file_sources,
                        str(e),
                    )
                    return False
            else:
                _logger.debug("already compiled by another process")

    def get_writelock(self, filename):
        """
        Obtain a writelock on a file.
        Only one write lock may be held at any given time.
        Parameters
        ----------
        filename : string
            Name of the file on which to obtain a writelock
        """

        # compilelock expect locks to be on folder. Since we want a lock on a
        # file, we will have to ask compilelock for a folder with a different
        # name from the file we want a lock on or else compilelock will
        # try to create a folder with the same name as the file
        compilelock.get_lock(filename + ".writelock")

def convnet_compile():
    # Compile .cu files in cuda_convnet
    _logger.debug('nvcc_compiler.rpath_defaults: %s',
            str(nvcc_compiler.rpath_defaults))
    import time
    t1 = time.time()
    if should_recompile():
        _logger.debug('should recompile')

        # Concatenate all .cu files into one big mod.cu
        code = []
        for file_root in cuda_convnet_file_roots:
            source_file = file_root + '.cu'
            code.append(open(os.path.join(this_dir, source_file)).read())
        code = '\n'.join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug('recompiling')

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(cuda_convnet_loc):
                        os.makedirs(cuda_convnet_loc)
                    compiler.compile_str('cuda_convnet',
                            code,
                            location=cuda_convnet_loc,
                            include_dirs=[this_dir],
                            lib_dirs=nvcc_compiler.rpath_defaults,  # ???
                            libs=['cublas'],
                            preargs=['-O3'] + args,
                            py_module=False)
                except Exception, e:
                    _logger.error("Failed to compile %s.cu: %s",
                                  file_root, str(e))
                    return False
            else:
                _logger.debug('already compiled by another process')

    # will be imported and compile_str won't get called at all.
    sys.path.insert(0, config.compiledir)
    location = os.path.join(config.compiledir, 'cutils_ext')
    if not os.path.exists(location):
        try:
            os.mkdir(location)
        except OSError as e:
            assert e.errno == errno.EEXIST
            assert os.path.exists(location), location
    if not os.path.exists(os.path.join(location, '__init__.py')):
        open(os.path.join(location, '__init__.py'), 'w').close()

    try:
        from cutils_ext.cutils_ext import *  # noqa
    except ImportError:
        get_lock()
    # Ensure no-one else is currently modifying the content of the compilation
    # directory. This is important to prevent multiple processes from trying to
    # compile the cutils_ext module simultaneously.
        try:
            try:
                # We must retry to import it as some other process could
                # have been compiling it between the first failed import
                # and when we receive the lock
                from cutils_ext.cutils_ext import *  # noqa
            except ImportError:

                compile_cutils()
                from cutils_ext.cutils_ext import *  # noqa

        finally:

def abll_compile():
    # Compile .cu files in abll
    _logger.debug('nvcc_compiler.rpath_defaults: %s',
                  str(nvcc_compiler.rpath_defaults))
    import time
    t1 = time.time()
    if should_recompile():
        _logger.debug('should recompile')

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in srcs:
            code.append(open(os.path.join(src_dir, source_file)).read())
        code = '\n'.join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug('recompiling')

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(loc):
                        os.makedirs(loc)
                    compiler.compile_str(
                        'abll',
                        code,
                        location=loc,
                        include_dirs=[include_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults + [loc],
                        libs=libs,
                        preargs=['-O3'] + args,
                        py_module=False,)
                except Exception as e:
                    _logger.error('Failed to compile %s %s: %s',
                                  os.path.join(loc, 'mod.cu'),
                                  srcs, str(e))
                    return False
            else:
                _logger.debug('already compiled by another process')

        finally:
            release_lock()
    else:
        _logger.debug('not recompiling')

    # If necessary, create a symlink called libabll.so
    if not symlink_ok():
        if sys.platform == 'win32':
            # The Python `os` module does not support symlinks on win32.
            shutil.copyfile(abll_so, libabll_so)
        else:
            try:
                os.symlink(abll_so, libabll_so)
            except OSError as e:
                # This may happen for instance when running multiple
                # concurrent jobs, if two of them try to create the
                # symlink simultaneously.
                # If that happens, we verify that the existing symlink is
                # indeed working.
                if (getattr(e, 'errno', None) != errno.EEXIST
                        or not symlink_ok()):
                    raise

    # Raise an error if libabll_so is still not available
    open(libabll_so).close()

    # Add abll to the list of places that are hard-coded into
    # compiled modules' runtime library search list.
    nvcc_compiler.add_standard_rpath(loc)

    t2 = time.time()
    _logger.debug('successfully imported. Compiled in %fs', t2 - t1)

    return True

def convnet_compile():
    # Compile .cu files in cuda_convnet
    _logger.debug("nvcc_compiler.rpath_defaults: %s", str(nvcc_compiler.rpath_defaults))
    import time

    t1 = time.time()
    if should_recompile():
        _logger.debug("should recompile")

        # Concatenate all .cu files into one big mod.cu
        code = []
        for source_file in cuda_convnet_file_sources:
            code.append(open(os.path.join(this_dir, source_file)).read())
        code = "\n".join(code)

        get_lock()
        try:
            # Check if the compilation has already been done by another process
            # while we were waiting for the lock
            if should_recompile():
                _logger.debug("recompiling")

                try:
                    compiler = nvcc_compiler.NVCC_compiler()
                    args = compiler.compile_args()

                    # compiler.compile_args() can execute a
                    # compilation This currently will remove empty
                    # directory in the compile dir.  So we must make
                    # destination directory after calling it.
                    if not os.path.exists(cuda_convnet_loc):
                        os.makedirs(cuda_convnet_loc)
                    compiler.compile_str(
                        "cuda_convnet",
                        code,
                        location=cuda_convnet_loc,
                        include_dirs=[this_dir, config.pthreads.inc_dir] if config.pthreads.inc_dir else [this_dir],
                        lib_dirs=nvcc_compiler.rpath_defaults
                        + [cuda_convnet_loc]
                        + ([config.pthreads.lib_dir] if config.pthreads.lib_dir else []),
                        libs=["cublas", config.pthreads.lib] if config.pthreads.lib else ["cublas"],
                        preargs=["-O3"] + args,
                        py_module=False,
                    )
                except Exception as e:
                    _logger.error(
                        "Failed to compile %s %s: %s",
                        os.path.join(cuda_convnet_loc, "mod.cu"),
                        cuda_convnet_file_sources,
                        str(e),
                    )
                    return False
            else:
                _logger.debug("already compiled by another process")

        finally:
            release_lock()
    else:
        _logger.debug("not recompiling")

    # If necessary, create a symlink called libcuda_convnet.so
    if not symlink_ok():
        if sys.platform == "win32":
            # The Python `os` module does not support symlinks on win32.
            shutil.copyfile(cuda_convnet_so, libcuda_convnet_so)
        else:
            try:
                os.symlink(cuda_convnet_so, libcuda_convnet_so)
            except OSError as e:
                # This may happen for instance when running multiple
                # concurrent jobs, if two of them try to create the
                # symlink simultaneously.
                # If that happens, we verify that the existing symlink is
                # indeed working.
                if getattr(e, "errno", None) != errno.EEXIST or not symlink_ok():
                    raise

    # Raise an error if libcuda_convnet_so is still not available
    open(libcuda_convnet_so).close()

    # Add cuda_convnet to the list of places that are hard-coded into
    # compiled modules' runtime library search list.
    nvcc_compiler.add_standard_rpath(cuda_convnet_loc)

    t2 = time.time()
    _logger.debug("successfully imported. Compiled in %fs", t2 - t1)

    return True