Python protocol.Py4JJavaError方法代碼示例

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def install_exception_handler():
    """
    Hook an exception handler into Py4j, which could capture some exceptions in Java.

    When calling Java API, it will call `get_return_value` to parse the returned object.
    If any exception happened in JVM, the result will be a Java exception object, it raise
    py4j.protocol.Py4JJavaError. We replace the original `get_return_value` with one that
    could capture the Java exception and throw a Python one (with the same error message).

    It's idempotent, could be called multiple times.
    """
    original = py4j.protocol.get_return_value
    # The original `get_return_value` is not patched, it's idempotent.
    patched = capture_java_exception(original)
    # only patch the one used in py4j.java_gateway (call Java API)
    py4j.java_gateway.get_return_value = patched 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def run(self):
        while True:
            try:
                self.connect_to_rabbitmq()
                self.init_rabbitmq_channels()
                current_app.logger.info('Request consumer started!')

                try:
                    self.request_channel.start_consuming()
                except pika.exceptions.ConnectionClosed as e:
                    current_app.logger.error('connection to rabbitmq closed: %s', str(e), exc_info=True)
                    self.rabbitmq.close()
                    continue
                self.rabbitmq.close()
            except Py4JJavaError as e:
                current_app.logger.critical("Critical: JAVA error in spark-request consumer: %s, message: %s", str(e), str(e.java_exception), exc_info=True)
                time.sleep(2)
            except Exception as e:
                current_app.logger.critical("Error in spark-request-consumer: %s", str(e), exc_info=True)
                time.sleep(2) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def init_spark_session(app_name):
    """ Initializes a Spark Session with the given application name.

        Args:
            app_name (str): Name of the Spark application. This will also occur in the Spark UI.
    """
    global session, context, sql_context
    try:
        session = SparkSession \
                .builder \
                .appName(app_name) \
                .config("spark.hadoop.dfs.client.use.datanode.hostname", "true") \
                .config("spark.hadoop.dfs.datanode.use.datanode.hostname", "true") \
                .config("spark.driver.maxResultSize", "4g") \
                .getOrCreate()
        context = session.sparkContext
        context.setLogLevel("ERROR")
        sql_context = SQLContext(context)
    except Py4JJavaError as err:
        raise SparkSessionNotInitializedException(app_name, err.java_exception) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def compute_rmse(model, data, n, model_id):
    """ Compute RMSE (Root Mean Squared Error).

        Args:
            model: Trained model.
            data (rdd): Rdd used for validation i.e validation_data
            n (int): Number of rows/elements in validation_data.
            model_id (str): Model identification string.
    """
    try:
        predictions = model.predictAll(data.map(lambda x: (x.user, x.product)))
        predictionsAndRatings = predictions.map(lambda x: ((x[0], x[1]), x[2])) \
                                           .join(data.map(lambda x: ((x[0], x[1]), x[2]))) \
                                           .values()
        return sqrt(predictionsAndRatings.map(lambda x: (x[0] - x[1]) ** 2).reduce(add) / float(n))
    except Py4JJavaError as err:
        current_app.logger.error('Root Mean Squared Error for model "{}" not computed\n{}'.format(
                                 model_id, str(err.java_exception)), exc_info=True)
        sys.exit(-1) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def train(training_data, rank, iteration, lmbda, alpha, model_id):
    """ Train model.

        Args:
            training_data (rdd): Used for training.
            rank (int): Number of factors in ALS model.
            iteration (int): Number of iterations to run.
            lmbda (float): Controls regularization.
            alpha (float): Constant for computing confidence.
            model_id (str): Model identification string.

        Returns:
            model: Trained model.

    """
    try:
        model = ALS.trainImplicit(training_data, rank, iterations=iteration, lambda_=lmbda, alpha=alpha)
        return model
    except Py4JJavaError as err:
        current_app.logger.error('Unable to train model "{}"\n{}'.format(model_id, str(err.java_exception)), exc_info=True)
        sys.exit(-1) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def save_model(model_id, model):
    """ Save model to HDFS.

        Args:
            model_id (str): Model identification string.
            model: Trained model
    """
    # delete previously saved model before saving a new model
    delete_model()

    dest_path = get_model_path(model_id)
    try:
        current_app.logger.info('Saving model...')
        model.save(listenbrainz_spark.context, dest_path)
        current_app.logger.info('Model saved!')
    except Py4JJavaError as err:
        current_app.logger.error('Unable to save model "{}"\n{}. Aborting...'.format(model_id,
                                 str(err.java_exception)), exc_info=True)
        sys.exit(-1) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def get_mock_stage_completed():
    def _inner(failure_reason):
        class JavaException:
            def __init__(self):
                self._target_id = "id"

        class FailureReason:
            def get(self):
                if failure_reason:
                    return "failure-reason"
                else:
                    raise Py4JJavaError("msg", JavaException())

        class StageInfo:
            def stageId(self):  # noqa: N802
                return "sample-stage-id-submit"

            def name(self):
                return "run-job"

            def attemptId(self):  # noqa: N802
                return 14

            def failureReason(self):  # noqa: N802
                return FailureReason()

        class MockStageCompleted:
            def stageInfo(self):  # noqa: N802
                return StageInfo()

        return MockStageCompleted()

    return _inner 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def capture_java_exception(f):
    def deco(*a, **kw):
        try:
            return f(*a, **kw)
        except Py4JJavaError as e:
            s = e.java_exception.toString()
            stack_trace = '\n\t at '.join(map(lambda x: x.toString(),
                                              e.java_exception.getStackTrace()))
            for exception in exception_mapping.keys():
                if s.startswith(exception):
                    raise exception_mapping[exception](s.split(': ', 1)[1], stack_trace)
            raise
    return deco 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def get_private_field(java_obj, field_name):
    try:
        field = java_obj.getClass().getDeclaredField(field_name)
        field.setAccessible(True)
        return field.get(java_obj)
    except Py4JJavaError:
        cls = java_obj.getClass()
        while cls.getSuperclass() is not None:
            cls = cls.getSuperclass()
            try:
                field = cls.getDeclaredField(field_name)
                field.setAccessible(True)
                return field.get(java_obj)
            except Py4JJavaError:
                pass 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def test_stopiteration_is_raised(self):

        def stopit(*args, **kwargs):
            raise StopIteration()

        def legit_create_combiner(x):
            return [x]

        def legit_merge_value(x, y):
            return x.append(y) or x

        def legit_merge_combiners(x, y):
            return x.extend(y) or x

        data = [(x % 2, x) for x in range(100)]

        # wrong create combiner
        m = ExternalMerger(Aggregator(stopit, legit_merge_value, legit_merge_combiners), 20)
        with self.assertRaises((Py4JJavaError, RuntimeError)) as cm:
            m.mergeValues(data)

        # wrong merge value
        m = ExternalMerger(Aggregator(legit_create_combiner, stopit, legit_merge_combiners), 20)
        with self.assertRaises((Py4JJavaError, RuntimeError)) as cm:
            m.mergeValues(data)

        # wrong merge combiners
        m = ExternalMerger(Aggregator(legit_create_combiner, legit_merge_value, stopit), 20)
        with self.assertRaises((Py4JJavaError, RuntimeError)) as cm:
            m.mergeCombiners(map(lambda x_y1: (x_y1[0], [x_y1[1]]), data)) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def test_pipe_functions(self):
        data = ['1', '2', '3']
        rdd = self.sc.parallelize(data)
        with QuietTest(self.sc):
            self.assertEqual([], rdd.pipe('cc').collect())
            self.assertRaises(Py4JJavaError, rdd.pipe('cc', checkCode=True).collect)
        result = rdd.pipe('cat').collect()
        result.sort()
        for x, y in zip(data, result):
            self.assertEqual(x, y)
        self.assertRaises(Py4JJavaError, rdd.pipe('grep 4', checkCode=True).collect)
        self.assertEqual([], rdd.pipe('grep 4').collect()) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def test_stopiteration_in_user_code(self):

        def stopit(*x):
            raise StopIteration()

        seq_rdd = self.sc.parallelize(range(10))
        keyed_rdd = self.sc.parallelize((x % 2, x) for x in range(10))
        msg = "Caught StopIteration thrown from user's code; failing the task"

        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.map(stopit).collect)
        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.filter(stopit).collect)
        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.reduce, stopit)
        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.fold, 0, stopit)
        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
        self.assertRaisesRegexp(Py4JJavaError, msg,
                                seq_rdd.cartesian(seq_rdd).flatMap(stopit).collect)

        # these methods call the user function both in the driver and in the executor
        # the exception raised is different according to where the StopIteration happens
        # RuntimeError is raised if in the driver
        # Py4JJavaError is raised if in the executor (wraps the RuntimeError raised in the worker)
        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
                                keyed_rdd.reduceByKeyLocally, stopit)
        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
                                seq_rdd.aggregate, 0, stopit, lambda *x: 1)
        self.assertRaisesRegexp((Py4JJavaError, RuntimeError), msg,
                                seq_rdd.aggregate, 0, lambda *x: 1, stopit) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def test_with_different_versions_of_python(self):
        rdd = self.sc.parallelize(range(10))
        rdd.count()
        version = self.sc.pythonVer
        self.sc.pythonVer = "2.0"
        try:
            with QuietTest(self.sc):
                self.assertRaises(Py4JJavaError, lambda: rdd.count())
        finally:
            self.sc.pythonVer = version 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def test_py4j_exception_message(self):
        from pyspark.util import _exception_message

        with self.assertRaises(Py4JJavaError) as context:
            # This attempts java.lang.String(null) which throws an NPE.
            self.sc._jvm.java.lang.String(None)

        self.assertTrue('NullPointerException' in _exception_message(context.exception)) 

# 需要導入模塊: from py4j import protocol [as 別名]
# 或者: from py4j.protocol import Py4JJavaError [as 別名]
def _java2py(sc, r, encoding="bytes"):
    if isinstance(r, JavaObject):
        clsName = r.getClass().getSimpleName()
        # convert RDD into JavaRDD
        if clsName != 'JavaRDD' and clsName.endswith("RDD"):
            r = r.toJavaRDD()
            clsName = 'JavaRDD'

        if clsName == 'JavaRDD':
            jrdd = sc._jvm.org.apache.spark.ml.python.MLSerDe.javaToPython(r)
            return RDD(jrdd, sc)

        if clsName == 'Dataset':
            return DataFrame(r, SQLContext.getOrCreate(sc))

        if clsName in _picklable_classes:
            r = sc._jvm.org.apache.spark.ml.python.MLSerDe.dumps(r)
        elif isinstance(r, (JavaArray, JavaList)):
            try:
                r = sc._jvm.org.apache.spark.ml.python.MLSerDe.dumps(r)
            except Py4JJavaError:
                pass  # not pickable

    if isinstance(r, (bytearray, bytes)):
        r = PickleSerializer().loads(bytes(r), encoding=encoding)
    return r