Java CoreAttributes类代码示例

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
protected Map<String, String> createAttributes(final FileInfo fileInfo, final ProcessContext context) {
	final Map<String, String> attributes = new HashMap<>();
	final DateFormat formatter = new SimpleDateFormat(ListFile.FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
	attributes.put(getProtocolName() + ".remote.host",
			context.getProperty(HOSTNAME).evaluateAttributeExpressions().getValue());
	attributes.put(getProtocolName() + ".remote.port",
			context.getProperty(UNDEFAULTED_PORT).evaluateAttributeExpressions().getValue());
	attributes.put(getProtocolName() + ".listing.user",
			context.getProperty(USERNAME).evaluateAttributeExpressions().getValue());
	attributes.put(ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE,
			formatter.format(new Date(fileInfo.getLastModifiedTime())));
	attributes.put(ListFile.FILE_PERMISSIONS_ATTRIBUTE, fileInfo.getPermissions());
	attributes.put(ListFile.FILE_OWNER_ATTRIBUTE, fileInfo.getOwner());
	attributes.put(ListFile.FILE_GROUP_ATTRIBUTE, fileInfo.getGroup());
	attributes.put(CoreAttributes.FILENAME.key(), fileInfo.getFileName());
	final String fullPath = fileInfo.getFullPathFileName();
	if (fullPath != null) {
		final int index = fullPath.lastIndexOf("/");
		if (index > -1) {
			final String path = fullPath.substring(0, index);
			attributes.put(CoreAttributes.PATH.key(), path);
		}
	}
	return attributes;
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
/**
 * Send a flowfile with the stats as attributes: IF the report time is exceeded AND there are
 * stats to send
 */
private void sendStatsIfPresent(ProcessSession session, long currentTimestamp) {
    if (currentTimestamp < lastReportTime + reportingIntervalMillis) {
        return;
    }

    for (Map.Entry<String, Optional<Map<String, String>>> statsMap : latestStats.entrySet()) {

        String statsMapKey = statsMap.getKey();
        Optional<Map<String, String>> result = buildStatAttributes(currentTimestamp, momentsMap.get(statsMapKey));
        if (result.isPresent()) {
            Map<String, String> attrs = new HashMap<>(result.get());
            attrs.put("AbstractStatsProcessor.correlationKey", statsMapKey);
            String uuid = UUID.randomUUID().toString();
            attrs.put(CoreAttributes.UUID.key(), uuid);
            attrs.put(CoreAttributes.FILENAME.key(), statsMapKey + "_" + System.currentTimeMillis() + "_" + uuid);
            FlowFile statsFlowFile = session.create();
            statsFlowFile = session.putAllAttributes(statsFlowFile, attrs);
            session.getProvenanceReporter().create(statsFlowFile);
            session.transfer(statsFlowFile, REL_STATS);
        }
    }
    lastReportTime = currentTimestamp;
    momentsMap.values().forEach(MomentAggregator::reset);
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testSuccess() throws InterruptedException {
    TestRunner runner = TestRunners.newTestRunner(DelayProcessor.class);
    runner.setValidateExpressionUsage(false);
    runner.setProperty(DelayProcessor.TIME_PERIOD, "1 sec");

    runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file1"));

    runner.run();
    // nothing transferred, 1 file still on queue and cached
    runner.assertTransferCount(DelayProcessor.REL_SUCCESS, 0);
    assertEquals(1, runner.getQueueSize().getObjectCount());
    assertEquals(1, ((DelayProcessor) runner.getProcessor()).delayMap.size());

    // wait until delay has passed
    Thread.sleep(1010);

    runner.run();
    // file transferred, nothing on queue, cache empty
    runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 1);
    assertEquals(0, ((DelayProcessor) runner.getProcessor()).delayMap.size());
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testOneRejectRemainingSuccess() {
    runner.setProperty(REJECT_ZERO_BYTE, "true");

    runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-1.json"));
    runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "testRejectZeroLengthFile.json"));
    runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-2.json"));
    runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-3.json"));

    runner.run();
    runner.assertTransferCount(REL_REJECT, 1);
    runner.assertTransferCount(REL_SUCCESS, 3);
    assertFileExists(TMP_DIR, "testBatch-1.json");
    assertFileExists(TMP_DIR, "testBatch-2.json");
    assertFileExists(TMP_DIR, "testBatch-3.json");
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(ProcessContext processContext, ProcessSession processSession) throws ProcessException {

    // Get flowfile
    FlowFile flowFile = processSession.get();
    if (flowFile == null) {
        return;
    }

    try {

        // Get filename of flowFile
        String fileName = flowFile.getAttribute(CoreAttributes.FILENAME.key());

        // Invoke REST service with filename as parameter (For now parameter is just '1')
        String restEndpoint = processContext.getProperty(REST_ENDPOINT).getValue();
        JSONObject jsonResult = Unirest.get(restEndpoint)
                .header("accept", "application/json")
                .asJson()
                .getBody()
                .getObject();

        // Add attributes to flowfile based on REST call
        Map<String, String> newAttributes = new HashMap<>();
        newAttributes.put(ATT_ACCOUNT_NAME, jsonResult.getString("name"));
        newAttributes.put(ATT_ACCOUNT_USERNAME, jsonResult.getString("username"));
        FlowFile updatedFlowFile = processSession.putAllAttributes(flowFile, newAttributes);

        // Transfer flowfile to success state
        processSession.transfer(updatedFlowFile, REL_SUCCESS);

    } catch (UnirestException e) {
        processSession.transfer(flowFile, REL_FAILURE);
        throw new ProcessException(e);
    }
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    List<FlowFile> flowFiles = session.get(delayFilter);

    if (flowFiles.isEmpty()) {
        logger.debug("no flowFiles ready to be processed, yielding");
        context.yield();
    } else {
        for (FlowFile flowFile : flowFiles) {
            delayMap.remove(flowFile.getAttribute(CoreAttributes.UUID.key()));
        }
        logger.debug("transferring {} files to 'success': {}", new Object[]{flowFiles.size(), flowFiles});
        session.transfer(flowFiles, REL_SUCCESS);
    }
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public FlowFileFilterResult filter(FlowFile flowFile) {
    final String uuid = flowFile.getAttribute(CoreAttributes.UUID.key());
    Delay delay = delayMap.get(uuid);
    if (delay == null) {
        delay = new Delay(delayPeriodPropertyValue.evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.MILLISECONDS));
        delayMap.put(uuid, delay);
    }

    return delay.evaluate() ? FlowFileFilterResult.ACCEPT_AND_CONTINUE : FlowFileFilterResult.REJECT_AND_CONTINUE;
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context, final ProcessSession processSession) {
    List<FlowFile> flowFiles = processSession.get(batchSize);
    if (flowFiles.isEmpty()) {
        return;
    }

    Session jschSession = null;
    Channel channel = null;
    try {
        jschSession = openSession(context);
        final String remotePath = context.getProperty(REMOTE_PATH).evaluateAttributeExpressions().getValue();
        channel = openExecChannel(context, jschSession, "scp -r -d -t " + remotePath);

        InputStream channelIn = channel.getInputStream();
        OutputStream channelOut = channel.getOutputStream();

        channel.connect();
        waitForAck(channelIn);

        ListIterator<FlowFile> fileIt = flowFiles.listIterator();
        while (fileIt.hasNext()) {
            final FlowFile flowFile = fileIt.next();

            // conditionally reject files that are zero bytes or less
            if (context.getProperty(REJECT_ZERO_BYTE).asBoolean() && flowFile.getSize() == 0) {
                logger.warn("Rejecting {} because it is zero bytes", new Object[]{flowFile});
                processSession.transfer(processSession.penalize(flowFile), REL_REJECT);
                fileIt.remove();
                continue;
            }

            final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
            final String permissions = context.getProperty(PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue();

            // destination path + filename
            // final String fullPath = buildFullPath(context, flowFile, filename);

            processSession.read(flowFile, new InputStreamCallback() {
                @Override
                public void process(final InputStream flowFileIn) throws IOException {
                    // send "C0644 filesize filename", where filename should not include '/'
                    StringBuilder command = new StringBuilder("C").append(permissions).append(' ');
                    command.append(flowFile.getSize()).append(' ');
                    command.append(filename).append('\n');

                    channelOut.write(command.toString().getBytes(StandardCharsets.UTF_8));
                    channelOut.flush();
                    waitForAck(channelIn);

                    IOUtils.copy(flowFileIn, channelOut);
                    channelOut.flush();
                    sendAck(channelOut);
                    waitForAck(channelIn);
                }
            });

            processSession.transfer(flowFile, REL_SUCCESS);
            processSession.getProvenanceReporter().send(flowFile, remotePath);
            fileIt.remove();
            if (logger.isDebugEnabled()) {
                logger.debug("Sent {} to remote host", new Object[]{flowFile});
            }
        }

    } catch (JSchException | IOException ex) {
        context.yield();
        logger.error("Unable to create session to remote host due to {}", new Object[]{ex}, ex);
        processSession.transfer(flowFiles, REL_FAILURE);

    } finally {
        if (channel != null) {
            channel.disconnect();
        }
        if (jschSession != null) {
            jschSession.disconnect();
        }
    }
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testExpireMultipleFilesSimultaneously() throws InterruptedException {
    TestRunner runner = TestRunners.newTestRunner(DelayProcessor.class);
    runner.setValidateExpressionUsage(false);
    runner.setProperty(DelayProcessor.TIME_PERIOD, "1 sec");

    runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file1"));
    runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file2"));

    runner.run();
    // nothing transferred, 2 files still on queue and cached
    runner.assertTransferCount(DelayProcessor.REL_SUCCESS, 0);
    assertEquals(2, runner.getQueueSize().getObjectCount());
    assertEquals(2, ((DelayProcessor) runner.getProcessor()).delayMap.size());

    runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file3"));

    // wait until delay has passed
    Thread.sleep(1010);

    runner.run();
    // 1st 2 files transferred, 3rd still on queue and cached
    runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 2);
    assertEquals(1, ((DelayProcessor) runner.getProcessor()).delayMap.size());

    runner.clearTransferState();

    // wait until delay has passed
    Thread.sleep(1010);
    runner.run();

    // 3rd file now transferred, cache empty
    runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 1);
    assertEquals(0, ((DelayProcessor) runner.getProcessor()).delayMap.size());
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testBatch() {
    runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-1.json"));
    runner.enqueue(input2, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-2.json"));

    runner.run();
    runner.assertAllFlowFilesTransferred(REL_SUCCESS, 2);
    assertFileExists(TMP_DIR, "testBatch-1.json");
    assertFileExists(TMP_DIR, "testBatch-2.json");
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testLargerBatch() {
    final int count = 100;

    for (int i = 0; i < count; i++) {
        runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testLargeBatch-" + i + ".json"));
    }

    runner.run();
    runner.assertAllFlowFilesTransferred(REL_SUCCESS, count);
    for (int i = 0; i < count; i++) {
        assertFileExists(TMP_DIR, "testLargeBatch-" + i + ".json");
    }
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testSuccessiveRuns() {
    runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testSuccessiveRuns-1.json"));

    runner.run();
    runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
    assertFileExists(TMP_DIR, "testSuccessiveRuns-1.json");

    runner.clearTransferState();
    runner.enqueue(input2, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testSuccessiveRuns-2.json"));
    runner.run();
    runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
    assertFileExists(TMP_DIR, "testSuccessiveRuns-2.json");
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@BeforeClass
public static void setupClass() throws IOException {
    TMP_DIR = Files.createTempDirectory("TestPutScp").toFile();
    ATTRIBUTES = ImmutableMap.of(CoreAttributes.FILENAME.key(), TEST_FILE_NAME);
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context,
		final ProcessSession session) throws ProcessException {
	final Event event = eventQueue.poll();
	if (event != null) {
		switch (event.getEventType()) {
		case STOPPED_BY_ERROR:
			getLogger()
					.error("Received error {}: {} due to {}. Will not attempt to reconnect",
							new Object[] { event.getEventType(),
									event.getMessage(),
									event.getUnderlyingException() });
			break;
		case CONNECTION_ERROR:
		case HTTP_ERROR:
			getLogger()
					.error("Received error {}: {}. Will attempt to reconnect",
							new Object[] { event.getEventType(),
									event.getMessage() });
			client.reconnect();
			break;
		default:
			break;
		}
	}

	final String tweet = messageQueue.poll();
	if (tweet == null) {
		context.yield();
		return;
	}

	FlowFile flowFile = session.create();
	flowFile = session.write(flowFile, new OutputStreamCallback() {
		@Override
		public void process(final OutputStream out) throws IOException {
			out.write(tweet.getBytes(StandardCharsets.UTF_8));
		}
	});

	final Map<String, String> attributes = new HashMap<>();
	attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
	attributes.put(CoreAttributes.FILENAME.key(),
			flowFile.getAttribute(CoreAttributes.FILENAME.key()) + ".json");
	flowFile = session.putAllAttributes(flowFile, attributes);

	session.transfer(flowFile, REL_SUCCESS);
	session.getProvenanceReporter().receive(
			flowFile,
			Constants.STREAM_HOST
					+ client.getEndpoint().getURI().toString());
}
 

import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
        final ProcessorLog logger = getLogger();
        final Message<JsonObject> message = messageQueue.poll();
        if (message == null) {
            context.yield();
            return;
        }

        FlowFile flowFile = session.create();
        try {
            final StopWatch stopWatch = new StopWatch(true);

            flowFile = session.write(flowFile, new OutputStreamCallback() {
                @Override
                public void process(final OutputStream out) throws IOException {
                    out.write(message.body().encode().getBytes(StandardCharsets.UTF_8));
                }
            });

            if (flowFile.getSize() == 0L) {
                session.remove(flowFile);
            } else {
                final Map<String, String> attributes = new HashMap<>();

                if (headerPattern != null) {
                    MultiMap headers = message.headers();
                    headers.names()
                            .stream()
                            .filter(headerName -> headerPattern.matcher(headerName).matches())
                            .forEach(headerName -> attributes.put(headerName, headers.get(headerName)));

//                    attributes.putAll(
//                        headers.entries()
//                                .stream()
//                                .filter(entry -> headerPattern.matcher(entry.getKey()).matches())
//                                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
//                    );
                }
                attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
                attributes.put(CoreAttributes.FILENAME.key(), flowFile.getAttribute(CoreAttributes.FILENAME.key()) + ".json");
                attributes.put("eventbus.address", message.address());

                flowFile = session.putAllAttributes(flowFile, attributes);
                session.getProvenanceReporter().receive(flowFile, message.address(), "received eventBus message", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
                getLogger().info("Successfully received {} ({}) from EventBus in {} millis", new Object[]{flowFile, flowFile.getSize(), stopWatch.getElapsed(TimeUnit.MILLISECONDS)});
                session.transfer(flowFile, REL_SUCCESS);
            }
        } catch (Exception e) {
            session.remove(flowFile);
            throw e;
        }
    }