本文整理汇总了Java中org.apache.nifi.flowfile.attributes.CoreAttributes类的典型用法代码示例。如果您正苦于以下问题:Java CoreAttributes类的具体用法?Java CoreAttributes怎么用?Java CoreAttributes使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
CoreAttributes类属于org.apache.nifi.flowfile.attributes包,在下文中一共展示了CoreAttributes类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: createAttributes
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
protected Map<String, String> createAttributes(final FileInfo fileInfo, final ProcessContext context) {
final Map<String, String> attributes = new HashMap<>();
final DateFormat formatter = new SimpleDateFormat(ListFile.FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US);
attributes.put(getProtocolName() + ".remote.host",
context.getProperty(HOSTNAME).evaluateAttributeExpressions().getValue());
attributes.put(getProtocolName() + ".remote.port",
context.getProperty(UNDEFAULTED_PORT).evaluateAttributeExpressions().getValue());
attributes.put(getProtocolName() + ".listing.user",
context.getProperty(USERNAME).evaluateAttributeExpressions().getValue());
attributes.put(ListFile.FILE_LAST_MODIFY_TIME_ATTRIBUTE,
formatter.format(new Date(fileInfo.getLastModifiedTime())));
attributes.put(ListFile.FILE_PERMISSIONS_ATTRIBUTE, fileInfo.getPermissions());
attributes.put(ListFile.FILE_OWNER_ATTRIBUTE, fileInfo.getOwner());
attributes.put(ListFile.FILE_GROUP_ATTRIBUTE, fileInfo.getGroup());
attributes.put(CoreAttributes.FILENAME.key(), fileInfo.getFileName());
final String fullPath = fileInfo.getFullPathFileName();
if (fullPath != null) {
final int index = fullPath.lastIndexOf("/");
if (index > -1) {
final String path = fullPath.substring(0, index);
attributes.put(CoreAttributes.PATH.key(), path);
}
}
return attributes;
}
示例2: sendStatsIfPresent
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
/**
* Send a flowfile with the stats as attributes: IF the report time is exceeded AND there are
* stats to send
*/
private void sendStatsIfPresent(ProcessSession session, long currentTimestamp) {
if (currentTimestamp < lastReportTime + reportingIntervalMillis) {
return;
}
for (Map.Entry<String, Optional<Map<String, String>>> statsMap : latestStats.entrySet()) {
String statsMapKey = statsMap.getKey();
Optional<Map<String, String>> result = buildStatAttributes(currentTimestamp, momentsMap.get(statsMapKey));
if (result.isPresent()) {
Map<String, String> attrs = new HashMap<>(result.get());
attrs.put("AbstractStatsProcessor.correlationKey", statsMapKey);
String uuid = UUID.randomUUID().toString();
attrs.put(CoreAttributes.UUID.key(), uuid);
attrs.put(CoreAttributes.FILENAME.key(), statsMapKey + "_" + System.currentTimeMillis() + "_" + uuid);
FlowFile statsFlowFile = session.create();
statsFlowFile = session.putAllAttributes(statsFlowFile, attrs);
session.getProvenanceReporter().create(statsFlowFile);
session.transfer(statsFlowFile, REL_STATS);
}
}
lastReportTime = currentTimestamp;
momentsMap.values().forEach(MomentAggregator::reset);
}
示例3: testSuccess
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testSuccess() throws InterruptedException {
TestRunner runner = TestRunners.newTestRunner(DelayProcessor.class);
runner.setValidateExpressionUsage(false);
runner.setProperty(DelayProcessor.TIME_PERIOD, "1 sec");
runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file1"));
runner.run();
// nothing transferred, 1 file still on queue and cached
runner.assertTransferCount(DelayProcessor.REL_SUCCESS, 0);
assertEquals(1, runner.getQueueSize().getObjectCount());
assertEquals(1, ((DelayProcessor) runner.getProcessor()).delayMap.size());
// wait until delay has passed
Thread.sleep(1010);
runner.run();
// file transferred, nothing on queue, cache empty
runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 1);
assertEquals(0, ((DelayProcessor) runner.getProcessor()).delayMap.size());
}
示例4: testOneRejectRemainingSuccess
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testOneRejectRemainingSuccess() {
runner.setProperty(REJECT_ZERO_BYTE, "true");
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-1.json"));
runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "testRejectZeroLengthFile.json"));
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-2.json"));
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-3.json"));
runner.run();
runner.assertTransferCount(REL_REJECT, 1);
runner.assertTransferCount(REL_SUCCESS, 3);
assertFileExists(TMP_DIR, "testBatch-1.json");
assertFileExists(TMP_DIR, "testBatch-2.json");
assertFileExists(TMP_DIR, "testBatch-3.json");
}
示例5: onTrigger
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(ProcessContext processContext, ProcessSession processSession) throws ProcessException {
// Get flowfile
FlowFile flowFile = processSession.get();
if (flowFile == null) {
return;
}
try {
// Get filename of flowFile
String fileName = flowFile.getAttribute(CoreAttributes.FILENAME.key());
// Invoke REST service with filename as parameter (For now parameter is just '1')
String restEndpoint = processContext.getProperty(REST_ENDPOINT).getValue();
JSONObject jsonResult = Unirest.get(restEndpoint)
.header("accept", "application/json")
.asJson()
.getBody()
.getObject();
// Add attributes to flowfile based on REST call
Map<String, String> newAttributes = new HashMap<>();
newAttributes.put(ATT_ACCOUNT_NAME, jsonResult.getString("name"));
newAttributes.put(ATT_ACCOUNT_USERNAME, jsonResult.getString("username"));
FlowFile updatedFlowFile = processSession.putAllAttributes(flowFile, newAttributes);
// Transfer flowfile to success state
processSession.transfer(updatedFlowFile, REL_SUCCESS);
} catch (UnirestException e) {
processSession.transfer(flowFile, REL_FAILURE);
throw new ProcessException(e);
}
}
示例6: onTrigger
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
List<FlowFile> flowFiles = session.get(delayFilter);
if (flowFiles.isEmpty()) {
logger.debug("no flowFiles ready to be processed, yielding");
context.yield();
} else {
for (FlowFile flowFile : flowFiles) {
delayMap.remove(flowFile.getAttribute(CoreAttributes.UUID.key()));
}
logger.debug("transferring {} files to 'success': {}", new Object[]{flowFiles.size(), flowFiles});
session.transfer(flowFiles, REL_SUCCESS);
}
}
示例7: filter
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public FlowFileFilterResult filter(FlowFile flowFile) {
final String uuid = flowFile.getAttribute(CoreAttributes.UUID.key());
Delay delay = delayMap.get(uuid);
if (delay == null) {
delay = new Delay(delayPeriodPropertyValue.evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.MILLISECONDS));
delayMap.put(uuid, delay);
}
return delay.evaluate() ? FlowFileFilterResult.ACCEPT_AND_CONTINUE : FlowFileFilterResult.REJECT_AND_CONTINUE;
}
示例8: onTrigger
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context, final ProcessSession processSession) {
List<FlowFile> flowFiles = processSession.get(batchSize);
if (flowFiles.isEmpty()) {
return;
}
Session jschSession = null;
Channel channel = null;
try {
jschSession = openSession(context);
final String remotePath = context.getProperty(REMOTE_PATH).evaluateAttributeExpressions().getValue();
channel = openExecChannel(context, jschSession, "scp -r -d -t " + remotePath);
InputStream channelIn = channel.getInputStream();
OutputStream channelOut = channel.getOutputStream();
channel.connect();
waitForAck(channelIn);
ListIterator<FlowFile> fileIt = flowFiles.listIterator();
while (fileIt.hasNext()) {
final FlowFile flowFile = fileIt.next();
// conditionally reject files that are zero bytes or less
if (context.getProperty(REJECT_ZERO_BYTE).asBoolean() && flowFile.getSize() == 0) {
logger.warn("Rejecting {} because it is zero bytes", new Object[]{flowFile});
processSession.transfer(processSession.penalize(flowFile), REL_REJECT);
fileIt.remove();
continue;
}
final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
final String permissions = context.getProperty(PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue();
// destination path + filename
// final String fullPath = buildFullPath(context, flowFile, filename);
processSession.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream flowFileIn) throws IOException {
// send "C0644 filesize filename", where filename should not include '/'
StringBuilder command = new StringBuilder("C").append(permissions).append(' ');
command.append(flowFile.getSize()).append(' ');
command.append(filename).append('\n');
channelOut.write(command.toString().getBytes(StandardCharsets.UTF_8));
channelOut.flush();
waitForAck(channelIn);
IOUtils.copy(flowFileIn, channelOut);
channelOut.flush();
sendAck(channelOut);
waitForAck(channelIn);
}
});
processSession.transfer(flowFile, REL_SUCCESS);
processSession.getProvenanceReporter().send(flowFile, remotePath);
fileIt.remove();
if (logger.isDebugEnabled()) {
logger.debug("Sent {} to remote host", new Object[]{flowFile});
}
}
} catch (JSchException | IOException ex) {
context.yield();
logger.error("Unable to create session to remote host due to {}", new Object[]{ex}, ex);
processSession.transfer(flowFiles, REL_FAILURE);
} finally {
if (channel != null) {
channel.disconnect();
}
if (jschSession != null) {
jschSession.disconnect();
}
}
}
示例9: testExpireMultipleFilesSimultaneously
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testExpireMultipleFilesSimultaneously() throws InterruptedException {
TestRunner runner = TestRunners.newTestRunner(DelayProcessor.class);
runner.setValidateExpressionUsage(false);
runner.setProperty(DelayProcessor.TIME_PERIOD, "1 sec");
runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file1"));
runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file2"));
runner.run();
// nothing transferred, 2 files still on queue and cached
runner.assertTransferCount(DelayProcessor.REL_SUCCESS, 0);
assertEquals(2, runner.getQueueSize().getObjectCount());
assertEquals(2, ((DelayProcessor) runner.getProcessor()).delayMap.size());
runner.enqueue(new byte[0], ImmutableMap.of(CoreAttributes.FILENAME.key(), "file3"));
// wait until delay has passed
Thread.sleep(1010);
runner.run();
// 1st 2 files transferred, 3rd still on queue and cached
runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 2);
assertEquals(1, ((DelayProcessor) runner.getProcessor()).delayMap.size());
runner.clearTransferState();
// wait until delay has passed
Thread.sleep(1010);
runner.run();
// 3rd file now transferred, cache empty
runner.assertAllFlowFilesTransferred(DelayProcessor.REL_SUCCESS, 1);
assertEquals(0, ((DelayProcessor) runner.getProcessor()).delayMap.size());
}
示例10: testBatch
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testBatch() {
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-1.json"));
runner.enqueue(input2, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testBatch-2.json"));
runner.run();
runner.assertAllFlowFilesTransferred(REL_SUCCESS, 2);
assertFileExists(TMP_DIR, "testBatch-1.json");
assertFileExists(TMP_DIR, "testBatch-2.json");
}
示例11: testLargerBatch
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testLargerBatch() {
final int count = 100;
for (int i = 0; i < count; i++) {
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testLargeBatch-" + i + ".json"));
}
runner.run();
runner.assertAllFlowFilesTransferred(REL_SUCCESS, count);
for (int i = 0; i < count; i++) {
assertFileExists(TMP_DIR, "testLargeBatch-" + i + ".json");
}
}
示例12: testSuccessiveRuns
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Test
public void testSuccessiveRuns() {
runner.enqueue(input, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testSuccessiveRuns-1.json"));
runner.run();
runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
assertFileExists(TMP_DIR, "testSuccessiveRuns-1.json");
runner.clearTransferState();
runner.enqueue(input2, ImmutableMap.of(CoreAttributes.FILENAME.key(), "testSuccessiveRuns-2.json"));
runner.run();
runner.assertAllFlowFilesTransferred(REL_SUCCESS, 1);
assertFileExists(TMP_DIR, "testSuccessiveRuns-2.json");
}
示例13: setupClass
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@BeforeClass
public static void setupClass() throws IOException {
TMP_DIR = Files.createTempDirectory("TestPutScp").toFile();
ATTRIBUTES = ImmutableMap.of(CoreAttributes.FILENAME.key(), TEST_FILE_NAME);
}
示例14: onTrigger
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context,
final ProcessSession session) throws ProcessException {
final Event event = eventQueue.poll();
if (event != null) {
switch (event.getEventType()) {
case STOPPED_BY_ERROR:
getLogger()
.error("Received error {}: {} due to {}. Will not attempt to reconnect",
new Object[] { event.getEventType(),
event.getMessage(),
event.getUnderlyingException() });
break;
case CONNECTION_ERROR:
case HTTP_ERROR:
getLogger()
.error("Received error {}: {}. Will attempt to reconnect",
new Object[] { event.getEventType(),
event.getMessage() });
client.reconnect();
break;
default:
break;
}
}
final String tweet = messageQueue.poll();
if (tweet == null) {
context.yield();
return;
}
FlowFile flowFile = session.create();
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream out) throws IOException {
out.write(tweet.getBytes(StandardCharsets.UTF_8));
}
});
final Map<String, String> attributes = new HashMap<>();
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
attributes.put(CoreAttributes.FILENAME.key(),
flowFile.getAttribute(CoreAttributes.FILENAME.key()) + ".json");
flowFile = session.putAllAttributes(flowFile, attributes);
session.transfer(flowFile, REL_SUCCESS);
session.getProvenanceReporter().receive(
flowFile,
Constants.STREAM_HOST
+ client.getEndpoint().getURI().toString());
}
示例15: onTrigger
import org.apache.nifi.flowfile.attributes.CoreAttributes; //导入依赖的package包/类
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
final ProcessorLog logger = getLogger();
final Message<JsonObject> message = messageQueue.poll();
if (message == null) {
context.yield();
return;
}
FlowFile flowFile = session.create();
try {
final StopWatch stopWatch = new StopWatch(true);
flowFile = session.write(flowFile, new OutputStreamCallback() {
@Override
public void process(final OutputStream out) throws IOException {
out.write(message.body().encode().getBytes(StandardCharsets.UTF_8));
}
});
if (flowFile.getSize() == 0L) {
session.remove(flowFile);
} else {
final Map<String, String> attributes = new HashMap<>();
if (headerPattern != null) {
MultiMap headers = message.headers();
headers.names()
.stream()
.filter(headerName -> headerPattern.matcher(headerName).matches())
.forEach(headerName -> attributes.put(headerName, headers.get(headerName)));
// attributes.putAll(
// headers.entries()
// .stream()
// .filter(entry -> headerPattern.matcher(entry.getKey()).matches())
// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
// );
}
attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json");
attributes.put(CoreAttributes.FILENAME.key(), flowFile.getAttribute(CoreAttributes.FILENAME.key()) + ".json");
attributes.put("eventbus.address", message.address());
flowFile = session.putAllAttributes(flowFile, attributes);
session.getProvenanceReporter().receive(flowFile, message.address(), "received eventBus message", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
getLogger().info("Successfully received {} ({}) from EventBus in {} millis", new Object[]{flowFile, flowFile.getSize(), stopWatch.getElapsed(TimeUnit.MILLISECONDS)});
session.transfer(flowFile, REL_SUCCESS);
}
} catch (Exception e) {
session.remove(flowFile);
throw e;
}
}