Java DataCleanerConfigurationImpl类代码示例

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Test
public void testSingleResultElement() throws IOException {
    final Datastore datastore = TestHelper.createSampleDatabaseDatastore("orderdb");
    final SimpleDescriptorProvider descriptorProvider = new SimpleDescriptorProvider();
    descriptorProvider.addRendererBeanDescriptor(Descriptors.ofRenderer(ListResultHtmlRenderer.class));

    final DataCleanerEnvironment environment =
            new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider);
    final DataCleanerConfigurationImpl configuration =
            new DataCleanerConfigurationImpl().withDatastores(datastore).withEnvironment(environment);
    final AnalysisJob job;
    try (AnalysisJobBuilder jobBuilder = new AnalysisJobBuilder(configuration)) {
        jobBuilder.setDatastore(datastore);
        jobBuilder.addSourceColumns("customers.customername");

        jobBuilder.addAnalyzer(MockAnalyzer.class).addInputColumns(jobBuilder.getSourceColumns());

        job = jobBuilder.toAnalysisJob();
    }

    final AnalysisResult analysisResult = new AnalysisRunnerImpl(configuration).run(job);

    writeAndCompareWithBenchmark(analysisResult, configuration);
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testReadAndWriteCompoundComponentRequirementJob() throws Exception {
    final Datastore ds = TestHelper.createSampleDatabaseDatastore("my database");
    final SimpleDescriptorProvider descriptorProvider = new SimpleDescriptorProvider();
    descriptorProvider.addFilterBeanDescriptor(Descriptors.ofFilter(NullCheckFilter.class));
    descriptorProvider.addTransformerBeanDescriptor(Descriptors.ofTransformer(ConcatenatorTransformer.class));
    descriptorProvider.addAnalyzerBeanDescriptor(Descriptors.ofAnalyzer(StringAnalyzer.class));
    final DataCleanerConfiguration conf = new DataCleanerConfigurationImpl().withDatastores(ds)
            .withEnvironment(new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider));

    final JaxbJobReader reader = new JaxbJobReader(conf);
    final AnalysisJob job;
    try (AnalysisJobBuilder jobBuilder = reader
            .create(new File("src/test/resources/example-job-compound-component-requirement.xml"))) {
        job = jobBuilder.toAnalysisJob();
    }

    final ComponentRequirement requirement = job.getAnalyzerJobs().get(0).getComponentRequirement();
    assertEquals("FilterOutcome[category=NOT_NULL] OR FilterOutcome[category=NULL]", requirement.toString());

    assertMatchesBenchmark(job, "JaxbJobWriterTest-testReadAndWriteCompoundComponentRequirementJob.xml");
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testReadAndWriteOutputDataStreamsJob() throws Exception {
    final Datastore ds = TestHelper.createSampleDatabaseDatastore("my database");
    final SimpleDescriptorProvider descriptorProvider = new SimpleDescriptorProvider();
    descriptorProvider.addTransformerBeanDescriptor(Descriptors.ofTransformer(ConcatenatorTransformer.class));
    descriptorProvider.addAnalyzerBeanDescriptor(Descriptors.ofAnalyzer(CompletenessAnalyzer.class));
    descriptorProvider.addAnalyzerBeanDescriptor(Descriptors.ofAnalyzer(StringAnalyzer.class));
    descriptorProvider.addAnalyzerBeanDescriptor(Descriptors.ofAnalyzer(NumberAnalyzer.class));

    final DataCleanerConfiguration conf = new DataCleanerConfigurationImpl().withDatastores(ds)
            .withEnvironment(new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider));

    final JaxbJobReader reader = new JaxbJobReader(conf);
    final AnalysisJob job;
    try (AnalysisJobBuilder jobBuilder = reader
            .create(new File("src/test/resources/example-job-output-dataset.analysis.xml"))) {
        job = jobBuilder.toAnalysisJob();
    }

    assertMatchesBenchmark(job, "JaxbJobWriterTest-testReadAndWriteOutputDataStreamsJob.xml");
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testWriteCsv() throws Exception {

        final Datastore datastore = TestHelper.createSampleDatabaseDatastore("my db");
        final DataCleanerConfiguration configuration = new DataCleanerConfigurationImpl().withDatastores(datastore);
        try (AnalysisJobBuilder ajb = new AnalysisJobBuilder(configuration)) {
            ajb.setDatastore("my db");
            ajb.addSourceColumns("PUBLIC.EMPLOYEES.FIRSTNAME", "PUBLIC.EMPLOYEES.LASTNAME", "PUBLIC.EMPLOYEES.EMAIL");

            final InputColumn<?> fnCol = ajb.getSourceColumnByName("FIRSTNAME");
            final InputColumn<?> lnCol = ajb.getSourceColumnByName("LASTNAME");
            final InputColumn<?> emailCol = ajb.getSourceColumnByName("EMAIL");

            final FileResource file = new FileResource("tmp/myFile.csv");
            final AnalyzerComponentBuilder<CreateCsvFileAnalyzer> csvAnalyzer = ajb.addAnalyzer(
                    CreateCsvFileAnalyzer.class);
            csvAnalyzer.addInputColumns(fnCol, lnCol, emailCol);
            
            csvAnalyzer.setConfiguredProperty("File", file);

            assertMatchesBenchmark(ajb.toAnalysisJob(),"JaxbJobWriterTest-testWriteCsv.xml");
        }
    }
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testReadMetadataNone() throws Exception {
    final JobReader<InputStream> reader = new JaxbJobReader(new DataCleanerConfigurationImpl());
    final AnalysisJobMetadata metadata =
            reader.readMetadata(new FileInputStream(new File("src/test/resources/example-job-valid.xml")));

    assertNull(metadata.getAuthor());
    assertNull(metadata.getJobName());
    assertNull(metadata.getJobDescription());
    assertNull(metadata.getJobVersion());
    assertTrue(metadata.getProperties().isEmpty());
    assertEquals("my database", metadata.getDatastoreName());
    assertEquals("[PUBLIC.EMPLOYEES.FIRSTNAME, PUBLIC.EMPLOYEES.LASTNAME, PUBLIC.EMPLOYEES.EMAIL]",
            metadata.getSourceColumnPaths().toString());

    assertNull(metadata.getCreatedDate());
    assertNull(metadata.getUpdatedDate());
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testJobWithTemplateProperties2() throws IOException {
    final DescriptorProvider descriptorProvider = new ClasspathScanDescriptorProvider().scanPackage(
            "org.datacleaner", true);
    final Datastore datastore = TestHelper.createSampleDatabaseDatastore("my db");
    final DataCleanerConfiguration configuration = new DataCleanerConfigurationImpl().withDatastores(datastore)
            .withEnvironment(new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider));
    ;
    final JaxbJobReader reader = new JaxbJobReader(configuration);
    final AnalysisJobBuilder jobBuilder = reader.create(new File(
            "src/test/resources/JaxbJobWriterTest-testWriteCsvTemplate.xml"));

    final List<ComponentBuilder> componentBuilders = new ArrayList<>(jobBuilder.getComponentBuilders());
    assertEquals(1, componentBuilders.size());
    final ComponentBuilder componentBuilder = componentBuilders.get(0);
    final ComponentDescriptor<?> descriptor = componentBuilder.getDescriptor();
    assertEquals("Create CSV file", descriptor.getDisplayName());
    final LinkedList<Object> linkedList = new LinkedList<>(componentBuilder.getConfiguredProperties().values());
    final FileResource propertyFile = (FileResource) linkedList.get(3);
    String absolutePath = propertyFile.getFile().getAbsolutePath();
    absolutePath = absolutePath.replace("\\", "/");
    absolutePath = absolutePath.replace("C:", "");
    assertEquals("/Users/claudiap/Documents/OutgoingHotFolder/myFile/1482244133378-samples.csv", absolutePath);
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testScenario() throws Throwable {
    final CsvDatastore datastore =
            new CsvDatastore("my database", "../core/src/test/resources/example-name-lengths.csv");
    final ClasspathScanDescriptorProvider descriptorProvider =
            new ClasspathScanDescriptorProvider().scanPackage("org.datacleaner", true);
    final DataCleanerConfiguration configuration = new DataCleanerConfigurationImpl().withDatastores(datastore)
            .withEnvironment(new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider));
    final AnalysisJob job = new JaxbJobReader(configuration)
            .read(new FileInputStream("src/test/resources/example-job-components-without-inputcolumns.xml"));

    final AnalysisRunner runner = new AnalysisRunnerImpl(configuration);
    final AnalysisResultFuture resultFuture = runner.run(job);

    if (!resultFuture.isSuccessful()) {
        throw resultFuture.getErrors().get(0);
    }

    final InputColumn<?>[] input = job.getAnalyzerJobs().iterator().next().getInput();
    assertEquals(4, input.length);

    final StringAnalyzerResult result = (StringAnalyzerResult) resultFuture.getResults().get(0);
    for (int i = 0; i < input.length; i++) {
        assertEquals(5, result.getRowCount(input[i]));
    }
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testScenario() throws Exception {
    final DescriptorProvider descriptorProvider =
            new ClasspathScanDescriptorProvider().scanPackage("org.datacleaner.beans", true);
    final DataCleanerEnvironment environment =
            new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider);
    final DataCleanerConfiguration conf = new DataCleanerConfigurationImpl()
            .withDatastores(TestHelper.createSampleDatabaseDatastore("my database")).withEnvironment(environment);

    final AnalysisRunner runner = new AnalysisRunnerImpl(conf);

    final AnalysisJobBuilder jobBuilder = new JaxbJobReader(conf)
            .create(new File("src/test/resources/example-job-input-columns-from-different-tables.xml"));

    try {
        runner.run(jobBuilder.toAnalysisJob());
        fail("exception expected");
    } catch (final IllegalStateException e) {
        assertEquals("Multiple originating tables (CUSTOMERS, EMPLOYEES) found for source: "
                        + "TransformerComponentBuilder[transformer=Concatenator,inputColumns="
                        + "[MetaModelInputColumn[PUBLIC.EMPLOYEES.LASTNAME], "
                        + "MetaModelInputColumn[PUBLIC.CUSTOMERS.CONTACTLASTNAME]]]", e.getMessage());
    }

}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Test
public void testComponentBuilderIsDistributable() {
    final Datastore datastore = TestHelper.createSampleDatabaseDatastore("orderdb");
    final DataCleanerConfigurationImpl configuration = new DataCleanerConfigurationImpl().withDatastores(datastore);
    try (AnalysisJobBuilder ajb = new AnalysisJobBuilder(configuration)) {
        ajb.setDatastore(datastore);
        ajb.addSourceColumns("customers.country", "customers.city");

        final AnalyzerComponentBuilder<ValueDistributionAnalyzer> componentBuilder =
                ajb.addAnalyzer(ValueDistributionAnalyzer.class);
        assertTrue(componentBuilder.isDistributable());

        componentBuilder.addInputColumn(ajb.getSourceColumnByName("country"));
        assertTrue(componentBuilder.isDistributable());

        componentBuilder.addInputColumn(ajb.getSourceColumnByName("city"));
        assertTrue(componentBuilder.isDistributable());
    }
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Override
protected void setUp() throws Exception {
    super.setUp();
    final DatastoreCatalog datastoreCatalog =
            new DatastoreCatalogImpl(new CsvDatastore("foo", "src/test/resources/employees.csv"));
    configuration = new DataCleanerConfigurationImpl().withDatastoreCatalog(datastoreCatalog)
            .withEnvironment(TestEnvironment.getEnvironment());

    try (AnalysisJobBuilder jobBuilder = new AnalysisJobBuilder(configuration)) {
        jobBuilder.setDatastore("foo");
        jobBuilder.addSourceColumns("name");

        final TransformerComponentBuilder<MockBatchTransformer> transformerBuilder =
                jobBuilder.addTransformer(MockBatchTransformer.class);
        sourceColumn = jobBuilder.getSourceColumns().get(0);
        transformerBuilder.addInputColumns(sourceColumn);

        final AnalyzerComponentBuilder<MockAnalyzer> analyzer = jobBuilder.addAnalyzer(MockAnalyzer.class);
        analyzer.addInputColumns(sourceColumn);
        sortedColumn = transformerBuilder.getOutputColumns().get(0);
        analyzer.addInputColumns(sortedColumn);

        job = jobBuilder.toAnalysisJob();
    }
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Override
protected void setUp() throws Exception {
    super.setUp();
    final SimpleTableDef tableDef = new SimpleTableDef("table", new String[] { "col1" });
    final List<Object[]> rowData = new ArrayList<>();
    rowData.add(new Object[] { "foo" });
    rowData.add(new Object[] { "bar" });
    rowData.add(new Object[] { "baz" });
    rowData.add(new Object[] { "hello" });
    rowData.add(new Object[] { "world" });
    datastore = new PojoDatastore("ds", "sch", new ArrayTableDataProvider(tableDef, rowData));

    final DatastoreCatalog datastoreCatalog = new DatastoreCatalogImpl(datastore);
    configuration = new DataCleanerConfigurationImpl().withDatastoreCatalog(datastoreCatalog);

    jobBuilder = new AnalysisJobBuilder(configuration);
    jobBuilder.setDatastore(datastore);
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testGetRendererByHierarchyDistance() throws Exception {
    final ClasspathScanDescriptorProvider descriptorProvider =
            new ClasspathScanDescriptorProvider().scanPackage("org.datacleaner.result.renderer", true);

    final RendererFactory rendererFactory = new RendererFactory(new DataCleanerConfigurationImpl()
            .withEnvironment(new DataCleanerEnvironmentImpl().withDescriptorProvider(descriptorProvider)));
    Renderer<?, ? extends CharSequence> renderer;

    renderer = rendererFactory.getRenderer(new NumberResult(1), TextRenderingFormat.class);
    assertEquals(ToStringTextRenderer.class, renderer.getClass());

    renderer = rendererFactory.getRenderer(new CrosstabResult(null), TextRenderingFormat.class);
    assertEquals(CrosstabTextRenderer.class, renderer.getClass());

    renderer = rendererFactory.getRenderer(new DataSetResult(new LinkedList<>()), TextRenderingFormat.class);
    assertEquals(MetricBasedResultTextRenderer.class, renderer.getClass());
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Override
protected void setUp() throws Exception {
    super.setUp();

    // define the datastore that the wrapped job will refer to
    final TableDataProvider<?> origTableDataProvider =
            new ArrayTableDataProvider(new SimpleTableDef("table", new String[] { "foo", "bar", "baz" }),
                    new ArrayList<>(0));
    final Datastore origInput = new PojoDatastore("orig_input", origTableDataProvider);

    // define the datastore that our actual job refers to
    final ArrayList<Object[]> rows = new ArrayList<>();
    final TableDataProvider<?> actualTableDataProvider =
            new ArrayTableDataProvider(new SimpleTableDef("table", new String[] { "name" }), rows);
    rows.add(new Object[] { "Tomasz" });
    rows.add(new Object[] { "Kasper" });
    rows.add(new Object[] { "Claudia" });
    rows.add(new Object[] { "Anders" });
    final Datastore actualInput = new PojoDatastore("actual_input", actualTableDataProvider);

    final DatastoreCatalog datastoreCatalog = new DatastoreCatalogImpl(origInput, actualInput);
    _configuration = new DataCleanerConfigurationImpl().withDatastoreCatalog(datastoreCatalog);
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
public void testFilterSinglePattern() throws Exception {
    final LifeCycleHelper lifeCycleHelper = new LifeCycleHelper(new DataCleanerConfigurationImpl(), null, true);

    final StringPattern stringPattern = new RegexStringPattern("very simple email pattern", "[email protected]+", true);
    final MockInputColumn<String> column = new MockInputColumn<>("my col", String.class);
    StringPatternFilter filter =
            new StringPatternFilter(column, new StringPattern[] { stringPattern }, MatchFilterCriteria.ANY,
                    configuration);

    lifeCycleHelper.initialize(descriptor, filter);

    assertEquals(ValidationCategory.VALID,
            filter.categorize(new MockInputRow().put(column, "[email protected]")));
    assertEquals(ValidationCategory.INVALID, filter.categorize(new MockInputRow().put(column, "[email protected]")));

    lifeCycleHelper.close(descriptor, filter, true);

    // it shouldn't matter if ANY or ALL criteria is being used
    filter = new StringPatternFilter(column, new StringPattern[] { stringPattern }, MatchFilterCriteria.ALL,
            configuration);
    filter.init();
    assertEquals(ValidationCategory.VALID,
            filter.categorize(new MockInputRow().put(column, "[email protected]")));
    assertEquals(ValidationCategory.INVALID, filter.categorize(new MockInputRow().put(column, "[email protected]")));
    filter.close();
}
 

import org.datacleaner.configuration.DataCleanerConfigurationImpl; //导入依赖的package包/类
@Override
protected void setUp() throws Exception {
    super.setUp();
    // set up a common fixture with a simple Max rows filter and a String
    // analyzer on the LASTNAME
    // column
    datastore = TestHelper.createSampleDatabaseDatastore("mydb");
    conf = new DataCleanerConfigurationImpl().withDatastores(datastore);
    ajb = new AnalysisJobBuilder(conf);
    ajb.setDatastore(datastore);
    maxRowsBuilder = ajb.addFilter(MaxRowsFilter.class);
    stringAnalyzerBuilder = ajb.addAnalyzer(StringAnalyzer.class);
    stringAnalyzerBuilder.setRequirement(maxRowsBuilder, MaxRowsFilter.Category.VALID);
    con = conf.getDatastoreCatalog().getDatastore("mydb").openConnection();
    lastnameColumn = con.getSchemaNavigator().convertToColumn("EMPLOYEES.LASTNAME");
    ajb.addSourceColumn(lastnameColumn);
    lastNameInputColumn = ajb.getSourceColumnByName("lastname");
    stringAnalyzerBuilder.addInputColumn(lastNameInputColumn);

    consumers = new ArrayList<>();

    baseQuery = con.getDataContext().query().from("EMPLOYEES").select("LASTNAME").toQuery();
}