Java CharSequenceLexer类代码示例

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public static void main (String[] args)
{
	try {
		for (int i = 0; i < args.length; i++) {
			Instance carrier = new Instance (new File(args[i]), null, null, null);
			SerialPipes p = new SerialPipes (new Pipe[] {
				new Input2CharSequence (),
				new CharSequence2TokenSequence(new CharSequenceLexer())});
			carrier = p.newIteratorFrom (new SingleInstanceIterator(carrier)).next();
			TokenSequence ts = (TokenSequence) carrier.getData();
			System.out.println ("===");
			System.out.println (args[i]);
			System.out.println (ts.toString());
		}
	} catch (Exception e) {
		System.out.println (e);
		e.printStackTrace();
	}
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	// xxx This doesn't seem so efficient.  Perhaps have TokenSequence
	// use a LinkedList, and remove Tokens from it? -?
	// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
	TokenSequence ret = new TokenSequence ();
	Token prevToken = null;
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		String s = t.getText();
		if (CharSequenceLexer.LEX_ALPHA.matcher(s).matches()) {
			ret.add (t);
			prevToken = t;
		}	else if (markDeletions && prevToken != null)
			prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
	}
	carrier.setData(ret);
	return carrier;
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void testToXml () {
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label VB = dict.lookupLabel ("VERB");
  LabelSequence tags = new LabelSequence (new Label[] { O, ANML, ANML, ANML, VB, O, O, ANML, ANML });

  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, tags, "O");
  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown fox </ANIMAL><VERB>leapt </VERB>over the <ANIMAL>lazy dog</ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void testToXmlBIO () {
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label BANML = dict.lookupLabel ("B-ANIMAL");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label BVB = dict.lookupLabel ("B-VERB");
  Label VB = dict.lookupLabel ("I-VERB");
  LabelSequence tags = new LabelSequence (new Label[] { O, BANML, ANML, BANML, BVB, VB, O, ANML, ANML });

  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, tags, null, "O", new BIOTokenizationFilter());
  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown </ANIMAL><ANIMAL>fox </ANIMAL><VERB>leapt over </VERB>the <ANIMAL>lazy dog</ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public static void main (String[] args)
{
	try {
		for (int i = 0; i < args.length; i++) {
			Instance carrier = new Instance (new File(args[i]), null, null, null);
			BigSerialPipes p = new BigSerialPipes (new BigPipe[] {
				new BigInput2CharSequence (),
				new BigCharSequence2TokenSequence(new CharSequenceLexer())});
			carrier = p.newIteratorFrom (new BigSingleInstanceIterator(carrier)).next();
			TokenSequence ts = (TokenSequence) carrier.getData();

		}
	} catch (Exception e) {
		
		e.printStackTrace();
	}
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void ignoretestToXml () {
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label VB = dict.lookupLabel ("VERB");
  LabelSequence tags = new LabelSequence (new Label[] { O, ANML, ANML, ANML, VB, O, O, ANML, ANML });

  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, tags, "O");
  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown fox </ANIMAL><VERB>leapt </VERB>over the <ANIMAL>lazy dog</ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void ignoretestToXmlBIO () {
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label BANML = dict.lookupLabel ("B-ANIMAL");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label BVB = dict.lookupLabel ("B-VERB");
  Label VB = dict.lookupLabel ("I-VERB");
  LabelSequence tags = new LabelSequence (new Label[] { O, BANML, ANML, BANML, BVB, VB, O, ANML, ANML });

  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, tags, null, "O", new BIOTokenizationFilter());
  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown </ANIMAL><ANIMAL>fox </ANIMAL><VERB>leapt over </VERB>the <ANIMAL>lazy dog</ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
/**
	 @param lexer to tokenize input
	 @param backgroundTag default tag when not in any other tag
	 @param allowed set of tags (Strings) that will be converted to
	 labels
 */
public SelectiveSGML2TokenSequence (CharSequenceLexer lexer, String backgroundTag, Set allowed)
{
	this.lexer = lexer;
	this.backgroundTag = backgroundTag;
	this.allowedTags = allowed;
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException {
	int version = in.readInt ();
	sgmlPattern = (Pattern) in.readObject();
	lexer = (CharSequenceLexer) in.readObject();
	backgroundTag = (String) in.readObject();
	allowedTags = (Set) in.readObject();
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException {
	int version = in.readInt ();
	sgmlPattern = (Pattern) in.readObject();
	lexer = (CharSequenceLexer) in.readObject();
	backgroundTag = (String) in.readObject();
   if (version == 0) {
     saveSource = true;
   }
   else {
     saveSource = in.readBoolean();
   }
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
/**
 * Creates a tokenization of the given string.  Tokens are
 * added from all the matches of the given lexer.
 */
public StringTokenization (CharSequence string, CharSequenceLexer lexer)
{
  super();
  this.document = string;

  lexer.setCharSequence (string);
  while (lexer.hasNext()) {
    lexer.next ();
    this.add (new StringSpan (string, lexer.getStartOffset(), lexer.getEndOffset()));
  }
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
private Extraction createExtractionFrom (String[] predStrings, String[] trueStrings)
{
  Pipe pipe = new SerialPipes (new Pipe[] {
    new SGML2TokenSequence (new CharSequenceLexer (CharSequenceLexer.LEX_NONWHITESPACE_CLASSES	), "O"),
    new Target2LabelSequence (),
    new PrintInputAndTarget (),
  });

  InstanceList pred = new InstanceList (pipe);
  pred.addThruPipe (new ArrayIterator (predStrings));

  InstanceList targets = new InstanceList (pipe);
  targets.addThruPipe (new ArrayIterator (trueStrings));

  LabelAlphabet dict = (LabelAlphabet) pipe.getTargetAlphabet ();
  Extraction extraction = new Extraction (null, dict);

  for (int i = 0; i < pred.size(); i++) {
    Instance aPred = pred.get (i);
    Instance aTarget = targets.get (i);
    Tokenization input = (Tokenization) aPred.getData ();
    Sequence predSeq = (Sequence) aPred.getTarget ();
    Sequence targetSeq = (Sequence) aTarget.getTarget ();
    DocumentExtraction docextr = new DocumentExtraction ("TEST"+i, dict, input, predSeq, targetSeq, "O");
    extraction.addDocumentExtraction (docextr);
  }

  return extraction;
}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void testNestedToXML ()
{
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label VB = dict.lookupLabel ("VERB");
  Label JJ = dict.lookupLabel ("ADJ");
  Label MAMMAL = dict.lookupLabel ("MAMMAL");

  LabelSequence tags = new LabelSequence (new Label[] { O, ANML, ANML, ANML, VB, O, ANML, ANML, ANML });

  LabeledSpans spans = new DefaultTokenizationFilter ().constructLabeledSpans (dict, document, O, toks, tags);

  Span foxToken = toks.subspan (3, 4);
  spans.add (new LabeledSpan (foxToken, MAMMAL, false));
  Span bigDogToken = toks.subspan (7, 8);
  spans.add (new LabeledSpan (bigDogToken, JJ, false));

  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, spans, null, "O");
  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown <MAMMAL>fox </MAMMAL></ANIMAL><VERB>leapt </VERB>over <ANIMAL>the <ADJ>lazy </ADJ>dog</ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);

}
 

import cc.mallet.util.CharSequenceLexer; //导入依赖的package包/类
public void testNestedXMLTokenizationFilter ()
{
  LabelAlphabet dict = new LabelAlphabet ();
  String document = "the quick brown fox leapt over the lazy dog";
  StringTokenization toks = new StringTokenization (document, new CharSequenceLexer ());

  Label O = dict.lookupLabel ("O");
  Label ANML = dict.lookupLabel ("ANIMAL");
  Label ANML_MAMM = dict.lookupLabel ("ANIMAL|MAMMAL");
  Label VB = dict.lookupLabel ("VERB");
  Label ANML_JJ = dict.lookupLabel ("ANIMAL|ADJ");
  Label ANML_JJ_MAMM = dict.lookupLabel ("ANIMAL|ADJ|MAMMAL");

  LabelSequence tags = new LabelSequence (new Label[] { O, ANML, ANML, ANML_MAMM, VB, O, ANML, ANML_JJ, ANML_JJ_MAMM });
  DocumentExtraction extr = new DocumentExtraction ("Test", dict, toks, tags, null, "O", new HierarchicalTokenizationFilter ());

  String actualXml = extr.toXmlString();
  String expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown <MAMMAL>fox </MAMMAL></ANIMAL><VERB>leapt </VERB>over <ANIMAL>the <ADJ>lazy <MAMMAL>dog</MAMMAL></ADJ></ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);

  // Test the ignore function

  extr = new DocumentExtraction ("Test", dict, toks, tags, null, "O", new HierarchicalTokenizationFilter (Pattern.compile ("AD.*")));

  actualXml = extr.toXmlString();
  expectedXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n" +
          "<doc>the <ANIMAL>quick brown <MAMMAL>fox </MAMMAL></ANIMAL><VERB>leapt </VERB>over <ANIMAL>the lazy <MAMMAL>dog</MAMMAL></ANIMAL></doc>\r\n";
  assertEquals (expectedXml, actualXml);



}