本文整理汇总了C#中Tokenizer.Tokenizing方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.Tokenizing方法的具体用法？C# Tokenizer.Tokenizing怎么用？C# Tokenizer.Tokenizing使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.Tokenizing方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: m_tokenizingWorker_DoWork
private void m_tokenizingWorker_DoWork(object sender, DoWorkEventArgs e)
{
string[] _lines = txtbox_input.Lines;
int _currentProgress = 0;
m_tokenizingWorker.ReportProgress(0);
try
{
foreach (string _line in _lines)
{
try
{
string _inputPath = _line.Replace("\r", String.Empty);
var _readStream = new System.IO.FileStream(_inputPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.ReadWrite);
var _reader = new System.IO.StreamReader(_readStream, System.Text.Encoding.UTF8, true, 128);
string _id = _reader.ReadLine();
string _friendly_url = _reader.ReadLine();
string _title = _reader.ReadLine();
string _tags = _reader.ReadLine();
string _content = _reader.ReadToEnd();
_reader.Dispose();
_readStream.Dispose();
Tokenizer _tokenizer = new Tokenizer();
ArrayList _result = null;
if (m_untokenizedOutputDirectory == "")
_result = _tokenizer.Tokenizing(_content);
else
_result = _tokenizer.Tokenizing(_content, true, m_untokenizedOutputDirectory + "\\" + _id + '-' + _friendly_url + ".not");
string _outputPath = m_outputDirectory + "\\" + _id + '-' + _friendly_url + ".tok";
var _writetream = new System.IO.FileStream(_outputPath,
System.IO.FileMode.Create,
System.IO.FileAccess.Write,
System.IO.FileShare.ReadWrite);
var _writer = new System.IO.StreamWriter(_writetream, System.Text.Encoding.UTF8, 128);
_writer.Write(_id + Environment.NewLine + _friendly_url + Environment.NewLine +
_title + Environment.NewLine + _tags + Environment.NewLine +
string.Join(Environment.NewLine, _result.ToArray(typeof(string)) as string[]));
_writer.Dispose();
_writetream.Dispose();
m_inputTokenized += _outputPath + Environment.NewLine;
_currentProgress++;
m_tokenizingWorker.ReportProgress(_currentProgress * 100 / _lines.Count());
if(m_tokenizingWorker.CancellationPending)
{
e.Cancel = true;
return;
}
}
catch
{
}
}
}
catch
{
MessageBox.Show("No file to tokenize. Process failed...");
}
}