本文整理汇总了C#中ITokenizer.Tokenize方法的典型用法代码示例。如果您正苦于以下问题：C# ITokenizer.Tokenize方法的具体用法？C# ITokenizer.Tokenize怎么用？C# ITokenizer.Tokenize使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类ITokenizer的用法示例。
在下文中一共展示了ITokenizer.Tokenize方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: AuthModule
public AuthModule(ITokenizer tokenizer, Users users)
: base("/auth")
{
Post["/"] = x =>
{
var request = this.Bind<AuthRequest>();
if (string.IsNullOrEmpty(request.Identifier))
{
return HttpStatusCode.Unauthorized;
}
var user = users.ValidateUsernameOrEmail(request.Identifier, request.Password);
if (user == null)
{
return HttpStatusCode.Unauthorized;
}
var userIdentity = new UserIdentity(user);
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
Token = token,
Username = user.Username,
UserId = user.Id
};
};
Get["/"] = _ =>
{
this.RequiresAuthentication();
return Context.CurrentUser;
};
}示例2: GetWordFrequency
/// <summary>
/// Gets a Hashtable of words and integers representing the number of each word.
/// </summary>
/// <param name="input">The string to get the word frequency of.</param>
/// <param name="caseSensitive">True if words should be treated as separate if they have different casing.</param>
/// <param name="tokenizer">A instance of ITokenizer.</param>
/// <param name="stopWordProvider">An instance of IStopWordProvider.</param>
/// <returns></returns>
public static Hashtable GetWordFrequency(string input, bool caseSensitive, ITokenizer tokenizer, IStopWordProvider stopWordProvider)
{
string convertedInput = input;
if (!caseSensitive)
convertedInput = input.ToLower();
string[] words = tokenizer.Tokenize(convertedInput);
Array.Sort(words);
string[] uniqueWords = GetUniqueWords(words);
Hashtable result = new Hashtable();
for (int i = 0; i < uniqueWords.Length; i++)
{
if (stopWordProvider == null || (IsWord(uniqueWords[i]) && !stopWordProvider.IsStopWord(uniqueWords[i])))
{
if (result.ContainsKey(uniqueWords[i]))
result[uniqueWords[i]] = (int)result[uniqueWords[i]] + CountWords(uniqueWords[i], words);
else
result.Add(uniqueWords[i], CountWords(uniqueWords[i], words));
}
}
return result;
}示例3: AuthModule
public AuthModule(ITokenizer tokenizer)
: base("/auth")
{
Post["/"] = x =>
{
var userName = (string)this.Request.Form.UserName;
var password = (string)this.Request.Form.Password;
var userIdentity = UserDatabase.ValidateUser(userName, password);
if (userIdentity == null)
{
return HttpStatusCode.Unauthorized;
}
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
Token = token,
};
};
Get["/validation"] = _ =>
{
this.RequiresAuthentication();
return "Yay! You are authenticated!";
};
Get["/admin"] = _ =>
{
this.RequiresClaims(new[] { "admin" });
return "Yay! You are authorized!";
};
}示例4: AuthModule
public AuthModule(ITokenizer tokenizer)
: base("/auth")
{
Options["/"] = o =>
{
return HttpStatusCode.OK;
};
Post["/"] = x =>
{
var userName = (string)this.Request.Form.UserName;
var password = (string)this.Request.Form.Password;
var userIdentity = UserDatabase.ValidateUser(userName, password);
if(userIdentity == null)
{
return HttpStatusCode.Unauthorized;
}
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
Token = token
};
};
}示例5: AuthModule
public AuthModule(ILogger<AuthModule> logger,
IUserManager userManager,
ITokenizer tokenizer)
: base("auth")
{
Get["/setup"] = _ => !userManager.HasUsers();
Post["/login"] = _ =>
{
var userData = this.Bind<UserDto>();
// First login creates user
if (!userManager.HasUsers())
{
logger.Info("Creating user account {UserName}.", userData.UserName);
userManager.CreateUser(userData.UserName, userData.Password);
}
var user = userManager.GetUser(userData.UserName, userData.Password);
if (user == null)
{
logger.Warn("Invalid username/password: {UserName}.", userData.UserName);
return HttpStatusCode.Unauthorized;
}
var identity = new UserIdentity(user.UserName, user.Claims);
var token = tokenizer.Tokenize(identity, Context);
return new
{
Token = token
};
};
}示例6: AuthModule
public AuthModule(ITokenizer tokenizer)
: base("/auth")
{
Post["/"] = x =>
{
var userName = this.Request.Form.UserName;
var password = this.Request.Form.Password;
var userIdentity = new UserIdentity();
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
Token = token
};
};
Get["/validation"] = _ =>
{
this.RequiresAuthentication();
return "Yay! You are authenticated!";
};
Get["/admin"] = _ =>
{
this.RequiresClaims(new[] { "admin" });
return "Yay! You are authorized!";
};
}示例7: AuthModule
public AuthModule(IConfiguration configuration, IAuthenticationManager authenticationManager, ITokenizer tokenizer)
: base("auth")
{
Get["/setup"] = _ => string.IsNullOrEmpty(configuration.UserName);
Post["/login"] = _ =>
{
var loginParameters = this.Bind<LoginParameters>();
if (string.IsNullOrEmpty(configuration.UserName))
{
SetAuth(configuration, loginParameters);
}
if (!authenticationManager.IsValid(loginParameters.UserName, loginParameters.Password))
{
return HttpStatusCode.Unauthorized;
}
var identity = new UserIdentity(loginParameters.UserName, null);
var token = tokenizer.Tokenize(identity, Context);
return new
{
Token = token
};
};
}示例8: AuthModule
public AuthModule(ITokenizer tokenizer)
: base("/auth")
{
this.EnableCors();
Post["/"] = x =>
{
var userName = (string) this.Request.Form.UserName;
var password = (string) this.Request.Form.Password;
var userIdentity = UserDatabase.ValidateUser(userName, password);
if (userIdentity == null)
{
return HttpStatusCode.Unauthorized;
}
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
access_token = token,
};
};
Get["/validation"] = _ =>
{
this.RequiresAuthentication();
return HttpStatusCode.OK;
};
Options["/authenticated-content"] = _ => HttpStatusCode.OK;
Get["/authenticated-content"] = _ =>
{
this.RequiresAuthentication();
return Response.AsJson(new
{
message = "Yay! You are authenticated! :)"
});
};
Get["/admin"] = _ =>
{
this.RequiresClaims(new[] { "admin" });
return Response.AsJson(new
{
message = "Yay! You are the admin :P"
});
};
Options["/"] = _ => HttpStatusCode.OK;
}示例9: TestWithAMillionWords
public static void TestWithAMillionWords(ITokenizer tokenizer)
{
string text = File.ReadAllText(path, Encoding.UTF8);
Stopwatch sw = Stopwatch.StartNew();
for (int i = 0; i < 1000; i++)
{
tokenizer.Tokenize(text);
}
sw.Stop();
Console.WriteLine("Time taken to tokenizer 85 MB text: {0}ms", sw.Elapsed.TotalMilliseconds);
GC.Collect();
}示例10: TestTokenizer
internal static void TestTokenizer(ITokenizer tokenizer) {
var tokens = tokenizer.Tokenize("Sounds like it's not properly thought through!");
Assert.AreEqual(9, tokens.Length);
Assert.AreEqual("Sounds", tokens[0]);
Assert.AreEqual("like", tokens[1]);
Assert.AreEqual("it", tokens[2]);
Assert.AreEqual("'s", tokens[3]);
Assert.AreEqual("not", tokens[4]);
Assert.AreEqual("properly", tokens[5]);
Assert.AreEqual("thought", tokens[6]);
Assert.AreEqual("through", tokens[7]);
Assert.AreEqual("!", tokens[8]);
}示例11: AuthModule
public AuthModule(ITokenizer tokenizer)
: base("/auth")
{
Post["/token"] = x =>
{
IUserIdentity indentity = new BasicIdentity { UserName = "admin", Claims = new List<string> { "Money", "Power" } };
var token = tokenizer.Tokenize(indentity, Context);
return new { Token = token };
};
Post["/apikey"] = x =>
{
return new { ApiKey = "c5c16da7-e0fc-4e14-a522-3dab15a7a593" };
};
}示例12: ImportSparse
public static IDictionary<string, double>[] ImportSparse(string filePath, IReworder reworder, IReader reader, ITokenizer tokenizer)
{
List<IDictionary<string, double>> encyclopedia = new List<IDictionary<string, double>>();
int linesRead = 0;
foreach (string line in LinesEnumerator.YieldLines(filePath))
{
IDictionary<string, double> res = tokenizer.Tokenize(reader.Read(ReworderHelper.Map(line,reworder)));
encyclopedia.Add(res);
linesRead++;
if ((linesRead % DisplaySettings.PrintProgressEveryLine) == 0)
{
Console.Write('.');
}
}
return encyclopedia.ToArray();
}示例13: AuthenticateModule
public AuthenticateModule(ITokenizer tokenizer)
{
Post["/authenticate"] = x => {
var user = this.Bind<User>();
var username = Environment.GetEnvironmentVariable(adminKey);
var password = Environment.GetEnvironmentVariable(passwordKey);
if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) return HttpStatusCode.Unauthorized;
if (user.Username != username || user.Password != password) return HttpStatusCode.Unauthorized;
var identity = new UserIdentity {
UserName = user.Username,
Claims = new[] { "admin" }
};
var token = tokenizer.Tokenize(identity, Context);
return Response.AsJson(new Authenticate { Token = token });
};
}示例14: SecureModule
public SecureModule(ITokenizer tokenizer)
: base("/secure")
{
this.RequiresAuthentication();
Get["/"] = _ =>
{
ViewBag.CurrentUser = this.Context.CurrentUser.UserName;
return View["secure"];
};
Get["/token"] = _ =>
{
ViewBag.Token = tokenizer.Tokenize(Context.CurrentUser, Context);
return View["token"];
};
Get["/logout"] = _ =>
{
return this.LogoutAndRedirect("/");
};
}示例15: AuthModule
public AuthModule(IMongoDatabase database, ITokenizer tokenizer)
: base("/api/auth")
{
Post["/"] = x =>
{
User user = this.Bind<User>(); // Nancy.ModelBinding
var userIdentity = UserDAO.ValidateUser(user);
if (userIdentity == null)
{
return HttpStatusCode.Unauthorized;
}
var token = tokenizer.Tokenize(userIdentity, Context);
return new
{
id = token,
user = new {
id = userIdentity.UserName,
role = userIdentity.Claims
}
};
};
Get["/validation"] = _ =>
{
this.RequiresAuthentication();
return "Yay! You are authenticated!";
};
Get["/admin"] = _ =>
{
this.RequiresClaims(new[] { "admin" });
return "Yay! You are authorized!";
};
}