Class DefaultTokenContextGenerator

Namespace

NOpenNLP.Tools.Tokenize

Assembly

NOpenNLP.Tools.dll

Generate events for maxent decisions for tokenization.

public class DefaultTokenContextGenerator : ITokenContextGenerator

Inheritance

object

DefaultTokenContextGenerator

Implements

ITokenContextGenerator

Inherited Members

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Constructors

DefaultTokenContextGenerator()

Creates a default context generator for tokenizer.

public DefaultTokenContextGenerator()

DefaultTokenContextGenerator(ISet<string>)

Creates a default context generator for tokenizer.

public DefaultTokenContextGenerator(ISet<string> inducedAbbreviations)

Parameters

inducedAbbreviations ISet<string>

the induced abbreviations

Fields

inducedAbbreviations

protected readonly ISet<string> inducedAbbreviations

Field Value

ISet<string>

Methods

AddCharPreds(string, char, IList<string>)

Helper function for getContext.

protected virtual void AddCharPreds(string key, char c, IList<string> preds)

Parameters

key string

c char

preds IList<string>

CreateContext(string, int)

Returns a list of features for the specified sentence string at the specified index. Extensions of this class can override this method to create a customized ITokenContextGenerator

protected virtual IList<string> CreateContext(string sentence, int index)

Parameters

sentence string

the token been analyzed

index int

the index of the character been analyzed

Returns

IList<string>

a list of features for the specified sentence string at the specified index.

GetContext(string, int)

Returns an array of features for the specified sentence string at the specified index.

public virtual string[] GetContext(string sentence, int index)

Parameters

sentence string

The string for a sentence.

index int

The index to consider splitting as a token.

Returns

string[]

an array of features for the specified sentence string at the specified index.