nlp :: NLPIndex :: accept_token
token
accepted by this index?See whitelist_pos
and blacklist_pos
.
# Is `token` accepted by this index?
#
# See `whitelist_pos` and `blacklist_pos`.
fun accept_token(token: NLPToken): Bool do
var pos = token.pos
if whitelist_pos.not_empty and not whitelist_pos.has(pos) then return false
if blacklist_pos.has(pos) then return false
if stoplist.has(token.lemma) then return false
return true
end
lib/nlp/nlp.nit:48,2--57,4