vsm :: StringIndex :: parse_string
string as a VectorReturns a vector containing the terms of string.
# Parse the `string` as a Vector
#
# Returns a vector containing the terms of `string`.
fun parse_string(string: String): Vector do
var reader = new StringReader(string)
var vector = new Vector
loop
var token = reader.read_word
if token == "" then break
vector.inc(token)
end
return vector
end
lib/vsm/vsm.nit:240,2--252,4
redef fun parse_string(string) do
var vector = new Vector
if string.trim.is_empty then return vector
var doc = nlp_processor.process(string)
for sentence in doc.sentences do
for token in sentence.tokens do
if not accept_token(token) then continue
var lemma = token.lemma
if not vector.has_key(lemma) then
vector[lemma] = 1.0
else
vector[lemma] += 1.0
end
end
end
return vector
end
lib/nlp/nlp.nit:30,2--46,4