JuliaText / JuliaText/TextAnalysis.jl
Performance issue with prepare!( doc, strip_stopwords)
- Dominant language
- Julia
- Stars
- 384
- Forks
- 92
- PR merge metrics
- No merged PRs in 30d
Description
Calling `prepare!(StringDocument, strip_case | strip_stopwords)` even on a small ~3.4MB file takes forever to return (other than for small strings I have not seen this function finishing successfully).
```julia
function method_textanalysis(str_rec)
sdoc = StringDocument(str_rec)
prepare!(sdoc, strip_case | strip_stopwords)
return sdoc
end
```
I have tracked the slowness to the following function.
https://github.com/JuliaText/TextAnalysis.jl/blob/c8ae7a217d19f19d8c8e3e22da9ea5970ece40d4/src/preprocessing.jl#L253
```julia
function remove_patterns(s::AbstractString, rex::Regex)
iob = IOBuffer()
ibegin = 1
v=codeunits(s)
for m in eachmatch(rex, s)
len = m.match.offset-ibegin+1
next = nextind(s, lastindex(m.match)+m.match.offset)
if len > 0
Base.write_sub(iob, v, ibegin, len)
if next != length(s)+1
write(iob, ' ')
end
end
ibegin = next
end
len = length(v) - ibegin + 1
(len > 0) && Base.write_sub(iob, v, ibegin, len)
String(take!(iob))
end
```
Manually performing similar task takes ~1.4 second on a 3.4MB text file. Reason I say similar is because to eliminate stop words manually I first tokenize the document and then filter out the stop words. Which functionally is very different than executing regex on a large string and may not be the ideal approach for preserving the structure of document. (**complete code at the end**)
```julia
function method_manual(str_rec)
stop_words = Languages.stopwords(Languages.English())
str_rec = lowercase(str_rec)
word_tokens = tokenize(str_rec)
res = filter(x->!in(x, stop_words), word_tokens)
return res
end
```
**I was wondering if there could be a more efficient way to perform this elimination of keywords from a String Document?**
---
```julia
using Pkg
@info "Installing required packages.."
required_pkg = ["TextAnalysis", "Languages", "WordTokenizers"]
installed_pkg = Pkg.installed()
[(in(p, keys(installed_pkg)) || Pkg.add(p)) for p in required_pkg]
using TextAnalysis
using WordTokenizers
using Languages
"""
Download data if it does not already exists
"""
function download_data(;
url="http://www.gutenberg.org/files/2600/2600-0.txt",
localfile="2600-0.txt")
if !isfile(localfile)
download(url, localfile)
else
@info "file $(localfile) already exists, skipping download"
end
end
"""
Return data (~100MB uncompressed) in form of a string
"""
function getdata(fn="2600-0.txt")
download_data()
str_rec = read(fn, String)
end
"""
Pre Process data using TextAnalysis - strip_case | strip_stopwords
"""
function method_textanalysis(str_rec)
sdoc = StringDocument(str_rec)
#prepare!(sdoc, strip_case | strip_stopwords)
prepare!(sdoc, strip_stopwords)
return sdoc
end
"""
Pre Process data without using `prepare!`
"""
function method_manual(str_rec)
stop_words = Languages.stopwords(Languages.English())
str_rec = lowercase(str_rec)
word_tokens = tokenize(str_rec)
res = filter(x->!in(x, stop_words), word_tokens)
return res
end
"""
Main
"""
function main()
str_rec = getdata()
@info "Manual Pre Processing"
@time res2 = method_manual(str_rec)
@info "Pre Processing using TextAnalysis"
@time res1 = method_textanalysis(str_rec)
end
main()
```
Contributor guide
No contributing guide indexed for this repository
Assessment
This issue has not been assessed yet.