intel / intel/hyperscan

Potential memory leak in `hs_compile` and `hs_compile_multi`

Open
#235 10 comments 0 reactions 0 assignees View on GitHub
Dominant language
C++
Stars
5.5k
Forks
816
Avg merge
4d 21h
Merged PRs (30d)
2

Description

Hi, I found a potential memory leak issue in both `hs_compile` and `hs_compile_multi`.

A toy example C++ code generating DFA on top of 1000 regexes for multiple rounds (2000 rounds each) shows increasing memory usage.

```c++
#include
#include
#include
#include
#include
#include
#include
#include

using namespace std;

const unsigned FLAG = HS_FLAG_CASELESS | HS_FLAG_ALLOWEMPTY;
const int LIMIT = 1000;

vector readPatternsFromFile() {
FILE *pfile = fopen("regex.txt", "r");
char in[1010];
vector patterns;
int cnt = 0;
while (fgets(in, 1000, pfile) != NULL and cnt < LIMIT)
{
in[strlen(in) - 1] = 0;
patterns.push_back(in);
cnt += 1;
}
fclose(pfile);
return patterns;
}

vector validateRegexes(
vector& untested_patterns
) {
vector valid_patterns;
for (auto& untested_pattern : untested_patterns){
hs_database_t *db;
hs_compile_error_t *compileErr;
if (hs_compile(untested_pattern.c_str(), FLAG, HS_MODE_BLOCK, NULL, &db, &compileErr) == HS_SUCCESS) {
valid_patterns.push_back(untested_pattern);
} else {
//fprintf(stderr, "ERROR: Unable to compile pattern \"%s\": %s\n", untested_pattern.c_str(), compileErr->message);
}
hs_free_compile_error(compileErr);
hs_free_database(db);
}
return valid_patterns;
}

hs_database_t *buildDatabase(vector& valid_patterns) {
vector expressions;
vector flags;
vector ids;
for (size_t idx = 0; idx < valid_patterns.size(); idx += 1) {
expressions.push_back(valid_patterns[idx].c_str());
flags.push_back(FLAG);
ids.push_back(idx);
}

hs_database_t *db;
hs_compile_error_t *compileErr;
assert(hs_compile_multi(&expressions[0], &flags[0], &ids[0], valid_patterns.size(), HS_MODE_BLOCK, NULL, &db, &compileErr) == 0);
hs_free_compile_error(compileErr);
return db;
}

int onMatch(unsigned int id, unsigned long long from, unsigned long long to, unsigned int flags, void *ctx) {
// Our context points to a size_t storing the match count
size_t *matches = (size_t *)ctx;
(*matches)++;
return 0; // continue matching
}

int main()
{
vector untested_patterns = readPatternsFromFile();
vector valid_patterns;
for (int idx = 0; idx < 2000; idx += 1)
{
valid_patterns = validateRegexes(untested_patterns);
if (idx % 10 == 0) {
printf("Now is at %d\n", idx);
}
}

for (int idx = 0; idx < 2000; idx += 1){
hs_database_t *db_block = buildDatabase(valid_patterns);
if (idx % 10 == 0) {
printf("Now is at %d\n", idx);
}
hs_free_database(db_block);
}
hs_database_t *db_block = buildDatabase(valid_patterns);

string text = "fxck";
hs_scratch_t *scratch(nullptr);
assert(hs_alloc_scratch(db_block, &scratch) == 0);

for (int idx = 0; idx < 1000; idx += 1) {
size_t matchCount = 0;
assert(hs_scan(db_block, text.c_str(), text.length(), 0, scratch, onMatch, &matchCount) == 0);
if (idx % 1000 == 0) {
printf("%d: Matched %u\n", idx, (unsigned int)matchCount);
sleep(1);
}
}
}
```

After running this toy C++ example, `runner.cpp`, I used `psrecord $(pgrep -f 'runner') --interval 1 --plot runner.png` to capture the CPU and MEM usages on my local machine, which is shown as follows.

![image](https://user-images.githubusercontent.com/1198644/80922359-725f8d00-8d31-11ea-9d62-3c3df72b840e.png)

I believe I have already freed up all the allocated memory by calling both `hs_free_compile_error` and `hs_free_database` and should not be seeing memory increase over time. Can someone shed some light on this?

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.