gorse-io / gorse-io/gorse

Slow 'Find neighbors of users' task

Open
#854 0 comments 0 reactions 0 assignees View on GitHub
bug
Dominant language
Go
Stars
9.8k
Forks
912
Avg merge
2h 35m
Merged PRs (30d)
9

Description

Please answer these questions before submitting your issue. Thanks!

**Gorse version**
0.4.15

**Describe the bug**
I have a Gorse instance deployed on a Kubernetes cluster with `master`, `server`, `proxy`, and `worker` instances. We have about 1.5 Million users with aprx. 20 labels for each user. The issue here is the process of the `Find neighbors of users` task which is very slow. It's been more than 3 days that the instance is up but this task has not been progressing.

![image](https://github.com/user-attachments/assets/6d5b1dff-3b2a-48d3-8422-b11a6769ba65)

**Expected behavior**
I want to be fast. If I should scale any service, which service would it be?

**Additional context**

Cache Store: Redis

Database: MySQL

Configs:

```yaml
[database]
cache_store = "redis://host:6379"

data_store = "mysql://user:pass@tcp(host)/database_name"
table_prefix = ""
cache_table_prefix = ""
data_table_prefix = ""

[master]
port = 8086
host = "0.0.0.0"
http_port = 8088
http_host = "0.0.0.0"

http_cors_domains = []
http_cors_methods = []

n_jobs = 20

meta_timeout = "10s"

dashboard_user_name = ""

dashboard_password = ""
admin_api_key = ""

[server]

default_n = 10

api_key = ""

clock_error = "5s"

auto_insert_user = false

auto_insert_item = false

cache_expire = "10s"

[recommend]

# The cache size for recommended/popular/latest items. The default value is 10.
cache_size = 500

# Recommended cache expire time. The default value is 72h.
cache_expire = "10m"

# The time-to-live (days) of active users, 0 means disabled. Recommendation won't be cached for inactive users. The default value is 0.
active_user_ttl = 20

[recommend.data_source]

# The feedback types for positive events.
positive_feedback_types = ["open", "watch","like"]

# The feedback types for read events.
read_feedback_types = ["read"]

# The time-to-live (days) of positive feedback, 0 means disabled. The default value is 0.
positive_feedback_ttl = 90

# The time-to-live (days) of items, 0 means disabled. The default value is 0.
item_ttl = 60

[recommend.popular]

# The time window of popular items. The default values is 4320h.
popular_window = "720h"

[recommend.user_neighbors]

neighbor_type = "similar"

enable_index = true

index_recall = 0.8

index_fit_epoch = 3

[recommend.item_neighbors]

neighbor_type = "similar"

enable_index = true

index_recall = 0.8

index_fit_epoch = 3

[recommend.collaborative]

enable_index = true

# Minimal recall for approximate collaborative filtering recommend. The default value is 0.9.
index_recall = 0.9

# Maximal number of fit epochs for approximate collaborative filtering recommend vector index. The default value is 3.
index_fit_epoch = 3
model_fit_period = "60m"
model_search_period = "360m"
model_search_epoch = 100

model_search_trials = 10

enable_model_size_search = false

[recommend.replacement]

enable_replacement = true

positive_replacement_decay = 0.8

read_replacement_decay = 0.2

[recommend.offline]
check_recommend_period = "1m"
refresh_recommend_period = "4h"
enable_latest_recommend = true
enable_popular_recommend = true
enable_user_based_recommend = true
enable_item_based_recommend = true
enable_collaborative_recommend = true
enable_click_through_prediction = true

explore_recommend = { popular = 0.2, latest = 0.1 }

[recommend.online]
fallback_recommend = ["item_based", "popular"]

num_feedback_fallback_item_based = 10

[tracing]

enable_tracing = false

exporter = "jaeger"

collector_endpoint = "http://localhost:14268/api/traces"

sampler = "always"

ratio = 1

[experimental]
enable_deep_learning = false
deep_learning_batch_size = 128
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.