Slow 'Find neighbors of users' task
- Dominant language
- Go
- Stars
- 9.8k
- Forks
- 912
- Avg merge
- 2h 35m
- Merged PRs (30d)
- 9
Description
Please answer these questions before submitting your issue. Thanks!
**Gorse version**
0.4.15
**Describe the bug**
I have a Gorse instance deployed on a Kubernetes cluster with `master`, `server`, `proxy`, and `worker` instances. We have about 1.5 Million users with aprx. 20 labels for each user. The issue here is the process of the `Find neighbors of users` task which is very slow. It's been more than 3 days that the instance is up but this task has not been progressing.

**Expected behavior**
I want to be fast. If I should scale any service, which service would it be?
**Additional context**
Cache Store: Redis
Database: MySQL
Configs:
```yaml
[database]
cache_store = "redis://host:6379"
data_store = "mysql://user:pass@tcp(host)/database_name"
table_prefix = ""
cache_table_prefix = ""
data_table_prefix = ""
[master]
port = 8086
host = "0.0.0.0"
http_port = 8088
http_host = "0.0.0.0"
http_cors_domains = []
http_cors_methods = []
n_jobs = 20
meta_timeout = "10s"
dashboard_user_name = ""
dashboard_password = ""
admin_api_key = ""
[server]
default_n = 10
api_key = ""
clock_error = "5s"
auto_insert_user = false
auto_insert_item = false
cache_expire = "10s"
[recommend]
# The cache size for recommended/popular/latest items. The default value is 10.
cache_size = 500
# Recommended cache expire time. The default value is 72h.
cache_expire = "10m"
# The time-to-live (days) of active users, 0 means disabled. Recommendation won't be cached for inactive users. The default value is 0.
active_user_ttl = 20
[recommend.data_source]
# The feedback types for positive events.
positive_feedback_types = ["open", "watch","like"]
# The feedback types for read events.
read_feedback_types = ["read"]
# The time-to-live (days) of positive feedback, 0 means disabled. The default value is 0.
positive_feedback_ttl = 90
# The time-to-live (days) of items, 0 means disabled. The default value is 0.
item_ttl = 60
[recommend.popular]
# The time window of popular items. The default values is 4320h.
popular_window = "720h"
[recommend.user_neighbors]
neighbor_type = "similar"
enable_index = true
index_recall = 0.8
index_fit_epoch = 3
[recommend.item_neighbors]
neighbor_type = "similar"
enable_index = true
index_recall = 0.8
index_fit_epoch = 3
[recommend.collaborative]
enable_index = true
# Minimal recall for approximate collaborative filtering recommend. The default value is 0.9.
index_recall = 0.9
# Maximal number of fit epochs for approximate collaborative filtering recommend vector index. The default value is 3.
index_fit_epoch = 3
model_fit_period = "60m"
model_search_period = "360m"
model_search_epoch = 100
model_search_trials = 10
enable_model_size_search = false
[recommend.replacement]
enable_replacement = true
positive_replacement_decay = 0.8
read_replacement_decay = 0.2
[recommend.offline]
check_recommend_period = "1m"
refresh_recommend_period = "4h"
enable_latest_recommend = true
enable_popular_recommend = true
enable_user_based_recommend = true
enable_item_based_recommend = true
enable_collaborative_recommend = true
enable_click_through_prediction = true
explore_recommend = { popular = 0.2, latest = 0.1 }
[recommend.online]
fallback_recommend = ["item_based", "popular"]
num_feedback_fallback_item_based = 10
[tracing]
enable_tracing = false
exporter = "jaeger"
collector_endpoint = "http://localhost:14268/api/traces"
sampler = "always"
ratio = 1
[experimental]
enable_deep_learning = false
deep_learning_batch_size = 128
```
Contributor guide
Assessment
This issue has not been assessed yet.