mars-project / mars-project/mars
Using mars remote does not speed up the function, please help to see the reason,the following is the source code
- Dominant language
- Python
- Stars
- 2.7k
- Forks
- 325
- PR merge metrics
- No merged PRs in 30d
Description
Reproduce code:
``` Python
import math
import mars.dataframe as md
import mars.tensor as mt
import os
import random
import time
import pandas as pd
import numpy as np
import json
import mars.remote as mr
class Towerrank(object):
def __init__(self, filename, SafeFileDir, middlePath):
self.SafeFileDir, self.middlefiledir = SafeFileDir, middlePath
self.dir_name = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.Kvalue = 0.000062
self.start = 7#8
self.x_step = 20
self.n = 10
self.block = 1000000
self.save_line = 10000
first_tower = eval(filename.split('.csv')[0].split('-')[0])
end_tower = eval(filename.split('.csv')[0].split('-')[1])
self.df = pd.read_csv(filename,index_col=0)
self.tower_point = str(first_tower) + '---' + str(end_tower)
with open('angle.json', 'r') as f:
self.angle = json.load(f)
# ????????????
def merge_data(self, route_tag, v, left_pd, route_dis):
for (index, t) in enumerate(route_tag):
t = np.array(eval(t)) # str->list->array
# t = eval(str(t.execute()))
right_pd = pd.DataFrame(t)
right_pd.columns = [index, index + 1]
if left_pd.empty != True:
if len(v) >= self.start and index >= 6:
left_pd = pd.merge(left_pd, right_pd)
else:
left_pd = pd.merge(left_pd, right_pd)
else:
left_pd = right_pd
if left_pd.shape[1] != len(v):
return False
data_shape = left_pd.shape
self.arr_index += data_shape[0]
route_point_dis = [(value - 1) * self.x_step for value in v]
attitude_li = [n*10 for n in v]
route_sign = np.tile(np.array(v, dtype='int64'), (data_shape[0], 1))
attitude_array = np.tile(np.array(attitude_li, dtype='float'), (data_shape[0], 1))
route_array_dis = np.tile(np.array(route_point_dis, dtype='float'), (data_shape[0], 1))
return (left_pd, route_array_dis, data_shape, route_sign, attitude_array)
def remoteFun(self,xs):
return xs
def singleRoute(self,v,route_tag,route_dis,left_pd):
for (index, loc) in enumerate(v):
if index == 0:
continue
route_tag.append(self.df.iloc[v[index - 1], v[index]])
route_dis.append(v[index] - v[index - 1])
data = self.merge_data(route_tag, v, left_pd, route_dis)
return data
def ranking(self):
# max_point = 7
# with open("array.txt","r") as f:
# route = eval(f.read())
# route = route[:20]
route = [[1, 10, 19, 39, 59, 67],
[1, 11, 19, 39, 59, 67],
[1, 12, 19, 39, 59, 67],
[1, 13, 19, 39, 59, 67],
[1, 14, 19, 39, 59, 67],
[1, 10, 20, 39, 59, 67],
[1, 11, 20, 39, 59, 67],
[1, 12, 20, 39, 59, 67],
[1, 13, 20, 39, 59, 67],
[1, 14, 20, 39, 59, 67],
[1, 10, 21, 39, 59, 67],
[1, 11, 21, 39, 59, 67],
[1, 12, 21, 39, 59, 67],
[1, 13, 21, 39, 59, 67],
[1, 14, 21, 39, 59, 67],
[1, 11, 22, 39, 59, 67],
[1, 12, 22, 39, 59, 67],
[1, 13, 22, 39, 59, 67],
[1, 14, 22, 39, 59, 67],
[1, 11, 23, 39, 59, 67]]
print("route length:",len(route))
merge_data_li = []
self.arr_index = 0
for k, v in enumerate(route):
route_tag = []
route_dis = []
left_pd = pd.DataFrame()
data = mr.spawn(self.singleRoute, args=(v, route_tag, route_dis, left_pd))
if not data:
continue
merge_data_li.append(data)
return mr.ExecutableTuple(merge_data_li).execute().fetch()
def main(self):
print(self.ranking())
def run_tower_ranking(SafeFileDir,middlePath):
for file in os.listdir(SafeFileDir):
if file.endswith('.csv'):
start_time = time.time()
top = Towerrank(file, SafeFileDir, middlePath).main()
end_time = time.time()
cost_time = end_time - start_time
print('??:{}'.format(file, cost_time))
if __name__ == '__main__':
start = time.time()
SAFE_DIR = "./"
middlePath = "./"
run_tower_ranking(SAFE_DIR, middlePath)
end = time.time()
print('??', end - start)
```
Contributor guide
Research direction
Start with the reproducible Python script, especially ranking(), mr.spawn(), and mr.ExecutableTuple(merge_data_li).execute().fetch(). Run it with the supplied route data and compare the execution timing with and without the remote calls. Done means identifying the cause of the missing speedup and documenting a reproducible explanation or required change.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- numpy, pandas, python
- Domain
- data, distributed-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 15/100