mars-project / mars-project/mars

Using mars remote does not speed up the function, please help to see the reason,the following is the source code

Open
#2,047 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
2.7k
Forks
325
PR merge metrics
No merged PRs in 30d

Description

Reproduce code:
``` Python
import math
import mars.dataframe as md
import mars.tensor as mt
import os
import random
import time
import pandas as pd
import numpy as np
import json
import mars.remote as mr

class Towerrank(object):
def __init__(self, filename, SafeFileDir, middlePath):
self.SafeFileDir, self.middlefiledir = SafeFileDir, middlePath
self.dir_name = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.Kvalue = 0.000062
self.start = 7#8
self.x_step = 20
self.n = 10
self.block = 1000000
self.save_line = 10000
first_tower = eval(filename.split('.csv')[0].split('-')[0])
end_tower = eval(filename.split('.csv')[0].split('-')[1])
self.df = pd.read_csv(filename,index_col=0)
self.tower_point = str(first_tower) + '---' + str(end_tower)

with open('angle.json', 'r') as f:
self.angle = json.load(f)

# ????????????
def merge_data(self, route_tag, v, left_pd, route_dis):
for (index, t) in enumerate(route_tag):
t = np.array(eval(t)) # str->list->array
# t = eval(str(t.execute()))
right_pd = pd.DataFrame(t)
right_pd.columns = [index, index + 1]

if left_pd.empty != True:
if len(v) >= self.start and index >= 6:
left_pd = pd.merge(left_pd, right_pd)
else:
left_pd = pd.merge(left_pd, right_pd)
else:
left_pd = right_pd

if left_pd.shape[1] != len(v):
return False
data_shape = left_pd.shape
self.arr_index += data_shape[0]
route_point_dis = [(value - 1) * self.x_step for value in v]
attitude_li = [n*10 for n in v]
route_sign = np.tile(np.array(v, dtype='int64'), (data_shape[0], 1))
attitude_array = np.tile(np.array(attitude_li, dtype='float'), (data_shape[0], 1))
route_array_dis = np.tile(np.array(route_point_dis, dtype='float'), (data_shape[0], 1))
return (left_pd, route_array_dis, data_shape, route_sign, attitude_array)

def remoteFun(self,xs):
return xs

def singleRoute(self,v,route_tag,route_dis,left_pd):
for (index, loc) in enumerate(v):
if index == 0:
continue
route_tag.append(self.df.iloc[v[index - 1], v[index]])
route_dis.append(v[index] - v[index - 1])
data = self.merge_data(route_tag, v, left_pd, route_dis)
return data

def ranking(self):
# max_point = 7
# with open("array.txt","r") as f:
# route = eval(f.read())
# route = route[:20]
route = [[1, 10, 19, 39, 59, 67],
[1, 11, 19, 39, 59, 67],
[1, 12, 19, 39, 59, 67],
[1, 13, 19, 39, 59, 67],
[1, 14, 19, 39, 59, 67],
[1, 10, 20, 39, 59, 67],
[1, 11, 20, 39, 59, 67],
[1, 12, 20, 39, 59, 67],
[1, 13, 20, 39, 59, 67],
[1, 14, 20, 39, 59, 67],
[1, 10, 21, 39, 59, 67],
[1, 11, 21, 39, 59, 67],
[1, 12, 21, 39, 59, 67],
[1, 13, 21, 39, 59, 67],
[1, 14, 21, 39, 59, 67],
[1, 11, 22, 39, 59, 67],
[1, 12, 22, 39, 59, 67],
[1, 13, 22, 39, 59, 67],
[1, 14, 22, 39, 59, 67],
[1, 11, 23, 39, 59, 67]]
print("route length:",len(route))
merge_data_li = []
self.arr_index = 0
for k, v in enumerate(route):
route_tag = []
route_dis = []
left_pd = pd.DataFrame()
data = mr.spawn(self.singleRoute, args=(v, route_tag, route_dis, left_pd))
if not data:
continue
merge_data_li.append(data)
return mr.ExecutableTuple(merge_data_li).execute().fetch()

def main(self):
print(self.ranking())

def run_tower_ranking(SafeFileDir,middlePath):
for file in os.listdir(SafeFileDir):
if file.endswith('.csv'):
start_time = time.time()
top = Towerrank(file, SafeFileDir, middlePath).main()
end_time = time.time()
cost_time = end_time - start_time
print('??:{}'.format(file, cost_time))

if __name__ == '__main__':
start = time.time()
SAFE_DIR = "./"
middlePath = "./"
run_tower_ranking(SAFE_DIR, middlePath)
end = time.time()
print('??', end - start)

```

Contributor guide

Open the contributing guide

Research direction

Start with the reproducible Python script, especially ranking(), mr.spawn(), and mr.ExecutableTuple(merge_data_li).execute().fetch(). Run it with the supplied route data and compare the execution timing with and without the remote calls. Done means identifying the cause of the missing speedup and documenting a reproducible explanation or required change.

Written by the indexing model from the issue text.

Assessment

Tech stack
numpy, pandas, python
Domain
data, distributed-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
15/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.