vllm-project / vllm-project/aibrix

RM should report instance type non exist error

Open
#2,249 4 comments 0 reactions 0 assignees View on GitHub
Dominant language
Go
Stars
5.1k
Forks
694
Avg merge
1d 19h
Merged PRs (30d)
98

Description

### 🐛 Describe the bug

Image

### Steps to Reproduce

my local test have not checked in yet.

```

// runLifecycle provisions one replica, polls List until it reaches Running (the
// reconcile/"sync" the planner relies on), then releases it. It is gated behind
// RM_MANUAL_LAUNCH=1 because it spends real money.
func runLifecycle(t *testing.T, rm *ResourceManager, provider types.ResourceProvisionType, gpu string) {
t.Helper()
os.Setenv("RM_MANUAL_LAUNCH", "1")
if os.Getenv("RM_MANUAL_LAUNCH") != "1" {
t.Skip("RM_MANUAL_LAUNCH != 1; skipping (this test launches a real, billed resource)")
}

ctx := context.Background()
gpus := envIntOr("RM_MANUAL_GPUS", 1)
req := &types.ResourceProvision{
IdempotencyKey: "manual-" + string(provider) + "-" + strconv.FormatInt(time.Now().Unix(), 10),
Spec: types.ResourceProvisionSpec{
Credential: types.ResourceCredential{Provider: provider},
Groups: &[]types.ResourceGroupSpec{{
GpusPerReplica: gpus,
Replicas: ptr.To(1),
AcceleratorPreference: &types.AcceleratorPreference{PreferredTypes: ptr.To([]string{gpu})},
}},
},
}

t.Logf("provisioning %s x%d on %s ...", gpu, gpus, provider)
// TODO: provision which one?
res, err := rm.Provisioner.Provision(ctx, req)
if err != nil {
t.Fatalf("Provision: %v", err)
}
t.Logf("provision accepted: id=%s status=%s region=%s", res.ProvisionID, res.Status, res.Region)

// Always release whatever we launched, even if the wait below fails.
defer func() {
t.Logf("releasing %s ...", res.ProvisionID)
if err := rm.Provisioner.Release(context.Background(), res.ProvisionID); err != nil {
t.Errorf("Release FAILED (manual cleanup may be required for %s): %v", res.ProvisionID, err)
return
}
final := waitForStatus(t, rm, res.ProvisionID, types.ProvisionStatusReleased, 5*time.Minute)
t.Logf("released: final status=%s", final.Status)
}()

final := waitForStatus(t, rm, res.ProvisionID, types.ProvisionStatusRunning, manualPollTimeout)
if final.Status != types.ProvisionStatusRunning {
t.Fatalf("provision did not reach Running (last status=%s, err=%s)", final.Status, final.ErrorMessage)
}
t.Logf("provision is RUNNING — sync works. endpoints:")
logEndpoints(t, final)
}

// waitForStatus polls List until the provision reaches want (or a terminal
// state) or the timeout elapses, logging each observed status so the reconcile
// can be watched live with `-v`.
func waitForStatus(t *testing.T, rm *ResourceManager, id string, want types.ProvisionStatus, timeout time.Duration) *types.ProvisionResult {
t.Helper()
ctx := context.Background()
filter := &types.ListOptions{ProvisionIDs: ptr.To([]string{id})}
deadline := time.Now().Add(timeout)
var last *types.ProvisionResult

for {
results, err := rm.Provisioner.List(ctx, filter)
switch {
case err != nil:
t.Logf(" list error: %v", err)
case len(results) == 0:
t.Logf(" provision %s not found yet", id)
default:
last = results[0]
t.Logf(" [%s] status=%s", time.Now().Format("15:04:05"), last.Status)
switch last.Status {
case want, types.ProvisionStatusFailed, types.ProvisionStatusReleaseFailed:
return last
}
}
if time.Now().After(deadline) {
t.Logf(" timed out after %s", timeout)
if last == nil {
return &types.ProvisionResult{Status: "unknown"}
}
return last
}
time.Sleep(manualPollInterval)
}
}

```

### Expected behavior

it show throw error earlier

### Environment

nightly

Contributor guide

Open the contributing guide

Research direction

Start by tracing the ResourceManager Provisioner.Provision entry point used by runLifecycle, then follow how the requested accelerator or instance type is validated before provisioning. The issue does not name a file or checked-in test; done means a nonexistent instance type is rejected earlier with an explicit error instead of proceeding into the lifecycle.

Written by the indexing model from the issue text.

Assessment

Tech stack
go
Domain
backend, cloud, infrastructure
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Quiet
Clarity
Needs clarification
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.