Stacked bars are rendered with wrong colors when using custom plugin

### Expected behavior

I added custom plugin for sorting sections inside stacked bar depends on value. In two words - it fills `sortedData` at `beforeDraw`, then at `beforeDatasetDraw` it changes Y and BASE of section bar due to `sortedData` (just swaps sections) . Additionally at `beforeDatasetDraw` I try to paint sections to grey color, if there are more than 10 sections, so 10, 11, etc would be grey.

### Current behavior

For now depends of datasets (for some datasets it works properly, for some don't) some bar's sections filled already with grey color despite that there are less than 10 sections in a bar. It started to colorise with right color on hover.

### Reproducible sample

https://codepen.io/mi1ord/pen/qBKdRvp

### Optional extra steps/info to reproduce

custom plugin

```javascript
const stackedBarSorterPlugin = {
id: 'stackedBarSorter',

sortedData: {},

/*
* Sorts data by value and calculates values for bar stacks
*/
beforeDraw(chart: Chart) {
this.sortedData = {};
// iterate over datasets
chart.data.datasets.forEach((dataset, datasetIndex) => {
// iterate over dataset records
dataset.data.forEach((_, index) => {
// create data container for bar stack data
if (!this.sortedData[index]) {
this.sortedData[index] = {
data: [],
};
}

const datasetMeta = chart.getDatasetMeta(datasetIndex);

// save data
this.sortedData[index].data[datasetIndex] = {
color: dataset.backgroundColor,
datasetIndex,
hidden: datasetMeta.hidden,
value: dataset.data[index],
y: datasetMeta.data[index].y,
base: datasetMeta.data[index].base,
};

this.sortedData[index].data = this.sortedData[index].data.filter(({ value }) => value > 0);
});
});

const chartTop = chart.scales.y.top;
const { max } = chart.scales.y;
const h = chart.scales.y.height / max;

// iterate over datasets
chart.data.datasets.forEach((dataset) => {
// iterate over dataset records
dataset.data.forEach((_, index) => {
// sort data in bar stack by value
this.sortedData[index].data = Object.keys(this.sortedData[index].data)
.map((k) => this.sortedData[index].data[k])
.sort((a, b) => b.value - a.value);

this.sortedData[index].data = this.sortedData[index].data.map((dataItem, idx) => {
// calculate base value
let base =
chartTop +
(max -
Object.keys(this.sortedData[index].data)
.map((k) => this.sortedData[index].data[k].value)
.reduce((a, b) => a + b, 0)) *
h +
Object.keys(this.sortedData[index].data)
.map((k) => this.sortedData[index].data[k])
.filter((d) => d.hidden)
.reduce((a, b) => a + b.value, 0) *
h;

// increase base value with values of previous records
for (let i = 0; i < idx; i += 1) {
const { hidden, value } = this.sortedData[index].data[i];
base += hidden ? 0 : h * value;
}

return {
...dataItem,
base,
y: base + h * dataItem.value,
};
});
});
});
},

/*
* Sets values for base and y
*/
beforeDatasetDraw(chart, args) {
chart.getDatasetMeta(args.index).data.forEach((barElement, index) => {
const data = this.sortedData[index].data.filter(({ value }) => value > 0);

const storedElementIndex = data.findIndex(
(el) => el.datasetIndex === args.index,
);

if (storedElementIndex === -1) {
return;
}

const storedElement = data[storedElementIndex];

// !!!!!!!!!!!!!!!!!
// for somehow barElement.options.backgroundColor in some sections already has grey color
// !!!!!!!!!!!!!!!!!

if (storedElementIndex > 9) {
barElement.options.backgroundColor = 'grey';
}
if (storedElementIndex > 10) {
barElement.options.borderWidth = 0;
}
// swaps sections
barElement.y = storedElement.y;
barElement.base = storedElement.base;
});
},
};

```

### Possible solution

_No response_

### Context

_No response_

### chart.js version

v3.9.1

### Browser name and version

tried in chrome, safari, firefox

### Link to your project

_No response_