Issue with union type

TypeScript Version: 3.7.5

Expected behavior:
Type of parameter qm in method Service.getItems should be union type assignable QueryModel<KeyToEntityMap<E>>

Actual behavior:

Argument of type 'QueryModel<KeyToEntityMap<E>>' is not assignable to parameter of type 'QueryModel<{ x: string; }> & QueryModel<{ y: string; }>'.
  Type 'QueryModel<KeyToEntityMap<E>>' is not assignable to type 'QueryModel<{ x: string; }>'.
    Type '{ x: string; }' is not assignable to type 'KeyToEntityMap<E>'.

Code

type Key = "a" | "b";
type KeyToEntityMap<E extends Key> =
    E extends "a" ? { x: string; } :
    E extends "b" ? { y: string; } :
    never;

class QueryModel<T>
{
    filters: [keyof T, string][] = [];
}

class Service<E extends Key>
{
    getItems(qm: QueryModel<KeyToEntityMap<E>>): KeyToEntityMap<E>[]
    {
        return [];
    }
}

type IServiceFactory = { [E in Key]: Service<E> };
class ServicesFactory implements IServiceFactory
{
    a: Service<"a"> = new Service<"a">();
    b: Service<"b"> = new Service<"b">();
}
const sf: IServiceFactory = new ServicesFactory();

class Same<E extends Key>
{
    fail(e: E)
    {
        let queryModel = new QueryModel<KeyToEntityMap<E>>();
        //Here getItems parameter gets wrong intersection type QueryModel<{ x: string; }> & QueryModel<{ y: string; }>
        let items: KeyToEntityMap<E> = sf[e].getItems(queryModel); 
    }
}

"use strict";
class QueryModel {
    constructor() {
        this.filters = [];
    }
}
class Service {
    getItems(qm) {
        return [];
    }
}
class ServicesFactory {
    constructor() {
        this.a = new Service();
        this.b = new Service();
    }
}
const sf = new ServicesFactory();
class Same {
    fail(e) {
        let items = sf[e].getItems(new QueryModel());
    }
}

{
  "compilerOptions": {
    "noImplicitAny": true,
    "strictNullChecks": true,
    "strictFunctionTypes": true,
    "strictPropertyInitialization": true,
    "strictBindCallApply": true,
    "noImplicitThis": true,
    "noImplicitReturns": true,
    "useDefineForClassFields": false,
    "alwaysStrict": true,
    "allowUnreachableCode": false,
    "allowUnusedLabels": false,
    "downlevelIteration": false,
    "noEmitHelpers": false,
    "noLib": false,
    "noStrictGenericChecks": false,
    "noUnusedLocals": false,
    "noUnusedParameters": false,
    "esModuleInterop": true,
    "preserveConstEnums": false,
    "removeComments": false,
    "skipLibCheck": false,
    "checkJs": false,
    "allowJs": false,
    "declaration": true,
    "experimentalDecorators": false,
    "emitDecoratorMetadata": false,
    "target": "ES2017",
    "module": "ESNext"
  }
}

Playground Link: Provided

Start with the provided Playground link and the minimal TypeScript reproduction under the stated compiler options. Trace how the indexed Service lookup and generic conditional type are checked, then add a regression case for the reported assignability error. Done means the example type-checks without the incorrect intersection-type diagnostic while preserving the shown return type.