Potential for significant perf improvements in large repos

First, thanks for your work on this project 🙂

The current implementation is fairly slow with large repos, for instance vscode, which has around 5000 files or typescript, which has around 50k. It takes about a minute to clone vscode with `--singleBranch` and `--depth 1`, and doesn't manage to clone typescript in the ~15 minutes I waited.

By adding batching to the indexdb writes (Put all writes into a single transaction rather than one transaction per file) and changing the autoinc in the cachefs to increment a counter rather than search for the highest inode (the search means writing N files is N^2 time), I am able to see vscode clone in ~20 seconds and typescript clone in about 2 minutes. this is approx 3x slower than native for vscode and 6x slower than native for typescript.

Batching:
```diff
diff --git a/idb-keyval.ts b/idb-keyval.ts
index 45a0d97..94920ef 100644
--- a/idb-keyval.ts
+++ b/idb-keyval.ts
@@ -2,10 +2,12 @@ export class Store {
private _dbp: Promise | undefined;
readonly _dbName: string;
readonly _storeName: string;
+ readonly id: string

constructor(dbName = 'keyval-store', readonly storeName = 'keyval') {
this._dbName = dbName;
this._storeName = storeName;
+ this.id = `dbName:${dbName};;storeName:${storeName}`
this._init();
}

@@ -44,6 +46,31 @@ export class Store {
}
}

+class Batcher {
+ private ongoing: Promise | undefined
+ private items: { item: T, onProcessed: () => void }[] = []
+
+ constructor(private executor: (items: T[]) => Promise) { }
+
+ private async process() {
+ const toProcess = this.items;
+ this.items = [];
+ await this.executor(toProcess.map(({ item }) => item))
+ toProcess.map(({ onProcessed }) => onProcessed())
+ if (this.items.length) {
+ this.ongoing = this.process()
+ } else {
+ this.ongoing = undefined
+ }
+ }
+
+ async queue(item: T): Promise {
+ const result = new Promise((resolve) => this.items.push({ item, onProcessed: resolve }))
+ if (!this.ongoing) this.ongoing = this.process()
+ return result
+ }
+}
+
let store: Store;

function getDefaultStore() {
@@ -58,10 +85,17 @@ export function get(key: IDBValidKey, store = getDefaultStore()): Promise<
}).then(() => req.result);
}

+const setBatchers: Record> = {}
export function set(key: IDBValidKey, value: any, store = getDefaultStore()): Promise {
- return store._withIDBStore('readwrite', store => {
- store.put(value, key);
- });
+ if (!setBatchers[store.id]) {
+ setBatchers[store.id] = new Batcher((items) =>
+ store._withIDBStore('readwrite', store => {
+ for (const item of items) {
+ store.put(item.value, item.key)
+ }
+ }))
+ }
+ return setBatchers[store.id].queue({ key, value })
}

export function update(key: IDBValidKey, updater: (val: any) => any, store = getDefaultStore()): Promise {

```

Counter:
```diff
diff --git a/src/CacheFS.js b/src/CacheFS.js
index ed26c57..0dc6950 100755
--- a/src/CacheFS.js
+++ b/src/CacheFS.js
@@ -5,6 +5,7 @@ const STAT = 0;

module.exports = class CacheFS {
constructor() {
+ this._maxInode = 0
}
_makeRoot(root = new Map()) {
root.set(STAT, { mode: 0o777, type: "dir", size: 0, ino: 0, mtimeMs: Date.now() });
@@ -38,16 +39,7 @@ module.exports = class CacheFS {
return count;
}
autoinc () {
- let val = this._maxInode(this._root.get("/")) + 1;
- return val;
- }
- _maxInode(map) {
- let max = map.get(STAT).ino;
- for (let [key, val] of map) {
- if (key === STAT) continue;
- max = Math.max(max, this._maxInode(val));
- }
- return max;
+ return ++this._maxInode;
}
print(root = this._root.get("/")) {
let str = "";
```

Please let me know if you'd consider incorporating these changes... the batching should be safe, I'm not super sure about the autoinc, but I don't see a reason why it would cause issues (the main difference is deleting a file would free up its inode value in the original implementation but doesn't here, but that shouldn't be a problem AFAIK)