microsoft / microsoft/TypeScript

⚡ Performance: Project service doesn't cache all fs.realpath

Open
#59,342 2 comments 1 reaction 1 assignee View on GitHub

@sheetalkamat is already working on this.

Since Jul 18, 2024.

Dominant language
Go
Stars
111k
Forks
14.4k
Avg merge
1d 19h
Merged PRs (30d)
117

Description

Acknowledgement
  • I acknowledge that issues using this template may be closed without further explanation at the maintainer's discretion.
Comment
Acknowledgement
  • I acknowledge that issues using this template may be closed without further explanation at the maintainer's discretion.
Comment
🔎 Search Terms

project service fs realpath realpathSync

🙁 Actual behavior

When using typescript-eslint's parserOptions.projectService, type checking APIs switch from the traditional manual TypeScript ts.Program approach to the editor-style ts.ProjectService. We're observing excess calls to the realpath function on some paths in node_modules/ - up to 3-4 for some paths (!).

🙂 Expected behavior

There should be no uncached realpath calls, I'd think?

As a draft, I added a basic caching Map to realpath and ran a before & after comparison with hyperfine (--runs 50). The results showed a ~0.5-2.5% improvement in lint time:

Variant Measurement User Time
Baseline 3.153 s ± 0.039 s 4.403 s
Caching 3.073 s ± 0.048 s 4.377 s
diff patch to switch to the Caching variant... 
diff --git a/node_modules/typescript/lib/typescript.js b/node_modules/typescript/lib/typescript.js
index 4baad59..e53476d 100644
--- a/node_modules/typescript/lib/typescript.js
+++ b/node_modules/typescript/lib/typescript.js
@@ -13,6 +13,8 @@ See the Apache Version 2.0 License for specific language governing permissions
 and limitations under the License.
 ***************************************************************************** */

+var realpathCache = new Map();
+
 var ts = {}; ((module) => {
 "use strict";
 var __defProp = Object.defineProperty;
@@ -8798,6 +8800,15 @@ var sys = (() => {
       return path.length < 260 ? _fs.realpathSync.native(path) : _fs.realpathSync(path);
     }
     function realpath(path) {
+      const cached = realpathCache.get(path);
+      if (cached) {
+        return cached;
+      }
+      const result = realpathWorker(path);
+      realpathCache.set(path, result);
+      return result;
+    }
+    function realpathWorker(path) {
       try {
         return fsRealpath(path);
       } catch {
Additional information about the issue

On the typescript-eslint side:

These are the top 10 most common paths called by realpath... 
4 /Users/josh/repos/performance/node_modules/@types/node/index.d.ts
4 /Users/josh/repos/performance/node_modules/prettier
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example0
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example1
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example10
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example10/nested1
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example10/nested10
3 /Users/josh/repos/performance/cases/files-1024-layout-even-singlerun-true-types-service/src/example10/nested10/nested1
Here's an example call stack from the most common one... 
Error
    at Object.realpath (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:8807:33)
    at host.compilerHost.realpath (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:127075:128)
    at realPath (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:44380:35)
    at getOriginalAndResolvedFileName (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:43301:28)
    at resolveTypeReferenceDirective (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:43389:74)
    at Object.resolve (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:123707:42)
    at resolveNamesWithLocalCache (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:129425:29)
    at Object.resolveTypeReferenceDirectiveReferences (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:129493:12)
    at ConfiguredProject2.resolveTypeReferenceDirectiveReferences (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:182460:33)
    at resolveTypeReferenceDirectiveNamesWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:124519:20)
    at resolveTypeReferenceDirectiveNamesReusingOldState (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:124672:14)
    at processTypeReferenceDirectives (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125992:156)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125881:9)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at processImportedModules (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126134:11)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125886:7)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at processImportedModules (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126134:11)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125886:7)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at processImportedModules (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126134:11)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125886:7)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at /Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125686:22
    at getSourceFileFromReferenceWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125657:26)
    at processSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125684:5)
    at /Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125978:7
    at forEach (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:2389:22)
    at processReferencedFiles (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125977:5)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125880:9)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at /Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125686:22
    at getSourceFileFromReferenceWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125657:26)
    at processSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125684:5)
    at processTypeReferenceDirectiveWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126015:7)
    at processTypeReferenceDirective (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126007:5)
    at processTypeReferenceDirectives (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126001:7)
    at findSourceFileWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125881:9)
    at findSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125737:20)
    at /Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125686:22
    at getSourceFileFromReferenceWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125657:26)
    at processSourceFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:125684:5)
    at processTypeReferenceDirectiveWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126015:7)
    at processTypeReferenceDirective (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:126007:5)
    at createProgram (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:124224:9)
    at synchronizeHostDataWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:148948:15)
    at synchronizeHostData (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:148844:7)
    at Object.getProgram (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:149020:5)
    at ConfiguredProject2.updateGraphWorker (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:183144:41)
    at ConfiguredProject2.updateGraph (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:183000:32)
    at ConfiguredProject2.updateGraph (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:184302:24)
    at updateWithTriggerFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:184785:11)
    at _ProjectService.reloadConfiguredProject (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:186401:5)
    at ConfiguredProject2.updateGraph (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:184298:29)
    at updateWithTriggerFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:184785:11)
    at updateConfiguredProject (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:184793:9)
    at _ProjectService.findCreateOrReloadConfiguredProject (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187262:44)
    at _ProjectService.tryFindDefaultConfiguredProjectForOpenScriptInfo (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187286:25)
    at _ProjectService.tryFindDefaultConfiguredProjectAndLoadAncestorsForOpenScriptInfo (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187332:25)
    at _ProjectService.assignProjectToOpenedScriptInfo (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187217:27)
    at _ProjectService.openClientFileWithNormalizedPath (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187411:48)
    at _ProjectService.openClientFile (/Users/josh/repos/performance/node_modules/typescript/lib/typescript.js:187120:17)
    at useProgramFromProjectService (/Users/josh/repos/performance/node_modules/@typescript-eslint/typescript-estree/dist/useProgramFromProjectService.js:61:28)
    at getProgramAndAST (/Users/josh/repos/performance/node_modules/@typescript-eslint/typescript-estree/dist/parser.js:44:100)
    at parseAndGenerateServices (/Users/josh/repos/performance/node_modules/@typescript-eslint/typescript-estree/dist/parser.js:155:11)
    at Object.parseForESLint (/Users/josh/repos/performance/node_modules/@typescript-eslint/parser/dist/parser.js:101:80)
    at Object.parse (/Users/josh/repos/performance/node_modules/eslint/lib/languages/js/index.js:186:26)
    at parse (/Users/josh/repos/performance/node_modules/eslint/lib/linter/linter.js:931:29)
    at Linter._verifyWithFlatConfigArrayAndWithoutProcessors (/Users/josh/repos/performance/node_modules/eslint/lib/linter/linter.js:1696:33)
    at Linter._verifyWithFlatConfigArray (/Users/josh/repos/performance/node_modules/eslint/lib/linter/linter.js:2062:21)
    at Linter.verify (/Users/josh/repos/performance/node_modules/eslint/lib/linter/linter.js:1528:61)
    at Linter.verifyAndFix (/Users/josh/repos/performance/node_modules/eslint/lib/linter/linter.js:2299:29)
    at verifyText (/Users/josh/repos/performance/node_modules/eslint/lib/eslint/eslint.js:498:48)
    at /Users/josh/repos/performance/node_modules/eslint/lib/eslint/eslint.js:939:40
    at async Promise.all (index 1)
    at async ESLint.lintFiles (/Users/josh/repos/performance/node_modules/eslint/lib/eslint/eslint.js:880:25)
    at async Object.execute (/Users/josh/repos/performance/node_modules/eslint/lib/cli.js:521:23)
    at async main (/Users/josh/repos/performance/node_modules/eslint/bin/eslint.js:153:22)

This looks very similar to #59338. My instinct is that it's the same root cause. Filing separately for visibility into the issue & just in case they're not actually the same.

Contributor guide

Open the contributing guide

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.