improve hydra-notify speed with a buildenv

stat("/nix/store/21hpiy9vb6zip7zacbz6k6880xl3qv4s-hydra-0.1.1234.1d613d/libexec/hydra/lib/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000018>
stat("/nix/store/21hpiy9vb6zip7zacbz6k6880xl3qv4s-hydra-0.1.1234.1d613d/libexec/hydra/lib/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000017>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000017>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000039>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000017>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/21pwj6wz5ppg2skll9w69hr6wpjjwfbq-hydra-perl-deps/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000014>
stat("/etc/profiles/per-user/hydra/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000013>
stat("/etc/profiles/per-user/hydra/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000013>
stat("/var/lib/hydra/.nix-profile/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000017>
stat("/var/lib/hydra/.nix-profile/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/var/nix/profiles/default/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000020>
stat("/nix/var/nix/profiles/default/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000019>
stat("/run/current-system/sw/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000020>
stat("/run/current-system/sw/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000018>
stat("/run/current-system/sw/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000022>
stat("/run/current-system/sw/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000020>
stat("/run/current-system/sw/lib/perl5/site_perl/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000018>
stat("/run/current-system/sw/lib/perl5/site_perl/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000017>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000016>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/site_perl/5.24.3/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/5.24.3/x86_64-linux-thread-multi/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/5.24.3/x86_64-linux-thread-multi/strict.pm", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000013>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/5.24.3/strict.pmc", 0x7fff57d7f900) = -1 ENOENT (No such file or directory) <0.000015>
stat("/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/5.24.3/strict.pm", {st_mode=S_IFREG|0444, st_size=4738, ...}) = 0 <0.000016>
openat(AT_FDCWD, "/nix/store/s029gmjpgvm776wfb56naqny9qja9339-perl-5.24.3/lib/perl5/5.24.3/strict.pm", O_RDONLY) = 4 <0.000018>

it took a ridiculous amount of syscalls just to find strict.pm

[hydra@iohk-infra:~]$ strace -c hydra-notify
Syntax: hydra-notify build-started BUILD | build-finished BUILD-ID [BUILD-IDs...] | step-finished BUILD-ID STEP-NR LOG-PATH
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 61.95    0.027491           4      6689      6014 stat
  8.23    0.003654           5       801       204 openat

and that occupies over 60% of the cpu time spent, and ~90% of the paths it checked didnt exist
i think doing a buildEnv over every path in the PERL5LIB and then only having a single entry in there would improve the startup time of the perl code
related: https://github.com/NixOS/hydra/issues/611

Start with the hydra-notify entry point and run the shown strace -c hydra-notify command to reproduce the PERL5LIB lookup overhead. Trace how PERL5LIB is assembled and evaluate a buildEnv containing the dependency paths; done means hydra-notify starts with fewer failed path lookups and lower startup syscall overhead.