adobe / adobe/hyde

Alter project so builds fail faster

Open
#82 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
C++
Stars
340
Forks
45
PR merge metrics
No merged PRs in 30d

Description

Right now the Clang dependencies are built before the `hyde` sources are. If there is an error in the `hyde` sources (especially building on Linux) the build takes more than two hours before those errors are discovered.

It would be great to build only the minimal number of Clang dependencies first to get the `hyde` sources built, then build the rest of Clang. That way if there are any failures in the `hyde` sources, they will be discovered sooner, saving us time.

Here are some initial attempts to alter `CMakeLists.txt` (the error message it produces will follow):

```cmake
# Copyright 2018 Adobe
# All Rights Reserved.

# NOTICE: Adobe permits you to use, modify, and distribute this file in
# accordance with the terms of the Adobe license agreement accompanying
# it. If you have received this file from a source other than Adobe,
# then your use, modification, or distribution of it requires the prior
# written permission of Adobe.

cmake_minimum_required(VERSION 3.23)

set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) # Avoid overriding normal variables with option()
set(CMAKE_POLICY_DEFAULT_CMP0126 NEW) # Avoid overriding normal variables with set(CACHE)

include(FetchContent)

set(FETCHCONTENT_QUIET FALSE)

set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64")

project(hyde)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_XCODE_GENERATE_SCHEME OFF)

message(STATUS "INFO: Setting up LLVM...")

FetchContent_Declare(
llvm
GIT_REPOSITORY https://github.com/llvm/llvm-project.git
GIT_TAG 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a # llvmorg-15.0.7
GIT_SHALLOW TRUE
GIT_PROGRESS TRUE
SOURCE_SUBDIR llvm
)

set(LLVM_ENABLE_PROJECTS "clang")
set(LLVM_TARGETS_TO_BUILD "X86;AArch64")
set(LLVM_ENABLE_ZSTD OFF)

FetchContent_MakeAvailable(llvm)

message(STATUS "INFO: LLVM source dir: ${llvm_SOURCE_DIR}")
message(STATUS "INFO: LLVM binary dir: ${llvm_BINARY_DIR}")

set(SRC_SOURCES
${PROJECT_SOURCE_DIR}/sources/autodetect.cpp
${PROJECT_SOURCE_DIR}/sources/main.cpp
${PROJECT_SOURCE_DIR}/sources/output_yaml.cpp
)

set(SRC_EMITTERS
${PROJECT_SOURCE_DIR}/emitters/yaml_base_emitter.cpp
${PROJECT_SOURCE_DIR}/emitters/yaml_class_emitter.cpp
${PROJECT_SOURCE_DIR}/emitters/yaml_enum_emitter.cpp
${PROJECT_SOURCE_DIR}/emitters/yaml_function_emitter.cpp
${PROJECT_SOURCE_DIR}/emitters/yaml_library_emitter.cpp
${PROJECT_SOURCE_DIR}/emitters/yaml_sourcefile_emitter.cpp
)

set(SRC_MATCHERS
${PROJECT_SOURCE_DIR}/matchers/class_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/enum_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/function_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/namespace_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/typealias_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/typedef_matcher.cpp
${PROJECT_SOURCE_DIR}/matchers/utilities.cpp
)

set(SRC_YAMLCPP
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/binary.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/convert.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/depthguard.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/directives.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/emit.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/emitfromevents.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/emitter.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/emitterstate.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/emitterutils.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/exceptions.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/exp.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/memory.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/node.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/node_data.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/nodebuilder.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/nodeevents.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/null.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/ostream_wrapper.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/parse.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/parser.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/regex_yaml.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/scanner.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/scanscalar.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/scantag.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/scantoken.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/simplekey.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/singledocparser.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/stream.cpp
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/src/tag.cpp
)

# We make the hyde sources a library so that we can define these sources as a dependency of the
# executable. This will cause cmake to order the compilation of these files ahead of any other
# dependencies like Clang (because we specify this library as the first dependency of the
# executable.) The big benefit to this change is that they'll get compiled (and possibly
# break) _first_, letting builds fail faster than if Clang we're compiled ahead of these sources.

add_library(hyde_lib STATIC)

target_sources(hyde_lib
PRIVATE
${SRC_SOURCES}
${SRC_EMITTERS}
${SRC_MATCHERS}
${SRC_YAMLCPP}
)

source_group(sources FILES ${SRC_SOURCES})
source_group(emitters FILES ${SRC_EMITTERS})
source_group(matchers FILES ${SRC_MATCHERS})
source_group(yaml-cpp FILES ${SRC_YAMLCPP})

target_include_directories(hyde_lib
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/include
${PROJECT_SOURCE_DIR}/submodules/yaml-cpp/include/
${PROJECT_SOURCE_DIR}/submodules/json/include/
${llvm_SOURCE_DIR}/clang/include
${llvm_BINARY_DIR}/tools/clang/include
${llvm_SOURCE_DIR}/llvm/include
${llvm_BINARY_DIR}/include
)

target_compile_options(hyde_lib
PUBLIC
-Wall
-Wno-comment
-Werror
-Wno-range-loop-analysis
)

if (NOT LLVM_ENABLE_RTTI)
target_compile_options(hyde_lib PRIVATE -fno-rtti)
endif()

# cmake won't let you make an executable with no source files, so we create a dummy empty file.
# See: https://stackoverflow.com/a/34234515/153535
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/null.cpp "// See CMakeLists.txt for the origin of this file.")
add_executable(hyde ${CMAKE_CURRENT_BINARY_DIR}/null.cpp)

target_link_libraries(hyde
hyde_lib
clang
clangAST
clangASTMatchers
clangBasic
clangFrontend
clangLex
clangTooling
)

if (PROJECT_IS_TOP_LEVEL)
set_target_properties(hyde PROPERTIES XCODE_GENERATE_SCHEME ON)
endif()
```

Here is the error message produced by the build system:

```
[1/3639] Building CXX object CMakeFiles/hyde_lib.dir/sources/main.cpp.o
FAILED: CMakeFiles/hyde_lib.dir/sources/main.cpp.o
/usr/bin/c++ -I/home/runner/work/hyde/hyde -I/home/runner/work/hyde/hyde/include -I/home/runner/work/hyde/hyde/submodules/yaml-cpp/include -I/home/runner/work/hyde/hyde/submodules/json/include -I/home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include -I/home/runner/work/hyde/hyde/build/_deps/llvm-build/tools/clang/include -I/home/runner/work/hyde/hyde/build/_deps/llvm-src/llvm/include -I/home/runner/work/hyde/hyde/build/_deps/llvm-build/include -O3 -DNDEBUG -std=c++20 -Wall -Wno-comment -Werror -Wno-range-loop-analysis -fno-rtti -MD -MT CMakeFiles/hyde_lib.dir/sources/main.cpp.o -MF CMakeFiles/hyde_lib.dir/sources/main.cpp.o.d -o CMakeFiles/hyde_lib.dir/sources/main.cpp.o -c /home/runner/work/hyde/hyde/sources/main.cpp
In file included from /home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include/clang/AST/ASTContext.h:18,
from /home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include/clang/Frontend/ASTUnit.h:17,
from /home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include/clang/Frontend/FrontendAction.h:23,
from /home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include/clang/Frontend/FrontendActions.h:12,
from /home/runner/work/hyde/hyde/sources/main.cpp:23:
/home/runner/work/hyde/hyde/build/_deps/llvm-src/clang/include/clang/AST/ASTFwd.h:21:10: fatal error: clang/AST/DeclNodes.inc: No such file or directory
21 | #include "clang/AST/DeclNodes.inc"
| ^~~~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.
[2/3[6](https://github.com/adobe/hyde/actions/runs/5613466345/job/15209547361#step:4:7)3[9](https://github.com/adobe/hyde/actions/runs/5613466345/job/15209547361#step:4:10)] Building CXX object CMakeFiles/hyde_lib.dir/sources/autodetect.cpp.o
[3/3639] Building CXX object CMakeFiles/hyde_lib.dir/sources/output_yaml.cpp.o
ninja: build stopped: subcommand failed.
Error: Process completed with exit code 1.
```

It would seem no reordering of dependencies in `target_link_libraries` causes the Clang dependencies to build before the `hyde_lib` dependencies. I suspect the `clangAST` and/or `clangASTMatchers` dependency is responsible for constructing `clang/AST/DeclNodes.inc` which the rest of the Clang headers depend on.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.