mlc-ai / mlc-ai/tokenizers-cpp

Not building on IOS. Always get ld: library 'System' not found.

Open
#39 0 comments 0 reactions 0 assignees View on GitHub

Nobody has claimed this yet.

Dominant language
C++
Stars
512
Forks
132
PR merge metrics
No merged PRs in 30d

Description

This is my CmakeLists.txt

Example cmake project

cmake_minimum_required(VERSION 3.18)
project(dart_tokenizer C CXX)

include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-std=c++17" SUPPORT_CXX17)
set(CMAKE_CXX_FLAGS "-std=c++17 ${CMAKE_CXX_FLAGS}")
set(CMAKE_CUDA_STANDARD 17)

Add include directories

include tokenizer cpp as a sub directory

set(TOKENZIER_CPP_PATH tokenizers-cpp)
add_subdirectory(${TOKENZIER_CPP_PATH} tokenizers EXCLUDE_FROM_ALL)
if (IOS)
set(CMAKE_SYSTEM_NAME iOS)
set(CMAKE_OSX_ARCHITECTURES arm64)
set(CMAKE_XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH "NO")
set(CMAKE_XCODE_ATTRIBUTE_DEAD_CODE_STRIPPING "YES")
set(CMAKE_XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC "YES")
set(CMAKE_XCODE_ATTRIBUTE_CLANG_ENABLE_MODULES "YES")
set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "iPhone Developer")
set(CMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM "55FMS4PK2T")

if (CMAKE_OSX_SYSROOT MATCHES ".*iPhoneSimulator\\.platform.*")

if(CMAKE_OSX_ARCHITECTURES MATCHES "x86_64")
# Target the simulator explicitly
set(TOKENIZERS_CPP_CARGO_TARGET x86_64-apple-ios-sim)
# Make sure these are not set (or unset them if they are)
set(IPHONEOS_DEPLOYMENT_TARGET "")
set(SDKROOT "")
else ()
# This part seems to be correct already
set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-ios-sim)
endif ()
else ()

Target the device if not building for the simulator

set(TOKENIZERS_CPP_CARGO_TARGET aarch64-apple-ios)
endif ()

find_library(FOUNDATION_LIB Foundation)
find_library(SECURITY_LIB Security)
list(APPEND TOKENIZERS_C_LINK_LIBS ${FOUNDATION_LIB} ${SECURITY_LIB})

endif()
add_library(tokenizer SHARED main.cpp)

target_compile_definitions(tokenizer PUBLIC DART_SHARED_LIB)

target_include_directories(tokenizer PRIVATE ${TOKENZIER_CPP_PATH}/include json)

You can link tokenizers_cpp, it will automatically link tokenizers_c

and sentencepiece libary

target_link_libraries(tokenizer PRIVATE tokenizers_cpp)

This is my command:
cmake .. -GXcode \ -DIOS=ON \
-DCMAKE_TOOLCHAIN_FILE=/usr/local/lib/cmake/ios-cmake/ios.toolchain.cmake
-DPLATFORM=SIMULATORARM64
-DCMAKE_SYSTEM_NAME=iOS
-DCMAKE_OSX_ARCHITECTURES=arm64
-DCMAKE_OSX_SYSROOT="$(xcrun --sdk iphonesimulator --show-sdk-path)"

And i always get this error (If i build just the rust library, there's no problem):
PhaseScriptExecution Generate\ tokenizers/aarch64-apple-darwin/release/libtokenizers_c.a /Users/gn1ra/Documents/tokenizers/build-ios/build/dart_tokenizer.build/Debug-iphonesimulator/tokenizers_c.build/Script-A32C8CE6D36C5400AD409641.sh (in target 'tokenizers_c' from project 'dart_tokenizer')
cd /Users/gn1ra/Documents/tokenizers
/bin/sh -c /Users/gn1ra/Documents/tokenizers/build-ios/build/dart_tokenizer.build/Debug-iphonesimulator/tokenizers_c.build/Script-A32C8CE6D36C5400AD409641.sh
warning: unused manifest key: target.aarch64-apple-ios-sim.rustflags
Compiling proc-macro2 v1.0.86
Compiling crossbeam-utils v0.8.20
Compiling libc v0.2.155
Compiling serde v1.0.203
Compiling rayon-core v1.12.1
Compiling onig_sys v69.8.1
Compiling paste v1.0.15
Compiling aho-corasick v1.1.3
error: linking with cc failed: exit status: 1
|
= note: env -u IPHONEOS_DEPLOYMENT_TARGET -u SDKROOT -u TVOS_DEPLOYMENT_TARGET -u XROS_DEPLOYMENT_TARGET LC_ALL="C" PATH="/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/bin:/Applications/Xcode.app/Contents/SharedFrameworks/XCBuild.framework/Versions/A/PlugIns/XCBBuildService.bundle/Contents/PlugIns/XCBSpecifications.ideplugin/Contents/Resources:/Applications/Xcode.app/Contents/SharedFrameworks/XCBuild.framework/Versions/A/PlugIns/XCBBuildService.bundle/Contents/PlugIns/XCBSpecifications.ideplugin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/appleinternal/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/local/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/libexec:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/appleinternal/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/local/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/local/bin:/Applications/Xcode.app/Contents/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/usr/local/bin:/Users/gn1ra/.shorebird/bin:/Users/gn1ra/Users/gn1ra/Library/Android/sdk/ndk/27.0.11718014:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/homebrew/opt/ruby/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/System/Cryptexes/App/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/local/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/appleinternal/bin:/Library/Apple/usr/bin:/Users/gn1ra/.cargo/bin:/Users/gn1ra/flutter/bin:/Users/gn1ra/.pub-cache/bin" VSLANG="1033" ZERO_AR_DATE="1" "cc" "-arch" "arm64" "/var/folders/bl/qk_5q0752153x6nkkxg2cv340000gn/T/rustcO4DoxM/symbols.o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/rayon-core-9537acb8ad0b368f/build_script_build-9537acb8ad0b368f.build_script_build.7b19b8dec1e46eb2-cgu.0.rcgu.o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/rayon-core-9537acb8ad0b368f/build_script_build-9537acb8ad0b368f.4iaoyr4nl2pyk4kp.rcgu.o" "-L" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/deps" "-L" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libstd-7f95fd207255b41e.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libpanic_unwind-97393370543e3207.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libobject-a3d7c43b176960f9.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libmemchr-6371ee3a4bb27ae4.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libaddr2line-a7cf1a11761951bf.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libgimli-dad018a8d4ef8593.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_demangle-913776ff724458d1.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libstd_detect-364b04ba0ead75b3.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libhashbrown-918cdfbdd32cf1d1.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_std_workspace_alloc-424311e8ba1b9112.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libminiz_oxide-d4352744eff98311.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libadler-dd9df5b2170395b8.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libunwind-6b98b9caf95b8907.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcfg_if-d4913584d9013c52.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/liblibc-04f062133e43a7ac.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/liballoc-5397ff6f88f2713a.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_std_workspace_core-5142793ba4eb4c05.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcore-24a1e099f55e6f0d.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcompiler_builtins-9ca5cbf9e98b80b9.rlib" "-lSystem" "-lc" "-lm" "-L" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib" "-o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/rayon-core-9537acb8ad0b368f/build_script_build-9537acb8ad0b368f" "-Wl,-dead_strip" "-nodefaultlibs"
= note: ld: library 'System' not found
clang: error: linker command failed with exit code 1 (use -v to see invocation)IDEPackageSupportUseBuiltinSCM = YES

Build settings from command line:
VERBOSE = 1

ComputeTargetDependencyGraph
note: Building targets in dependency order
note: Target dependency graph (7 targets)
Target 'ALL_BUILD' in project 'dart_tokenizer'
➜ Explicit dependency on target 'ZERO_CHECK' in project 'dart_tokenizer'
➜ Explicit dependency on target 'tokenizer' in project 'dart_tokenizer'
Target 'tokenizer' in project 'dart_tokenizer'
➜ Explicit dependency on target 'ZERO_CHECK' in project 'dart_tokenizer'
➜ Explicit dependency on target 'tokenizers_c' in project 'dart_tokenizer'
➜ Explicit dependency on target 'tokenizers_cpp' in project 'dart_tokenizer'
➜ Explicit dependency on target 'sentencepiece-static' in project 'dart_tokenizer'
Target 'tokenizers_cpp' in project 'dart_tokenizer'
➜ Explicit dependency on target 'tokenizer_cpp_objs' in project 'dart_tokenizer'
➜ Explicit dependency on target 'tokenizers_c' in project 'dart_tokenizer'
➜ Explicit dependency on target 'sentencepiece-static' in project 'dart_tokenizer'
Target 'sentencepiece-static' in project 'dart_tokenizer' (no dependencies)
Target 'tokenizer_cpp_objs' in project 'dart_tokenizer' (no dependencies)
Target '

error: could not compile rayon-core (build script) due to 1 previous error
warning: build failed, waiting for other jobs to finish...
error: linking with cc failed: exit status: 1
|
= note: env -u IPHONEOS_DEPLOYMENT_TARGET -u SDKROOT -u TVOS_DEPLOYMENT_TARGET -u XROS_DEPLOYMENT_TARGET LC_ALL="C" PATH="/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/bin:/Applications/Xcode.app/Contents/SharedFrameworks/XCBuild.framework/Versions/A/PlugIns/XCBBuildService.bundle/Contents/PlugIns/XCBSpecifications.ideplugin/Contents/Resources:/Applications/Xcode.app/Contents/SharedFrameworks/XCBuild.framework/Versions/A/PlugIns/XCBBuildService.bundle/Contents/PlugIns/XCBSpecifications.ideplugin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/appleinternal/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/local/bin:/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/libexec:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/appleinternal/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/usr/local/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/usr/local/bin:/Applications/Xcode.app/Contents/Developer/usr/bin:/Applications/Xcode.app/Contents/Developer/usr/local/bin:/Users/gn1ra/.shorebird/bin:/Users/gn1ra/Users/gn1ra/Library/Android/sdk/ndk/27.0.11718014:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/homebrew/opt/ruby/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/System/Cryptexes/App/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/local/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/appleinternal/bin:/Library/Apple/usr/bin:/Users/gn1ra/.cargo/bin:/Users/gn1ra/flutter/bin:/Users/gn1ra/.pub-cache/bin" VSLANG="1033" ZERO_AR_DATE="1" "cc" "-arch" "arm64" "/var/folders/bl/qk_5q0752153x6nkkxg2cv340000gn/T/rustcMrw8zu/symbols.o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/paste-c4cff7755354c8ff/build_script_build-c4cff7755354c8ff.build_script_build.3a95369d1d2443b-cgu.0.rcgu.o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/paste-c4cff7755354c8ff/build_script_build-c4cff7755354c8ff.1b32pdr4kc6qo5mf.rcgu.o" "-L" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/deps" "-L" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libstd-7f95fd207255b41e.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libpanic_unwind-97393370543e3207.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libobject-a3d7c43b176960f9.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libmemchr-6371ee3a4bb27ae4.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libaddr2line-a7cf1a11761951bf.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libgimli-dad018a8d4ef8593.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_demangle-913776ff724458d1.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libstd_detect-364b04ba0ead75b3.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libhashbrown-918cdfbdd32cf1d1.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_std_workspace_alloc-424311e8ba1b9112.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libminiz_oxide-d4352744eff98311.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libadler-dd9df5b2170395b8.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libunwind-6b98b9caf95b8907.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcfg_if-d4913584d9013c52.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/liblibc-04f062133e43a7ac.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/liballoc-5397ff6f88f2713a.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/librustc_std_workspace_core-5142793ba4eb4c05.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcore-24a1e099f55e6f0d.rlib" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib/libcompiler_builtins-9ca5cbf9e98b80b9.rlib" "-lSystem" "-lc" "-lm" "-L" "/Users/gn1ra/.rustup/toolchains/stable-aarch64-apple-darwin/lib/rustlib/aarch64-apple-darwin/lib" "-o" "/Users/gn1ra/Documents/tokenizers/build-ios/tokenizers/release/build/paste-c4cff7755354c8ff/build_script_build-c4cff7755354c8ff" "-Wl,-dead_strip" "-nodefaultlibs"
= note: ld: library 'System' not found
clang: error: linker command failed with exit code 1 (use -v to see invocation)

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

Start with the supplied CMakeLists.txt and reproduce the Xcode simulator build using the listed cmake command. Inspect the generated tokenizers_c build step, its Rust target selection, and the linker invocation; done means the iOS simulator build completes without the missing System library error.

Written by the indexing model from the issue text.

Assessment

Tech stack
cmake, cpp, ios, rust
Domain
build-system, mobile-dev
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.