Ubuntu 18.04 upstream and canonical packages broken?

I booted up a VM and attempted to follow the instructions to install bcc and the sample tools on Ubuntu 18.04. Building from source against master, everything works as expected. Installing either the canonical bpfcc packages or the iovisor bcc apt packages results in BPF compilation errors when trying to run most tools.

Any tips on where I should look to debug this?

An example from fileslower:
```bash
/usr/share/bcc/tools/fileslower
In file included from /virtual/main.c:2:
In file included from include/uapi/linux/ptrace.h:142:
In file included from ./arch/x86/include/asm/ptrace.h:5:
./arch/x86/include/asm/segment.h:266:2: error: expected '(' after 'asm'
alternative_io ("lsl %[seg],%[p]",
^
./arch/x86/include/asm/alternative.h:240:2: note: expanded from macro 'alternative_io'
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
In file included from include/linux/thread_info.h:38:
In file included from ./arch/x86/include/asm/thread_info.h:12:
In file included from ./arch/x86/include/asm/page.h:12:
./arch/x86/include/asm/page_64.h:49:2: error: expected '(' after 'asm'
alternative_call_2(clear_page_orig,
^
./arch/x86/include/asm/alternative.h:256:2: note: expanded from macro 'alternative_call_2'
asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
In file included from include/linux/thread_info.h:38:
In file included from ./arch/x86/include/asm/thread_info.h:53:
In file included from ./arch/x86/include/asm/cpufeature.h:5:
In file included from ./arch/x86/include/asm/processor.h:24:
./arch/x86/include/asm/special_insns.h:207:2: error: expected '(' after 'asm'
alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
^
./arch/x86/include/asm/alternative.h:240:2: note: expanded from macro 'alternative_io'
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
In file included from include/linux/thread_info.h:38:
In file included from ./arch/x86/include/asm/thread_info.h:53:
In file included from ./arch/x86/include/asm/cpufeature.h:5:
./arch/x86/include/asm/processor.h:795:2: error: expected '(' after 'asm'
alternative_input(BASE_PREFETCH, "prefetchnta %P1",
^
./arch/x86/include/asm/alternative.h:221:2: note: expanded from macro 'alternative_input'
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
In file included from include/linux/thread_info.h:38:
In file included from ./arch/x86/include/asm/thread_info.h:53:
In file included from ./arch/x86/include/asm/cpufeature.h:5:
./arch/x86/include/asm/processor.h:807:2: error: expected '(' after 'asm'
alternative_input(BASE_PREFETCH, "prefetchw %P1",
^
./arch/x86/include/asm/alternative.h:221:2: note: expanded from macro 'alternative_input'
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
include/linux/thread_info.h:134:2: error: expected '(' after 'asm'
WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
^
include/asm-generic/bug.h:124:3: note: expanded from macro 'WARN'
__WARN_printf(TAINT_WARN, format); \
^
include/asm-generic/bug.h:93:3: note: expanded from macro '__WARN_printf'
__WARN_FLAGS(BUGFLAG_NO_CUT_HERE | BUGFLAG_TAINT(taint));\
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:6:
In file included from include/linux/wait_bit.h:8:
In file included from include/linux/wait.h:9:
In file included from include/linux/spinlock.h:51:
In file included from include/linux/preempt.h:78:
In file included from ./arch/x86/include/asm/preempt.h:7:
include/linux/thread_info.h:150:6: error: expected '(' after 'asm'
if (WARN_ON_ONCE(bytes > INT_MAX))
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:8:
In file included from include/linux/dcache.h:7:
In file included from include/linux/rculist.h:11:
include/linux/rcupdate.h:893:2: error: expected '(' after 'asm'
WARN_ON_ONCE(func != (rcu_callback_t)~0L);
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
In file included from include/linux/xarray.h:14:
In file included from include/linux/gfp.h:6:
In file included from include/linux/mmzone.h:21:
In file included from include/linux/mm_types.h:14:
In file included from include/linux/uprobes.h:49:
In file included from ./arch/x86/include/asm/uprobes.h:13:
In file included from include/linux/notifier.h:16:
In file included from include/linux/srcu.h:21:
In file included from include/linux/workqueue.h:9:
In file included from include/linux/timer.h:6:
include/linux/ktime.h:171:2: error: expected '(' after 'asm'
WARN_ON(div < 0);
^
include/asm-generic/bug.h:115:3: note: expanded from macro 'WARN_ON'
__WARN(); \
^
include/asm-generic/bug.h:90:19: note: expanded from macro '__WARN'
#define __WARN() __WARN_FLAGS(BUGFLAG_TAINT(TAINT_WARN))
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
In file included from include/linux/xarray.h:14:
In file included from include/linux/gfp.h:6:
In file included from include/linux/mmzone.h:21:
In file included from include/linux/mm_types.h:14:
In file included from include/linux/uprobes.h:49:
In file included from ./arch/x86/include/asm/uprobes.h:13:
In file included from include/linux/notifier.h:16:
include/linux/srcu.h:179:2: error: expected '(' after 'asm'
WARN_ON_ONCE(idx & ~0x1);
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
In file included from include/linux/xarray.h:14:
In file included from include/linux/gfp.h:6:
In file included from include/linux/mmzone.h:22:
include/linux/page-flags.h:565:2: error: expected '(' after 'asm'
BUG_ON(!PageHead(page));
^
include/asm-generic/bug.h:62:57: note: expanded from macro 'BUG_ON'
#define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0)
^
./arch/x86/include/asm/bug.h:73:2: note: expanded from macro 'BUG'
_BUG_FLAGS(ASM_UD2, 0); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
In file included from include/linux/xarray.h:14:
In file included from include/linux/gfp.h:6:
In file included from include/linux/mmzone.h:978:
In file included from ./arch/x86/include/asm/mmzone.h:5:
In file included from ./arch/x86/include/asm/mmzone_64.h:11:
In file included from ./arch/x86/include/asm/smp.h:13:
In file included from ./arch/x86/include/asm/apic.h:11:
In file included from ./arch/x86/include/asm/fixmap.h:190:
include/asm-generic/fixmap.h:38:2: error: expected '(' after 'asm'
BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
^
include/asm-generic/bug.h:62:57: note: expanded from macro 'BUG_ON'
#define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0)
^
./arch/x86/include/asm/bug.h:73:2: note: expanded from macro 'BUG'
_BUG_FLAGS(ASM_UD2, 0); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
In file included from include/linux/xarray.h:14:
In file included from include/linux/gfp.h:6:
In file included from include/linux/mmzone.h:978:
In file included from ./arch/x86/include/asm/mmzone.h:5:
In file included from ./arch/x86/include/asm/mmzone_64.h:11:
In file included from ./arch/x86/include/asm/smp.h:13:
./arch/x86/include/asm/apic.h:107:2: error: expected '(' after 'asm'
alternative_io("movl %0, %P1", "xchgl %0, %P1", X86_BUG_11AP,
^
./arch/x86/include/asm/alternative.h:240:2: note: expanded from macro 'alternative_io'
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:15:
In file included from include/linux/radix-tree.h:18:
include/linux/xarray.h:54:2: error: expected '(' after 'asm'
WARN_ON((long)v < 0);
^
include/asm-generic/bug.h:115:3: note: expanded from macro 'WARN_ON'
__WARN(); \
^
include/asm-generic/bug.h:90:19: note: expanded from macro '__WARN'
#define __WARN() __WARN_FLAGS(BUGFLAG_TAINT(TAINT_WARN))
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:39:
In file included from include/linux/ioprio.h:7:
include/linux/iocontext.h:132:2: error: expected '(' after 'asm'
WARN_ON_ONCE(atomic_long_read(&ioc->refcount) <= 0);
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:39:
In file included from include/linux/ioprio.h:7:
include/linux/iocontext.h:133:2: error: expected '(' after 'asm'
WARN_ON_ONCE(atomic_read(&ioc->active_ref) <= 0);
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:39:
In file included from include/linux/ioprio.h:7:
include/linux/iocontext.h:142:2: error: expected '(' after 'asm'
WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0);
^
include/asm-generic/bug.h:98:3: note: expanded from macro 'WARN_ON_ONCE'
__WARN_FLAGS(BUGFLAG_ONCE | \
^
./arch/x86/include/asm/bug.h:79:2: note: expanded from macro '__WARN_FLAGS'
_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:245:
include/linux/quota.h:114:3: error: expected '(' after 'asm'
BUG();
^
./arch/x86/include/asm/bug.h:73:2: note: expanded from macro 'BUG'
_BUG_FLAGS(ASM_UD2, 0); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
In file included from /virtual/main.c:3:
In file included from include/linux/fs.h:245:
include/linux/quota.h:141:3: error: expected '(' after 'asm'
BUG();
^
./arch/x86/include/asm/bug.h:73:2: note: expanded from macro 'BUG'
_BUG_FLAGS(ASM_UD2, 0); \
^
./arch/x86/include/asm/bug.h:35:2: note: expanded from macro '_BUG_FLAGS'
asm_inline volatile("1:\t" ins "\n" \
^
include/linux/compiler_types.h:210:24: note: expanded from macro 'asm_inline'
#define asm_inline asm __inline
^
fatal error: too many errors emitted, stopping now [-ferror-limit=]
20 errors generated.
Traceback (most recent call last):
File "/usr/share/bcc/tools/fileslower", line 196, in
b = BPF(text=bpf_text)
File "/usr/lib/python2.7/dist-packages/bcc/__init__.py", line 325, in __init__
raise Exception("Failed to compile BPF text")
Exception: Failed to compile BPF text
```

This seems to me like some sort of feature mismatch in the compiler toolchain? I'm not savvy enough to interpret this error. I noticed both canonical and iovisor publish fairly old versions of bcc, which might be part of the problem. Any help or advice debugging this further would be greatly appreciated!

Reproduce the failure with /usr/share/bcc/tools/fileslower using the canonical and iovisor bcc apt packages on Ubuntu 18.04. Read the reported kernel-header paths, including include/linux/compiler_types.h and arch/x86/include/asm/alternative.h, alongside the package versions. Done means the packaged tools run without the shown BPF compilation errors.

Written by the indexing model from the issue text.