vcvttsd2usi Illegal instruction on fly.io but not locally or on github ci

Questions / Help
1

I’ve deployed an app with sqlite extension written in Zig and it’s getting

Program terminated with signal SIGILL, Illegal instruction.
#0  0x00007fbf1a79f66c in std.fmt.errol.errolFixed (val=1653920344.549083, buffer=...) at /deps/local/lib/std/fmt/errol.zig:362
362	/deps/local/lib/std/fmt/errol.zig: No such file or directory.

┌────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│  > 0x7fbf1a79f66c <std.fmt.errol.errolFixed+92>    vcvttsd2usi %xmm0,%rax                                                                          │
│    0x7fbf1a79f672 <std.fmt.errol.errolFixed+98>    mov    %rax,-0x80(%rbp)                                                                         │
│    0x7fbf1a79f676 <std.fmt.errol.errolFixed+102>   vcvtusi2sd %rax,%xmm1,%xmm1                                                                     │
│    0x7fbf1a79f67c <std.fmt.errol.errolFixed+108>   vsubsd %xmm1,%xmm0,%xmm0                                                                        │
│    0x7fbf1a79f680 <std.fmt.errol.errolFixed+112>   vmovsd -0x351a8(%rip),%xmm1        # 0x7fbf1a76a4e0                                             │
│    0x7fbf1a79f688 <std.fmt.errol.errolFixed+120>   vucomisd %xmm0,%xmm1                                                                            │
│    0x7fbf1a79f68c <std.fmt.errol.errolFixed+124>   seta   %al                                                                                      │
│    0x7fbf1a79f68f <std.fmt.errol.errolFixed+127>   vmovsd -0x351bf(%rip),%xmm1        # 0x7fbf1a76a4d8                                             │
│    0x7fbf1a79f697 <std.fmt.errol.errolFixed+135>   vucomisd %xmm1,%xmm0                                                                            │
│    0x7fbf1a79f69b <std.fmt.errol.errolFixed+139>   seta   %cl                                                                                      │
│    0x7fbf1a79f69e <std.fmt.errol.errolFixed+142>   and    %cl,%al                                                                                  │
│    0x7fbf1a79f6a0 <std.fmt.errol.errolFixed+144>   test   $0x1,%al                                                                                 │
│    0x7fbf1a79f6a2 <std.fmt.errol.errolFixed+146>   jne    0x7fbf1a79f838 <std.fmt.errol.errolFixed+552>                                            │
│    0x7fbf1a79f6a8 <std.fmt.errol.errolFixed+152>   jmp    0x7fbf1a79f8cc <std.fmt.errol.errolFixed+700>                                            │
│    0x7fbf1a79f6ad <std.fmt.errol.errolFixed+157>   jmp    0x7fbf1a79f6af <std.fmt.errol.errolFixed+159>                                            │
│    0x7fbf1a79f6af <std.fmt.errol.errolFixed+159>   vmovsd -0x18(%rbp),%xmm0                                                                        │
│    0x7fbf1a79f6b4 <std.fmt.errol.errolFixed+164>   vpxor  %xmm1,%xmm1,%xmm1                                                                        │
│    0x7fbf1a79f6b8 <std.fmt.errol.errolFixed+168>   vucomisd %xmm1,%xmm0                                                                            │
│    0x7fbf1a79f6bc <std.fmt.errol.errolFixed+172>   jne    0x7fbf1a79f6c2 <std.fmt.errol.errolFixed+178>                                            │
│    0x7fbf1a79f6be <std.fmt.errol.errolFixed+174>   jp     0x7fbf1a79f6c2 <std.fmt.errol.errolFixed+178>                                            │
│    0x7fbf1a79f6c0 <std.fmt.errol.errolFixed+176>   jmp    0x7fbf1a79f73b <std.fmt.errol.errolFixed+299>                                            │
│    0x7fbf1a79f6c2 <std.fmt.errol.errolFixed+178>   vmovsd -0x351da(%rip),%xmm0        # 0x7fbf1a76a4f0                                             │
│    0x7fbf1a79f6ca <std.fmt.errol.errolFixed+186>   vmulsd -0x20(%rbp),%xmm0,%xmm0                                                                  │
│    0x7fbf1a79f6cf <std.fmt.errol.errolFixed+191>   vmovsd %xmm0,-0x20(%rbp)                                                                        │
└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘

for @floatToInt(u64, val); at zig/errol.zig at 74ed7c1f0998e9dd89aa3f3480fff845afd6b422 · ziglang/zig · GitHub

Since I’m not getting this error neither locally when running my alpine container, nor on github ci, I wonder if this could be related to firecracker vms or amd cpus?

2

Seems like the AMD EPYC cpu the app is running on doesn’t support avx512:

> cat /proc/cpuinfo
processor	: 0
vendor_id	: AuthenticAMD
cpu family	: 23
model		: 49
model name	: AMD EPYC
stepping	: 0
microcode	: 0x1000065
cpu MHz		: 2495.308
physical id	: 0
siblings	: 1
core id		: 0
cpu cores	: 1
apicid		: 0
initial apicid	: 0
fpu		: yes
fpu_exception	: yes
cpuid level	: 16
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy svm cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd ibrs ibpb stibp vmmcall fsgsbase tsc_adjust bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr wbnoinvd arat npt nrip_save umip rdpid
bugs		: sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass
bogomips	: 4990.61
clflush size	: 64
cache_alignment	: 64
address sizes	: 48 bits physical, 48 bits virtual
power management:

The fix would probably be to compile the extension for the correct CPU on github ci. Or to use fly builders.

3

I’ve installed zig on a fly instance to check what cpu model it has

> apk add --no-cache --update curl xz
> export ZIGVER=0.10.0-dev.2669+74ed7c1f0
> curl https://ziglang.org/builds/zig-linux-$(uname -m)-$ZIGVER.tar.xz -O
> tar -xf zig-linux-$(uname -m)-$ZIGVER.tar.xz
> mv zig-linux-$(uname -m)-$ZIGVER/ local/
> local/zig version
0.10.0-dev.2669+74ed7c1f0

> local/zig build-lib --show-builtin
// ...snip
pub const cpu: std.Target.Cpu = .{
    .arch = .x86_64,
    .model = &std.Target.x86.cpu.znver2,
// ...etc

and that info was enough to pick the correct compile options: zig build-lib -mcpu znver2.

1 Like