[Qemu-devel] Re: kqemu does not build on hybrid x86_64/i386 system

[Jan Kiszka]

Jan Kiszka wrote:
> Chris Dukes wrote:
>> Hi,
>> I'm attempting to build kqemu 1.4.0~pre1 on a debian sid i386 system running
>> debian's 2.6.26-1-amd64 kernel.
>>
>> It fails attemping to assemble nexus_asm.S with a combination of
>> x86_64/nexus_asm.S:63: Error: bad register name `%rax'
>> type errors
>> and
>> x86_64/nexus_asm.S:84: Error: `movzwq' is only supported in 64-bit mode
>>
>> I believe this is happening because the kqemu script makes
>> extensive use of uname -m to determine the architecture for
>> build and incorrectly assumes that just because it's x86_64ish
>> that gcc will emit 64bit code.
>>
>> After much kludging (And it's incorrect for anything except Linux),
>> I was able to have common/Makefile apply -m32 or -m64 as I thought was
>> appropriate.  Now the .S files assemble fine, but link fails with
>> x86_64/monitor_asm.o: In function `__monitor_exception':
>> (.text+0x68d): undefined reference to `KQEMU_STATE_monitor_ss_null_sel'
>> x86_64/monitor_asm.o: In function `exception_return':
>> (.text+0x6d3): undefined reference to `KQEMU_STATE_monitor_ss_null_sel'
>> x86_64/monitor_asm.o: In function `exception_return16':
>> (.text+0x776): undefined reference to `KQEMU_STATE_monitor_cs32_sel'
>> x86_64/monitor_asm.o: In function `__monitor_interrupt':
>> (.text+0x81c): undefined reference to `KQEMU_STATE_monitor_ss_null_sel'
>>
>> It's somewhat confusing as it builds fine for an x86_64 kernel with
>> x86_64 userspace and 686 kernel with i386 userspace.
> 
> I recently hacked the kqemu's build system for such cross-compilation
> scenarios. Find my patches attached. The magic configure command I
> invoked was this:
> 
> ./configure --kernel-path=... --cpu=i386 --cc="gcc -m32" \
>       --extra-ldflags=-melf_i386
> 
> Please note that those patches are no real solutions, maybe even cause
> problems to other OSes. The build system is desperately looking for a
> rework, specifically for Linux platforms. I think it should be switched
> completely to kbuild (with included tools, to build the monitor image),
> and then checked if the other target OSes can be cleanly folded into
> that or better kept separately.

And today I actually ran an i386 kqemu.ko that was cross-built on a
x86_64 host. It took an additional patch and an extension of the second
one to get things working, see attachments. The extended configure line is

./configure --kernel-path=... --cpu=i386 --cc="gcc -m32" \
        --extra-ldflags=-melf_i386 --host-cc="gcc -m32"

But kqemu's way of generating assembler offsets is not cross-build
compatible, it only happens to work for x86-64 -> i386, not for i386 ->
x86_64 or non-x86 -> x86. Another reason to go for kbuild (though I
haven't checked yet if its offset generation works for out of tree
modules as well).

Jan

-- 
Siemens AG, Corporate Technology, CT SE 2 ES-OS
Corporate Competence Center Embedded Linux

>From f29be375bc123394a703d21b27591d7f168946ff Mon Sep 17 00:00:00 2001
From: Jan Kiszka <address@hidden>
Date: Tue, 28 Oct 2008 17:14:14 +0100
Subject: [PATCH] Improve cross-compilation

Improve cross-compilation support of the build system.

Signed-off-by: Jan Kiszka <address@hidden>
---
 Makefile        |    2 +-
 common/Makefile |    9 +++++----
 configure       |    1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 5a2607a..b09636c 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,7 @@ ifdef CONFIG_KBUILD26
 
 all:
        make -C common monitor-image.h
-       make -C $(KERNEL_PATH) M=`pwd` modules
+       make -C $(KERNEL_PATH) ARCH=$(ARCH) M=`pwd` modules
 
 else
 # 2.4 build
diff --git a/common/Makefile b/common/Makefile
index 9d109ab..36d83a6 100644
--- a/common/Makefile
+++ b/common/Makefile
@@ -27,8 +27,8 @@
 cc-option = $(shell if test -z "`$(1) $(2) -S -o /dev/null -xc \
               /dev/null 2>&1`"; then echo "$(2)"; else echo "$(3)"; fi  ;) 
 
-HOST_CC=gcc
-MON_CC=gcc
+HOST_CC:=$(CC)
+MON_CC:=$(CC)
 MON_LD=ld
 ifdef CONFIG_WIN32
 TARGET=../kqemu-mod-$(ARCH)-win32.o
@@ -50,6 +50,7 @@ endif
 CFLAGS=$(COMMON_CFLAGS)
 MON_CFLAGS=$(COMMON_CFLAGS)
 KERNEL_CFLAGS=$(COMMON_CFLAGS)
+MON_LDFLAGS=$(LDFLAGS)
 
 # Disable SSP if GCC supports it
 MON_CFLAGS+=$(call cc-option,$(MON_CC),-fno-stack-protector,)
@@ -75,7 +76,7 @@ kqemu_global_init kqemu_global_delete
 all: $(TARGET)
 
 $(TARGET): kernel.o $(ARCH)/kernel_asm.o
-       $(LD) -r -o $@ kernel.o $(ARCH)/kernel_asm.o
+       $(LD) $(LDFLAGS) -r -o $@ kernel.o $(ARCH)/kernel_asm.o
 
 monitor-image.h: monitor-image.bin genmon
        ./genmon < $< > $@
@@ -84,7 +85,7 @@ monitor-image.bin: monitor-image.out
        objcopy -O binary $< $@
 
 monitor-image.out: $(ARCH)/nexus_asm.o $(ARCH)/monitor_asm.o monitor.o 
monitor-utils.o interp.o
-       $(MON_LD) -T $(ARCH)/monitor.ld -o $@ $^
+       $(MON_LD) $(MON_LDFLAGS) -T $(ARCH)/monitor.ld -o $@ $^
 
 $(ARCH)/nexus_asm.o: $(ARCH)/nexus_asm.S monitor_def.h
 
diff --git a/configure b/configure
index a4a0585..106d301 100755
--- a/configure
+++ b/configure
@@ -248,6 +248,7 @@ echo "  --cross-prefix=PREFIX    use PREFIX for compile 
tools [$cross_prefix]"
 echo "  --cc=CC                  use C compiler CC [$cc]"
 echo "  --host-cc=CC             use C compiler CC [$host_cc]"
 echo "  --make=MAKE              use specified make [$make]"
+echo "  --cpu=CPU                specify target CPU [host CPU]"
 echo ""
 exit 1
 fi
-- 
1.5.6

>From 67f80f5e1bfb8fce2fb6768a8caf97d434536a1d Mon Sep 17 00:00:00 2001
From: Jan Kiszka <address@hidden>
Date: Wed, 5 Nov 2008 01:03:26 +0100
Subject: [PATCH] Switch kernel2monitor and monitor2kernel to regparm

Switch kernel2monitor and monitor2kernel to regparm(1) call convention -
the default under Linux anyway. This fixes i386 Linux hosts after moving
the build system to kbuild.

Signed-off-by: Jan Kiszka <address@hidden>
---
 common/i386/nexus_asm.S |   12 +++++++-----
 common/kernel.c         |    2 +-
 common/kqemu_int.h      |    3 +--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/common/i386/nexus_asm.S b/common/i386/nexus_asm.S
index 0d79f46..bac55fe 100644
--- a/common/i386/nexus_asm.S
+++ b/common/i386/nexus_asm.S
@@ -33,7 +33,8 @@ _start:
         .long ASM_NAME(monitor2kernel_jmp_offset) - _start
         .long ASM_NAME(monitor_exec) - _start
 /*
- * Assumptions when entering kernel2monitor:      
+ * Assumptions when entering kernel2monitor:
+ * %rax = kqemu_state
  * %ss, %ds, %es, %cs are 4G flat 32 bit segments
  * IRQ disabled        
  */
@@ -47,8 +48,8 @@ ASM_NAME(kernel2monitor):
         pushl %ds
         pushl %es
 
-        movl 8(%ebp), %ebx
-        
+        movl %eax, %ebx
+
         str  KQEMU_STATE_kernel_tr_sel(%ebx)
         sidt KQEMU_STATE_kernel_idt(%ebx)
         sgdt KQEMU_STATE_kernel_gdt(%ebx)
@@ -125,6 +126,7 @@ ASM_NAME(kernel2monitor_jmp_offset):
 
 /*
  * Assumptions when entering monitor2kernel:
+ * %rax = kqemu_state
  * IRQ disabled
  */
 ASM_NAME(monitor2kernel):
@@ -134,8 +136,8 @@ ASM_NAME(monitor2kernel):
         pushl %esi
         pushl %edi
 
-        movl 8(%ebp), %ebx
-        
+        movl %eax, %ebx
+
         movl %esp, KQEMU_STATE_monitor_esp(%ebx)
 
         /* clear the debug register if needed */
diff --git a/common/kernel.c b/common/kernel.c
index 3c11422..8ed04c6 100644
--- a/common/kernel.c
+++ b/common/kernel.c
@@ -1066,7 +1066,7 @@ static void apic_restore_nmi(struct kqemu_state *s, int 
lvt_mask)
 long CDECL kqemu_exec(struct kqemu_state *s)
 {
     const struct monitor_code_header *m = (void *)monitor_code;
-    void (*kernel2monitor)(struct kqemu_state *s) = 
+    void __attribute__((regparm(1))) (*kernel2monitor)(struct kqemu_state *s) =
         (void *)(m->kernel2monitor + s->nexus_kaddr);
     unsigned long *ptr;
     int ret, apic_nmi_mask, cpl;
diff --git a/common/kqemu_int.h b/common/kqemu_int.h
index f19f7ca..6f0b67d 100644
--- a/common/kqemu_int.h
+++ b/common/kqemu_int.h
@@ -824,8 +824,7 @@ static inline void wrmsrl(unsigned long msr, uint64_t val)
 
 /* nexus_asm.S */
 
-void kernel2monitor(struct kqemu_state *s);
-void monitor2kernel(struct kqemu_state *s);
+void __attribute__((regparm(1))) monitor2kernel(struct kqemu_state *s);
 
 /* kernel_asm.S */
 void __attribute__((regparm(1))) exec_irq(int intno);
-- 
1.5.6