diff --git a/configure b/configure
index fd544e7b55..c5d5f02c08 100755
--- a/configure
+++ b/configure
@@ -4206,8 +4206,6 @@ if test "$smartcard" != "no"; then
     if $pkg_config libcacard; then
         libcacard_cflags=$($pkg_config --cflags libcacard)
         libcacard_libs=$($pkg_config --libs libcacard)
-        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
-        libs_softmmu="$libs_softmmu $libcacard_libs"
         smartcard="yes"
     else
         if test "$smartcard" = "yes"; then
@@ -5785,6 +5783,8 @@ fi
 
 if test "$smartcard" = "yes" ; then
   echo "CONFIG_SMARTCARD=y" >> $config_host_mak
+  echo "SMARTCARD_CFLAGS=$libcacard_cflags" >> $config_host_mak
+  echo "SMARTCARD_LIBS=$libcacard_libs" >> $config_host_mak
 fi
 
 if test "$libusb" = "yes" ; then
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 757e365562..9aceb6cc3d 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -26,8 +26,10 @@ common-obj-$(CONFIG_USB_BLUETOOTH)    += dev-bluetooth.o
 
 ifeq ($(CONFIG_USB_SMARTCARD),y)
 common-obj-y                          += dev-smartcard-reader.o
-common-obj-$(CONFIG_SMARTCARD)        += ccid-card-passthru.o
-common-obj-$(CONFIG_SMARTCARD)        += ccid-card-emulated.o
+common-obj-$(CONFIG_SMARTCARD)        += smartcard.mo
+smartcard.mo-objs := ccid-card-passthru.o ccid-card-emulated.o
+smartcard.mo-cflags := $(SMARTCARD_CFLAGS)
+smartcard.mo-libs := $(SMARTCARD_LIBS)
 endif
 
 ifeq ($(CONFIG_POSIX),y)