diff --git a/drivers/base/base.h b/drivers/base/base.h
index 2d270b8c731a069a15261724a73cc2df0086b1b9..dccc76f52ac5ed115ff7aaf9a78b51ce9c15d367 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -160,6 +160,7 @@ static inline int devtmpfs_init(void) { return 0; }
 extern int device_links_read_lock(void);
 extern void device_links_read_unlock(int idx);
 extern int device_links_check_suppliers(struct device *dev);
+extern void device_links_force_bind(struct device *dev);
 extern void device_links_driver_bound(struct device *dev);
 extern void device_links_driver_cleanup(struct device *dev);
 extern void device_links_no_driver(struct device *dev);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 7f27b007b040671f9d1556897672e82d8478718a..9f527937a7f71a4af657d8d1bbece0ffd3c342a2 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -586,6 +586,41 @@ int device_links_check_suppliers(struct device *dev)
 	return ret;
 }
 
+/**
+ * device_links_force_bind - Prepares device to be force bound
+ * @dev: Consumer device.
+ *
+ * device_bind_driver() force binds a device to a driver without calling any
+ * driver probe functions. So the consumer really isn't going to wait for any
+ * supplier before it's bound to the driver. We still want the device link
+ * states to be sensible when this happens.
+ *
+ * In preparation for device_bind_driver(), this function goes through each
+ * supplier device links and checks if the supplier is bound. If it is, then
+ * the device link status is set to CONSUMER_PROBE. Otherwise, the device link
+ * is dropped. Links without the DL_FLAG_MANAGED flag set are ignored.
+ */
+void device_links_force_bind(struct device *dev)
+{
+   struct device_link *link, *ln;
+
+   device_links_write_lock();
+
+   list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) {
+       if (!(link->flags & DL_FLAG_MANAGED))
+           continue;
+
+       if (link->status != DL_STATE_AVAILABLE) {
+           device_link_drop_managed(link);
+           continue;
+       }
+       WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
+   }
+   dev->links.status = DL_DEV_PROBING;
+
+   device_links_write_unlock();
+}
+
 /**
  * device_links_driver_bound - Update device links after probing its driver.
  * @dev: Device to update the links for.
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 0377c3c0f2d4ab453f25306bb3448bac1a678224..96cedc96a78866f107a3d0cb5a78d97ac715c405 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -424,8 +424,10 @@ int device_bind_driver(struct device *dev)
 	int ret;
 
 	ret = driver_sysfs_add(dev);
-	if (!ret)
+	if (!ret) {
+		device_links_force_bind(dev);
 		driver_bound(dev);
+	}
 	else if (dev->bus)
 		blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 					     BUS_NOTIFY_DRIVER_NOT_BOUND, dev);
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index a608ac136a973b4081b7d4fe063a275531e0b42f..53dd814adeb57f94400b165f60d925188db887f2 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -161,7 +161,7 @@ config AMD_IOMMU
 	select PCI_PASID
 	select IOMMU_API
 	select IOMMU_IOVA
-	depends on X86_64 && PCI && ACPI
+	depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE
 	---help---
 	  With this option you can enable support for AMD IOMMU hardware in
 	  your system. An IOMMU is a hardware component which provides
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index a1f417694a48d8b3925e67857d12ffe1add4edb2..9db6a8af9617e813a0858b9dd974007b489f3e96 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3807,6 +3807,7 @@ static int alloc_irq_index(u16 devid, int count, bool align)
 static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte,
 			  struct amd_ir_data *data)
 {
+	bool ret;
 	struct irq_remap_table *table;
 	struct amd_iommu *iommu;
 	unsigned long flags;
@@ -3824,10 +3825,18 @@ static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte,
 
 	entry = (struct irte_ga *)table->table;
 	entry = &entry[index];
-	entry->lo.fields_remap.valid = 0;
-	entry->hi.val = irte->hi.val;
-	entry->lo.val = irte->lo.val;
-	entry->lo.fields_remap.valid = 1;
+
+	ret = cmpxchg_double(&entry->lo.val, &entry->hi.val,
+			entry->lo.val, entry->hi.val,
+			irte->lo.val, irte->hi.val);
+	/*
+	 * We use cmpxchg16 to atomically update the 128-bit IRTE,
+	 * and it cannot be updated by the hardware or other processors
+	 * behind us, so the return value of cmpxchg16 should be the
+	 * same as the old value.
+	 */
+	WARN_ON(!ret);
+
 	if (data)
 		data->ref = entry;
 
@@ -4379,10 +4388,12 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
 	} else {
 		/* Un-Setting */
 		struct irq_cfg *cfg = irqd_cfg(data);
+		u64 valid = irte->lo.fields_remap.valid;
 
 		irte->hi.val = 0;
 		irte->lo.val = 0;
 		irte->hi.fields.vector = cfg->vector;
+		irte->lo.fields_remap.valid = valid;
 		irte->lo.fields_remap.guest_mode = 0;
 		irte->lo.fields_remap.destination =
 				APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 29bf91e50d4131f562990d6ce21c222cbc8f578a..abd327653d339bed467dc8670f40c882817dbd95 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -1525,7 +1525,14 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
 		else
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
-		if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
+
+		/*
+		 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
+		 * GAM also requires GA mode. Therefore, we need to
+		 * check cmpxchg16b support before enabling it.
+		 */
+		if (!boot_cpu_has(X86_FEATURE_CX16) ||
+			((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
 		if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0))
 			amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
@@ -1536,7 +1543,14 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
 		else
 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
-		if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
+
+		/*
+		 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
+		 * XT, GAM also requires GA mode. Therefore, we need to
+		 * check cmpxchg16b support before enabling them.
+		 */
+		if (!boot_cpu_has(X86_FEATURE_CX16) ||
+			((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0))
 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
 		if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0))
 			amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
diff --git a/lib/genalloc.c b/lib/genalloc.c
index 0b8ee173cf3a6a3d976d882ad3f02a3c8f80fd91..2f15b9de12c4671dd53977b85760775e501c70f2 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -83,7 +83,8 @@ static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
  * users set the same bit, one user will return remain bits, otherwise
  * return 0.
  */
-static int bitmap_set_ll(unsigned long *map, unsigned long start, unsigned long nr)
+static unsigned long
+bitmap_set_ll(unsigned long *map, unsigned long start, unsigned long nr)
 {
 	unsigned long *p = map + BIT_WORD(start);
 	const unsigned long size = start + nr;