3 files changed, 372 insertions, 145 deletions

diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c
index edd856a000c5..b9fe80ec1089 100644
--- a/arch/tile/kernel/pci-dma.c
+++ b/arch/tile/kernel/pci-dma.c
@@ -14,6 +14,7 @@
 
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
+#include <linux/swiotlb.h>
 #include <linux/vmalloc.h>
 #include <linux/export.h>
 #include <asm/tlbflush.h>
@@ -31,10 +32,9 @@
 #define PAGE_HOME_DMA PAGE_HOME_HASH
 #endif
 
-void *dma_alloc_coherent(struct device *dev,
-			 size_t size,
-			 dma_addr_t *dma_handle,
-			 gfp_t gfp)
+static void *tile_dma_alloc_coherent(struct device *dev, size_t size,
+				     dma_addr_t *dma_handle, gfp_t gfp,
+				     struct dma_attrs *attrs)
 {
 	u64 dma_mask = dev->coherent_dma_mask ?: DMA_BIT_MASK(32);
 	int node = dev_to_node(dev);
@@ -68,19 +68,19 @@ void *dma_alloc_coherent(struct device *dev,
 	}
 
 	*dma_handle = addr;
+
 	return page_address(pg);
 }
-EXPORT_SYMBOL(dma_alloc_coherent);
 
 /*
- * Free memory that was allocated with dma_alloc_coherent.
+ * Free memory that was allocated with tile_dma_alloc_coherent.
  */
-void dma_free_coherent(struct device *dev, size_t size,
-		  void *vaddr, dma_addr_t dma_handle)
+static void tile_dma_free_coherent(struct device *dev, size_t size,
+				   void *vaddr, dma_addr_t dma_handle,
+				   struct dma_attrs *attrs)
 {
 	homecache_free_pages((unsigned long)vaddr, get_order(size));
 }
-EXPORT_SYMBOL(dma_free_coherent);
 
 /*
  * The map routines "map" the specified address range for DMA
@@ -199,38 +199,182 @@ static void __dma_complete_pa_range(dma_addr_t dma_addr, size_t size,
 	}
 }
 
+static int tile_dma_map_sg(struct device *dev, struct scatterlist *sglist,
+			   int nents, enum dma_data_direction direction,
+			   struct dma_attrs *attrs)
+{
+	struct scatterlist *sg;
+	int i;
 
-/*
- * dma_map_single can be passed any memory address, and there appear
- * to be no alignment constraints.
- *
- * There is a chance that the start of the buffer will share a cache
- * line with some other data that has been touched in the meantime.
- */
-dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
-			  enum dma_data_direction direction)
+	BUG_ON(!valid_dma_direction(direction));
+
+	WARN_ON(nents == 0 || sglist->length == 0);
+
+	for_each_sg(sglist, sg, nents, i) {
+		sg->dma_address = sg_phys(sg);
+		__dma_prep_pa_range(sg->dma_address, sg->length, direction);
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+		sg->dma_length = sg->length;
+#endif
+	}
+
+	return nents;
+}
+
+static void tile_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
+			      int nents, enum dma_data_direction direction,
+			      struct dma_attrs *attrs)
+{
+	struct scatterlist *sg;
+	int i;
+
+	BUG_ON(!valid_dma_direction(direction));
+	for_each_sg(sglist, sg, nents, i) {
+		sg->dma_address = sg_phys(sg);
+		__dma_complete_pa_range(sg->dma_address, sg->length,
+					direction);
+	}
+}
+
+static dma_addr_t tile_dma_map_page(struct device *dev, struct page *page,
+				    unsigned long offset, size_t size,
+				    enum dma_data_direction direction,
+				    struct dma_attrs *attrs)
 {
-	dma_addr_t dma_addr = __pa(ptr);
+	BUG_ON(!valid_dma_direction(direction));
+
+	BUG_ON(offset + size > PAGE_SIZE);
+	__dma_prep_page(page, offset, size, direction);
+
+	return page_to_pa(page) + offset;
+}
+
+static void tile_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
+				size_t size, enum dma_data_direction direction,
+				struct dma_attrs *attrs)
+{
+	BUG_ON(!valid_dma_direction(direction));
+
+	__dma_complete_page(pfn_to_page(PFN_DOWN(dma_address)),
+			    dma_address & PAGE_OFFSET, size, direction);
+}
 
+static void tile_dma_sync_single_for_cpu(struct device *dev,
+					 dma_addr_t dma_handle,
+					 size_t size,
+					 enum dma_data_direction direction)
+{
 	BUG_ON(!valid_dma_direction(direction));
-	WARN_ON(size == 0);
 
-	__dma_prep_pa_range(dma_addr, size, direction);
+	__dma_complete_pa_range(dma_handle, size, direction);
+}
 
-	return dma_addr;
+static void tile_dma_sync_single_for_device(struct device *dev,
+					    dma_addr_t dma_handle, size_t size,
+					    enum dma_data_direction direction)
+{
+	__dma_prep_pa_range(dma_handle, size, direction);
 }
-EXPORT_SYMBOL(dma_map_single);
 
-void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
-		      enum dma_data_direction direction)
+static void tile_dma_sync_sg_for_cpu(struct device *dev,
+				     struct scatterlist *sglist, int nelems,
+				     enum dma_data_direction direction)
 {
+	struct scatterlist *sg;
+	int i;
+
 	BUG_ON(!valid_dma_direction(direction));
-	__dma_complete_pa_range(dma_addr, size, direction);
+	WARN_ON(nelems == 0 || sglist->length == 0);
+
+	for_each_sg(sglist, sg, nelems, i) {
+		dma_sync_single_for_cpu(dev, sg->dma_address,
+					sg_dma_len(sg), direction);
+	}
 }
-EXPORT_SYMBOL(dma_unmap_single);
 
-int dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
-	       enum dma_data_direction direction)
+static void tile_dma_sync_sg_for_device(struct device *dev,
+					struct scatterlist *sglist, int nelems,
+					enum dma_data_direction direction)
+{
+	struct scatterlist *sg;
+	int i;
+
+	BUG_ON(!valid_dma_direction(direction));
+	WARN_ON(nelems == 0 || sglist->length == 0);
+
+	for_each_sg(sglist, sg, nelems, i) {
+		dma_sync_single_for_device(dev, sg->dma_address,
+					   sg_dma_len(sg), direction);
+	}
+}
+
+static inline int
+tile_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+	return 0;
+}
+
+static inline int
+tile_dma_supported(struct device *dev, u64 mask)
+{
+	return 1;
+}
+
+static struct dma_map_ops tile_default_dma_map_ops = {
+	.alloc = tile_dma_alloc_coherent,
+	.free = tile_dma_free_coherent,
+	.map_page = tile_dma_map_page,
+	.unmap_page = tile_dma_unmap_page,
+	.map_sg = tile_dma_map_sg,
+	.unmap_sg = tile_dma_unmap_sg,
+	.sync_single_for_cpu = tile_dma_sync_single_for_cpu,
+	.sync_single_for_device = tile_dma_sync_single_for_device,
+	.sync_sg_for_cpu = tile_dma_sync_sg_for_cpu,
+	.sync_sg_for_device = tile_dma_sync_sg_for_device,
+	.mapping_error = tile_dma_mapping_error,
+	.dma_supported = tile_dma_supported
+};
+
+struct dma_map_ops *tile_dma_map_ops = &tile_default_dma_map_ops;
+EXPORT_SYMBOL(tile_dma_map_ops);
+
+/* Generic PCI DMA mapping functions */
+
+static void *tile_pci_dma_alloc_coherent(struct device *dev, size_t size,
+					 dma_addr_t *dma_handle, gfp_t gfp,
+					 struct dma_attrs *attrs)
+{
+	int node = dev_to_node(dev);
+	int order = get_order(size);
+	struct page *pg;
+	dma_addr_t addr;
+
+	gfp |= __GFP_ZERO;
+
+	pg = homecache_alloc_pages_node(node, gfp, order, PAGE_HOME_DMA);
+	if (pg == NULL)
+		return NULL;
+
+	addr = page_to_phys(pg);
+
+	*dma_handle = phys_to_dma(dev, addr);
+
+	return page_address(pg);
+}
+
+/*
+ * Free memory that was allocated with tile_pci_dma_alloc_coherent.
+ */
+static void tile_pci_dma_free_coherent(struct device *dev, size_t size,
+				       void *vaddr, dma_addr_t dma_handle,
+				       struct dma_attrs *attrs)
+{
+	homecache_free_pages((unsigned long)vaddr, get_order(size));
+}
+
+static int tile_pci_dma_map_sg(struct device *dev, struct scatterlist *sglist,
+			       int nents, enum dma_data_direction direction,
+			       struct dma_attrs *attrs)
 {
 	struct scatterlist *sg;
 	int i;
@@ -242,14 +386,20 @@ int dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents,
 	for_each_sg(sglist, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
 		__dma_prep_pa_range(sg->dma_address, sg->length, direction);
+
+		sg->dma_address = phys_to_dma(dev, sg->dma_address);
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+		sg->dma_length = sg->length;
+#endif
 	}
 
 	return nents;
 }
-EXPORT_SYMBOL(dma_map_sg);
 
-void dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
-		  enum dma_data_direction direction)
+static void tile_pci_dma_unmap_sg(struct device *dev,
+				  struct scatterlist *sglist, int nents,
+				  enum dma_data_direction direction,
+				  struct dma_attrs *attrs)
 {
 	struct scatterlist *sg;
 	int i;
@@ -261,46 +411,60 @@ void dma_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents,
 					direction);
 	}
 }
-EXPORT_SYMBOL(dma_unmap_sg);
 
-dma_addr_t dma_map_page(struct device *dev, struct page *page,
-			unsigned long offset, size_t size,
-			enum dma_data_direction direction)
+static dma_addr_t tile_pci_dma_map_page(struct device *dev, struct page *page,
+					unsigned long offset, size_t size,
+					enum dma_data_direction direction,
+					struct dma_attrs *attrs)
 {
 	BUG_ON(!valid_dma_direction(direction));
 
 	BUG_ON(offset + size > PAGE_SIZE);
 	__dma_prep_page(page, offset, size, direction);
-	return page_to_pa(page) + offset;
+
+	return phys_to_dma(dev, page_to_pa(page) + offset);
 }
-EXPORT_SYMBOL(dma_map_page);
 
-void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
-		    enum dma_data_direction direction)
+static void tile_pci_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
+				    size_t size,
+				    enum dma_data_direction direction,
+				    struct dma_attrs *attrs)
 {
 	BUG_ON(!valid_dma_direction(direction));
+
+	dma_address = dma_to_phys(dev, dma_address);
+
 	__dma_complete_page(pfn_to_page(PFN_DOWN(dma_address)),
 			    dma_address & PAGE_OFFSET, size, direction);
 }
-EXPORT_SYMBOL(dma_unmap_page);
 
-void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
-			     size_t size, enum dma_data_direction direction)
+static void tile_pci_dma_sync_single_for_cpu(struct device *dev,
+					     dma_addr_t dma_handle,
+					     size_t size,
+					     enum dma_data_direction direction)
 {
 	BUG_ON(!valid_dma_direction(direction));
+
+	dma_handle = dma_to_phys(dev, dma_handle);
+
 	__dma_complete_pa_range(dma_handle, size, direction);
 }
-EXPORT_SYMBOL(dma_sync_single_for_cpu);
 
-void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
-				size_t size, enum dma_data_direction direction)
+static void tile_pci_dma_sync_single_for_device(struct device *dev,
+						dma_addr_t dma_handle,
+						size_t size,
+						enum dma_data_direction
+						direction)
 {
+	dma_handle = dma_to_phys(dev, dma_handle);
+
 	__dma_prep_pa_range(dma_handle, size, direction);
 }
-EXPORT_SYMBOL(dma_sync_single_for_device);
 
-void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
-			 int nelems, enum dma_data_direction direction)
+static void tile_pci_dma_sync_sg_for_cpu(struct device *dev,
+					 struct scatterlist *sglist,
+					 int nelems,
+					 enum dma_data_direction direction)
 {
 	struct scatterlist *sg;
 	int i;
@@ -313,10 +477,11 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
 					sg_dma_len(sg), direction);
 	}
 }
-EXPORT_SYMBOL(dma_sync_sg_for_cpu);
 
-void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
-			    int nelems, enum dma_data_direction direction)
+static void tile_pci_dma_sync_sg_for_device(struct device *dev,
+					    struct scatterlist *sglist,
+					    int nelems,
+					    enum dma_data_direction direction)
 {
 	struct scatterlist *sg;
 	int i;
@@ -329,31 +494,93 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
 					   sg_dma_len(sg), direction);
 	}
 }
-EXPORT_SYMBOL(dma_sync_sg_for_device);
 
-void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
-				   unsigned long offset, size_t size,
-				   enum dma_data_direction direction)
+static inline int
+tile_pci_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
-	dma_sync_single_for_cpu(dev, dma_handle + offset, size, direction);
+	return 0;
 }
-EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
 
-void dma_sync_single_range_for_device(struct device *dev,
-				      dma_addr_t dma_handle,
-				      unsigned long offset, size_t size,
-				      enum dma_data_direction direction)
+static inline int
+tile_pci_dma_supported(struct device *dev, u64 mask)
 {
-	dma_sync_single_for_device(dev, dma_handle + offset, size, direction);
+	return 1;
 }
-EXPORT_SYMBOL(dma_sync_single_range_for_device);
 
-/*
- * dma_alloc_noncoherent() is #defined to return coherent memory,
- * so there's no need to do any flushing here.
- */
-void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
-		    enum dma_data_direction direction)
+static struct dma_map_ops tile_pci_default_dma_map_ops = {
+	.alloc = tile_pci_dma_alloc_coherent,
+	.free = tile_pci_dma_free_coherent,
+	.map_page = tile_pci_dma_map_page,
+	.unmap_page = tile_pci_dma_unmap_page,
+	.map_sg = tile_pci_dma_map_sg,
+	.unmap_sg = tile_pci_dma_unmap_sg,
+	.sync_single_for_cpu = tile_pci_dma_sync_single_for_cpu,
+	.sync_single_for_device = tile_pci_dma_sync_single_for_device,
+	.sync_sg_for_cpu = tile_pci_dma_sync_sg_for_cpu,
+	.sync_sg_for_device = tile_pci_dma_sync_sg_for_device,
+	.mapping_error = tile_pci_dma_mapping_error,
+	.dma_supported = tile_pci_dma_supported
+};
+
+struct dma_map_ops *gx_pci_dma_map_ops = &tile_pci_default_dma_map_ops;
+EXPORT_SYMBOL(gx_pci_dma_map_ops);
+
+/* PCI DMA mapping functions for legacy PCI devices */
+
+#ifdef CONFIG_SWIOTLB
+static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size,
+					 dma_addr_t *dma_handle, gfp_t gfp,
+					 struct dma_attrs *attrs)
+{
+	gfp |= GFP_DMA;
+	return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
+}
+
+static void tile_swiotlb_free_coherent(struct device *dev, size_t size,
+				       void *vaddr, dma_addr_t dma_addr,
+				       struct dma_attrs *attrs)
 {
+	swiotlb_free_coherent(dev, size, vaddr, dma_addr);
 }
-EXPORT_SYMBOL(dma_cache_sync);
+
+static struct dma_map_ops pci_swiotlb_dma_ops = {
+	.alloc = tile_swiotlb_alloc_coherent,
+	.free = tile_swiotlb_free_coherent,
+	.map_page = swiotlb_map_page,
+	.unmap_page = swiotlb_unmap_page,
+	.map_sg = swiotlb_map_sg_attrs,
+	.unmap_sg = swiotlb_unmap_sg_attrs,
+	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
+	.sync_single_for_device = swiotlb_sync_single_for_device,
+	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
+	.sync_sg_for_device = swiotlb_sync_sg_for_device,
+	.dma_supported = swiotlb_dma_supported,
+	.mapping_error = swiotlb_dma_mapping_error,
+};
+
+struct dma_map_ops *gx_legacy_pci_dma_map_ops = &pci_swiotlb_dma_ops;
+#else
+struct dma_map_ops *gx_legacy_pci_dma_map_ops;
+#endif
+EXPORT_SYMBOL(gx_legacy_pci_dma_map_ops);
+
+#ifdef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
+int dma_set_coherent_mask(struct device *dev, u64 mask)
+{
+	struct dma_map_ops *dma_ops = get_dma_ops(dev);
+
+	/* Handle legacy PCI devices with limited memory addressability. */
+	if (((dma_ops == gx_pci_dma_map_ops) ||
+	    (dma_ops == gx_legacy_pci_dma_map_ops)) &&
+	    (mask <= DMA_BIT_MASK(32))) {
+		if (mask > dev->archdata.max_direct_dma_addr)
+			mask = dev->archdata.max_direct_dma_addr;
+	}
+
+	if (!dma_supported(dev, mask))
+		return -EIO;
+	dev->coherent_dma_mask = mask;
+	return 0;
+}
+EXPORT_SYMBOL(dma_set_coherent_mask);
+#endif
diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c
index 1b996bb628f1..27f7ab021137 100644
--- a/arch/tile/kernel/pci_gx.c
+++ b/arch/tile/kernel/pci_gx.c
@@ -40,22 +40,8 @@
 #include <arch/sim.h>
 
 /*
- * Initialization flow and process
- * -------------------------------
- *
- * This files containes the routines to search for PCI buses,
+ * This file containes the routines to search for PCI buses,
  * enumerate the buses, and configure any attached devices.
- *
- * There are two entry points here:
- * 1) tile_pci_init
- *    This sets up the pci_controller structs, and opens the
- *    FDs to the hypervisor.  This is called from setup_arch() early
- *    in the boot process.
- * 2) pcibios_init
- *    This probes the PCI bus(es) for any attached hardware.  It's
- *    called by subsys_initcall.  All of the real work is done by the
- *    generic Linux PCI layer.
- *
  */
 
 #define DEBUG_PCI_CFG	0
@@ -110,6 +96,21 @@ static struct pci_ops tile_cfg_ops;
 /* Mask of CPUs that should receive PCIe interrupts. */
 static struct cpumask intr_cpus_map;
 
+/* PCI I/O space support is not implemented. */
+static struct resource pci_ioport_resource = {
+	.name	= "PCI IO",
+	.start	= 0,
+	.end	= 0,
+	.flags	= IORESOURCE_IO,
+};
+
+static struct resource pci_iomem_resource = {
+	.name	= "PCI mem",
+	.start	= TILE_PCI_MEM_START,
+	.end	= TILE_PCI_MEM_END,
+	.flags	= IORESOURCE_MEM,
+};
+
 /*
  * We don't need to worry about the alignment of resources.
  */
@@ -334,8 +335,6 @@ free_irqs:
 }
 
 /*
- * First initialization entry point, called from setup_arch().
- *
  * Find valid controllers and fill in pci_controller structs for each
  * of them.
  *
@@ -583,10 +582,7 @@ static int __devinit setup_pcie_rc_delay(char *str)
 early_param("pcie_rc_delay", setup_pcie_rc_delay);
 
 /*
- * Second PCI initialization entry point, called by subsys_initcall.
- *
- * The controllers have been set up by the time we get here, by a call to
- * tile_pci_init.
+ * PCI initialization entry point, called by subsys_initcall.
  */
 int __init pcibios_init(void)
 {
@@ -594,15 +590,13 @@ int __init pcibios_init(void)
 	LIST_HEAD(resources);
 	int i;
 
+	tile_pci_init();
+
 	if (num_rc_controllers == 0 && num_ep_controllers == 0)
 		return 0;
 
-	pr_info("PCI: Probing PCI hardware\n");
-
 	/*
 	 * We loop over all the TRIO shims and set up the MMIO mappings.
-	 * This step can't be done in tile_pci_init because the MM subsystem
-	 * hasn't been initialized then.
 	 */
 	for (i = 0; i < TILEGX_NUM_TRIO; i++) {
 		gxio_trio_context_t *context = &trio_contexts[i];
@@ -645,9 +639,7 @@ int __init pcibios_init(void)
 		unsigned int class_code_revision;
 		int trio_index;
 		int mac;
-#ifndef USE_SHARED_PCIE_CONFIG_REGION
 		int ret;
-#endif
 
 		if (trio_context->fd < 0)
 			continue;
@@ -802,8 +794,6 @@ int __init pcibios_init(void)
 			pr_err("PCI: PCI CFG PIO alloc failure for mac %d "
 				"on TRIO %d, give up\n", mac, trio_index);
 
-			/* TBD: cleanup ... */
-
 			continue;
 		}
 
@@ -819,8 +809,6 @@ int __init pcibios_init(void)
 			pr_err("PCI: PCI CFG PIO init failure for mac %d "
 				"on TRIO %d, give up\n", mac, trio_index);
 
-			/* TBD: cleanup ... */
-
 			continue;
 		}
 
@@ -837,8 +825,6 @@ int __init pcibios_init(void)
 			pr_err("PCI: PIO map failure for mac %d on TRIO %d\n",
 				mac, trio_index);
 
-			/* TBD: cleanup ... */
-
 			continue;
 		}
 
@@ -852,7 +838,14 @@ int __init pcibios_init(void)
 			continue;
 		}
 
-		pci_add_resource(&resources, &iomem_resource);
+		/*
+		 * The PCI memory resource is located above the PA space.
+		 * The memory range for the PCI root bus should not overlap
+		 * with the physical RAM
+		 */
+		pci_add_resource_offset(&resources, &iomem_resource,
+					1ULL << CHIP_PA_WIDTH());
+
 		bus = pci_scan_root_bus(NULL, 0, controller->ops,
 					controller, &resources);
 		controller->root_bus = bus;
@@ -923,11 +916,6 @@ int __init pcibios_init(void)
 		}
 
 		/*
-		 * We always assign 32-bit PCI bus BAR ranges.
-		 */
-		BUG_ON(bus_address_hi != 0);
-
-		/*
 		 * Alloc a PIO region for PCI memory access for each RC port.
 		 */
 		ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
@@ -936,8 +924,6 @@ int __init pcibios_init(void)
 				"give up\n", controller->trio_index,
 				controller->mac);
 
-			/* TBD: cleanup ... */
-
 			continue;
 		}
 
@@ -950,15 +936,13 @@ int __init pcibios_init(void)
 		ret = gxio_trio_init_pio_region_aux(trio_context,
 						    controller->pio_mem_index,
 						    controller->mac,
-						    bus_address_hi,
+						    0,
 						    0);
 		if (ret < 0) {
 			pr_err("PCI: MEM PIO init failure on TRIO %d mac %d, "
 				"give up\n", controller->trio_index,
 				controller->mac);
 
-			/* TBD: cleanup ... */
-
 			continue;
 		}
 
@@ -980,8 +964,6 @@ int __init pcibios_init(void)
 					controller->trio_index,
 					controller->mac, j);
 
-				/* TBD: cleanup ... */
-
 				goto alloc_mem_map_failed;
 			}
 
@@ -991,9 +973,13 @@ int __init pcibios_init(void)
 			 * Initialize the Mem-Map and the I/O MMU so that all
 			 * the physical memory can be accessed by the endpoint
 			 * devices. The base bus address is set to the base CPA
-			 * of this memory controller, so is the base VA. The
+			 * of this memory controller plus an offset (see pci.h).
+			 * The region's base VA is set to the base CPA. The
 			 * I/O MMU table essentially translates the CPA to
-			 * the real PA.
+			 * the real PA. Implicitly, for node 0, we create
+			 * a separate Mem-Map region that serves as the inbound
+			 * window for legacy 32-bit devices. This is a direct
+			 * map of the low 4GB CPA space.
 			 */
 			ret = gxio_trio_init_memory_map_mmu_aux(trio_context,
 				controller->mem_maps[j],
@@ -1001,7 +987,8 @@ int __init pcibios_init(void)
 				nr_pages << PAGE_SHIFT,
 				trio_context->asid,
 				controller->mac,
-				start_pfn << PAGE_SHIFT,
+				(start_pfn << PAGE_SHIFT) +
+				TILE_PCI_MEM_MAP_BASE_OFFSET,
 				j,
 				GXIO_TRIO_ORDER_MODE_UNORDERED);
 			if (ret < 0) {
@@ -1010,11 +997,8 @@ int __init pcibios_init(void)
 					controller->trio_index,
 					controller->mac, j);
 
-				/* TBD: cleanup ... */
-
 				goto alloc_mem_map_failed;
 			}
-
 			continue;
 
 alloc_mem_map_failed:
@@ -1028,11 +1012,19 @@ alloc_mem_map_failed:
 subsys_initcall(pcibios_init);
 
 /*
- * No bus fixups needed.
+ * PCI scan code calls the arch specific pcibios_fixup_bus() each time it scans
+ * a new bridge. Called after each bus is probed, but before its children are
+ * examined.
  */
 void __devinit pcibios_fixup_bus(struct pci_bus *bus)
 {
-	/* Nothing needs to be done. */
+	struct pci_dev *dev = bus->self;
+
+	if (!dev) {
+		/* This is the root bus. */
+		bus->resource[0] = &pci_ioport_resource;
+		bus->resource[1] = &pci_iomem_resource;
+	}
 }
 
 /*
@@ -1069,6 +1061,17 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 	return pci_enable_resources(dev, mask);
 }
 
+/* Called for each device after PCI setup is done. */
+static void __init
+pcibios_fixup_final(struct pci_dev *pdev)
+{
+	set_dma_ops(&pdev->dev, gx_pci_dma_map_ops);
+	set_dma_offset(&pdev->dev, TILE_PCI_MEM_MAP_BASE_OFFSET);
+	pdev->dev.archdata.max_direct_dma_addr =
+		TILE_PCI_MAX_DIRECT_DMA_ADDRESS;
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_final);
+
 /* Map a PCI MMIO bus address into VA space. */
 void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
 {
@@ -1127,7 +1130,7 @@ got_it:
 	 * We need to keep the PCI bus address's in-page offset in the VA.
 	 */
 	return iorpc_ioremap(trio_fd, offset, size) +
-					(phys_addr & (PAGE_SIZE - 1));
+		(phys_addr & (PAGE_SIZE - 1));
 }
 EXPORT_SYMBOL(ioremap);
 
diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c
index fdde3b6986e5..2b8b689e596d 100644
--- a/arch/tile/kernel/setup.c
+++ b/arch/tile/kernel/setup.c
@@ -23,6 +23,7 @@
 #include <linux/irq.h>
 #include <linux/kexec.h>
 #include <linux/pci.h>
+#include <linux/swiotlb.h>
 #include <linux/initrd.h>
 #include <linux/io.h>
 #include <linux/highmem.h>
@@ -109,7 +110,7 @@ static unsigned int __initdata maxnodemem_pfn[MAX_NUMNODES] = {
 };
 static nodemask_t __initdata isolnodes;
 
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 enum { DEFAULT_PCI_RESERVE_MB = 64 };
 static unsigned int __initdata pci_reserve_mb = DEFAULT_PCI_RESERVE_MB;
 unsigned long __initdata pci_reserve_start_pfn = -1U;
@@ -160,7 +161,7 @@ static int __init setup_isolnodes(char *str)
 }
 early_param("isolnodes", setup_isolnodes);
 
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 static int __init setup_pci_reserve(char* str)
 {
 	unsigned long mb;
@@ -171,7 +172,7 @@ static int __init setup_pci_reserve(char* str)
 
 	pci_reserve_mb = mb;
 	pr_info("Reserving %dMB for PCIE root complex mappings\n",
-	       pci_reserve_mb);
+		pci_reserve_mb);
 	return 0;
 }
 early_param("pci_reserve", setup_pci_reserve);
@@ -411,7 +412,7 @@ static void __init setup_memory(void)
 			continue;
 		}
 #endif
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 		/*
 		 * Blocks that overlap the pci reserved region must
 		 * have enough space to hold the maximum percpu data
@@ -604,11 +605,9 @@ static void __init setup_bootmem_allocator_node(int i)
 	/* Free all the space back into the allocator. */
 	free_bootmem(PFN_PHYS(start), PFN_PHYS(end - start));
 
-#if defined(CONFIG_PCI)
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 	/*
-	 * Throw away any memory aliased by the PCI region.  FIXME: this
-	 * is a temporary hack to work around bug 10502, and needs to be
-	 * fixed properly.
+	 * Throw away any memory aliased by the PCI region.
 	 */
 	if (pci_reserve_start_pfn < end && pci_reserve_end_pfn > start)
 		reserve_bootmem(PFN_PHYS(pci_reserve_start_pfn),
@@ -1353,8 +1352,7 @@ void __init setup_arch(char **cmdline_p)
 	setup_cpu_maps();
 
 
-#ifdef CONFIG_PCI
-#if !defined (__tilegx__)
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 	/*
 	 * Initialize the PCI structures.  This is done before memory
 	 * setup so that we know whether or not a pci_reserve region
@@ -1362,7 +1360,6 @@ void __init setup_arch(char **cmdline_p)
 	 */
 	if (tile_pci_init() == 0)
 		pci_reserve_mb = 0;
-#endif
 
 	/* PCI systems reserve a region just below 4GB for mapping iomem. */
 	pci_reserve_end_pfn  = (1 << (32 - PAGE_SHIFT));
@@ -1384,6 +1381,10 @@ void __init setup_arch(char **cmdline_p)
 	 * any memory using the bootmem allocator.
 	 */
 
+#ifdef CONFIG_SWIOTLB
+	swiotlb_init(0);
+#endif
+
 	paging_init();
 	setup_numa_mapping();
 	zone_sizes_init();
@@ -1391,10 +1392,6 @@ void __init setup_arch(char **cmdline_p)
 	setup_cpu(1);
 	setup_clock();
 	load_hv_initrd();
-
-#if defined(CONFIG_PCI) && defined (__tilegx__)
-	tile_pci_init();
-#endif
 }
 
 
@@ -1538,11 +1535,11 @@ static struct resource code_resource = {
 };
 
 /*
- * We reserve all resources above 4GB so that PCI won't try to put
+ * On Pro, we reserve all resources above 4GB so that PCI won't try to put
  * mappings above 4GB; the standard allows that for some devices but
  * the probing code trunates values to 32 bits.
  */
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 static struct resource* __init
 insert_non_bus_resource(void)
 {
@@ -1588,7 +1585,7 @@ static int __init request_standard_resources(void)
 	enum { CODE_DELTA = MEM_SV_INTRPT - PAGE_OFFSET };
 
 	iomem_resource.end = -1LL;
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 	insert_non_bus_resource();
 #endif
 
@@ -1596,7 +1593,7 @@ static int __init request_standard_resources(void)
 		u64 start_pfn = node_start_pfn[i];
 		u64 end_pfn = node_end_pfn[i];
 
-#ifdef CONFIG_PCI
+#if defined(CONFIG_PCI) && !defined(__tilegx__)
 		if (start_pfn <= pci_reserve_start_pfn &&
 		    end_pfn > pci_reserve_start_pfn) {
 			if (end_pfn > pci_reserve_end_pfn)