aboutsummaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/Makefile22
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu.h213
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c269
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h65
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c670
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c543
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c838
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c283
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c213
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_device.c36
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_display.c81
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c154
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c49
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c580
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c40
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c25
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.c26
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_object.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c28
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c56
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c128
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c195
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_test.c51
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c76
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c156
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c187
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c305
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_dp.c3
-rw-r--r--drivers/gpu/drm/amd/amdgpu/atombios_encoders.c2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik.c23
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cik_sdma.c95
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cikd.h7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/cz_dpm.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v10_0.c29
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v11_0.c6
-rw-r--r--drivers/gpu/drm/amd/amdgpu/dce_v8_0.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/fiji_dpm.c181
-rw-r--r--drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h182
-rw-r--r--drivers/gpu/drm/amd/amdgpu/fiji_smc.c857
-rw-r--r--drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h (renamed from drivers/gpu/drm/amd/amdgpu/amdgpu_family.h)52
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c52
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c155
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c26
-rw-r--r--drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c50
-rw-r--r--drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/iceland_smc.c4
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c98
-rw-r--r--drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c127
-rw-r--r--drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h5
-rw-r--r--drivers/gpu/drm/amd/amdgpu/tonga_smc.c8
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c7
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v2_0.c1
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vce_v3_0.c58
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi.c120
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vi_dpm.h2
-rw-r--r--drivers/gpu/drm/amd/amdgpu/vid.h5
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Kconfig2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Makefile3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/cik_regs.h11
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_chardev.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c7
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c12
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c103
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c20
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c249
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c99
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h398
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.c5
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_topology.h1
-rw-r--r--drivers/gpu/drm/amd/include/amd_shared.h39
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h1246
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h1282
-rw-r--r--drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h6080
-rw-r--r--drivers/gpu/drm/amd/include/atom-bits.h (renamed from drivers/gpu/drm/amd/amdgpu/atom-bits.h)0
-rw-r--r--drivers/gpu/drm/amd/include/atom-names.h (renamed from drivers/gpu/drm/amd/amdgpu/atom-names.h)0
-rw-r--r--drivers/gpu/drm/amd/include/atom-types.h (renamed from drivers/gpu/drm/amd/amdgpu/atom-types.h)0
-rw-r--r--drivers/gpu/drm/amd/include/atombios.h (renamed from drivers/gpu/drm/amd/amdgpu/atombios.h)0
-rw-r--r--drivers/gpu/drm/amd/include/cgs_common.h624
-rw-r--r--drivers/gpu/drm/amd/include/cgs_linux.h135
-rw-r--r--drivers/gpu/drm/amd/include/kgd_kfd_interface.h3
-rw-r--r--drivers/gpu/drm/amd/include/pptable.h (renamed from drivers/gpu/drm/amd/amdgpu/pptable.h)6
-rw-r--r--drivers/gpu/drm/amd/include/vi_structs.h417
-rw-r--r--drivers/gpu/drm/amd/scheduler/gpu_scheduler.c424
-rw-r--r--drivers/gpu/drm/amd/scheduler/gpu_scheduler.h134
-rw-r--r--drivers/gpu/drm/amd/scheduler/sched_fence.c81
103 files changed, 17471 insertions, 1477 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 616dfd4a1398..04c270757030 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -3,7 +3,9 @@
# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/asic_reg \
- -Idrivers/gpu/drm/amd/include
+ -Idrivers/gpu/drm/amd/include \
+ -Idrivers/gpu/drm/amd/amdgpu \
+ -Idrivers/gpu/drm/amd/scheduler
amdgpu-y := amdgpu_drv.o
@@ -21,7 +23,8 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
# add asic specific block
amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o gmc_v7_0.o cik_ih.o kv_smc.o kv_dpm.o \
- ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o
+ ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o \
+ amdgpu_amdkfd_gfx_v7.o
amdgpu-y += \
vi.o
@@ -43,6 +46,7 @@ amdgpu-y += \
amdgpu_dpm.o \
cz_smc.o cz_dpm.o \
tonga_smc.o tonga_dpm.o \
+ fiji_smc.o fiji_dpm.o \
iceland_smc.o iceland_dpm.o
# add DCE block
@@ -71,6 +75,20 @@ amdgpu-y += \
amdgpu_vce.o \
vce_v3_0.o
+# add amdkfd interfaces
+amdgpu-y += \
+ amdgpu_amdkfd.o \
+ amdgpu_amdkfd_gfx_v8.o
+
+# add cgs
+amdgpu-y += amdgpu_cgs.o
+
+# GPU scheduler
+amdgpu-y += \
+ ../scheduler/gpu_scheduler.o \
+ ../scheduler/sched_fence.o \
+ amdgpu_sched.o
+
amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o
amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o
amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index f7b49d5ce4b8..668939a14206 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -42,17 +42,19 @@
#include <ttm/ttm_module.h>
#include <ttm/ttm_execbuf_util.h>
+#include <drm/drmP.h>
#include <drm/drm_gem.h>
#include <drm/amdgpu_drm.h>
#include "amd_shared.h"
-#include "amdgpu_family.h"
#include "amdgpu_mode.h"
#include "amdgpu_ih.h"
#include "amdgpu_irq.h"
#include "amdgpu_ucode.h"
#include "amdgpu_gds.h"
+#include "gpu_scheduler.h"
+
/*
* Modules parameters.
*/
@@ -77,7 +79,11 @@ extern int amdgpu_bapm;
extern int amdgpu_deep_color;
extern int amdgpu_vm_size;
extern int amdgpu_vm_block_size;
+extern int amdgpu_enable_scheduler;
+extern int amdgpu_sched_jobs;
+extern int amdgpu_sched_hw_submission;
+#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000
#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */
#define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2)
/* AMDGPU_IB_POOL_SIZE must be a power of 2 */
@@ -92,6 +98,9 @@ extern int amdgpu_vm_block_size;
#define AMDGPU_MAX_COMPUTE_RINGS 8
#define AMDGPU_MAX_VCE_RINGS 2
+/* max number of IP instances */
+#define AMDGPU_MAX_SDMA_INSTANCES 2
+
/* number of hw syncs before falling back on blocking */
#define AMDGPU_NUM_SYNCS 4
@@ -177,7 +186,9 @@ struct amdgpu_vm;
struct amdgpu_ring;
struct amdgpu_semaphore;
struct amdgpu_cs_parser;
+struct amdgpu_job;
struct amdgpu_irq_src;
+struct amdgpu_fpriv;
enum amdgpu_cp_irq {
AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -239,7 +250,7 @@ struct amdgpu_buffer_funcs {
unsigned copy_num_dw;
/* used for buffer migration */
- void (*emit_copy_buffer)(struct amdgpu_ring *ring,
+ void (*emit_copy_buffer)(struct amdgpu_ib *ib,
/* src addr in bytes */
uint64_t src_offset,
/* dst addr in bytes */
@@ -254,7 +265,7 @@ struct amdgpu_buffer_funcs {
unsigned fill_num_dw;
/* used for buffer clearing */
- void (*emit_fill_buffer)(struct amdgpu_ring *ring,
+ void (*emit_fill_buffer)(struct amdgpu_ib *ib,
/* value to write to memory */
uint32_t src_data,
/* dst addr in bytes */
@@ -332,6 +343,8 @@ struct amdgpu_ring_funcs {
int (*test_ring)(struct amdgpu_ring *ring);
int (*test_ib)(struct amdgpu_ring *ring);
bool (*is_lockup)(struct amdgpu_ring *ring);
+ /* insert NOP packets */
+ void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
};
/*
@@ -381,10 +394,10 @@ struct amdgpu_fence_driver {
uint64_t sync_seq[AMDGPU_MAX_RINGS];
atomic64_t last_seq;
bool initialized;
- bool delayed_irq;
struct amdgpu_irq_src *irq_src;
unsigned irq_type;
struct delayed_work lockup_work;
+ wait_queue_head_t fence_queue;
};
/* some special values for the owner field */
@@ -423,20 +436,20 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring);
int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
struct amdgpu_irq_src *irq_src,
unsigned irq_type);
+void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
+void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,
struct amdgpu_fence **fence);
-int amdgpu_fence_recreate(struct amdgpu_ring *ring, void *owner,
- uint64_t seq, struct amdgpu_fence **fence);
void amdgpu_fence_process(struct amdgpu_ring *ring);
int amdgpu_fence_wait_next(struct amdgpu_ring *ring);
int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
-bool amdgpu_fence_signaled(struct amdgpu_fence *fence);
-int amdgpu_fence_wait(struct amdgpu_fence *fence, bool interruptible);
-int amdgpu_fence_wait_any(struct amdgpu_device *adev,
- struct amdgpu_fence **fences,
- bool intr);
+signed long amdgpu_fence_wait_any(struct amdgpu_device *adev,
+ struct fence **array,
+ uint32_t count,
+ bool intr,
+ signed long t);
struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence);
void amdgpu_fence_unref(struct amdgpu_fence **fence);
@@ -481,7 +494,7 @@ static inline bool amdgpu_fence_is_earlier(struct amdgpu_fence *a,
return a->seq < b->seq;
}
-int amdgpu_user_fence_emit(struct amdgpu_ring *ring, struct amdgpu_user_fence *user,
+int amdgpu_user_fence_emit(struct amdgpu_ring *ring, struct amdgpu_user_fence *user,
void *owner, struct amdgpu_fence **fence);
/*
@@ -509,7 +522,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
uint64_t dst_offset,
uint32_t byte_count,
struct reservation_object *resv,
- struct amdgpu_fence **fence);
+ struct fence **fence);
int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
struct amdgpu_bo_list_entry {
@@ -532,14 +545,16 @@ struct amdgpu_bo_va_mapping {
struct amdgpu_bo_va {
/* protected by bo being reserved */
struct list_head bo_list;
- uint64_t addr;
- struct amdgpu_fence *last_pt_update;
+ struct fence *last_pt_update;
unsigned ref_count;
- /* protected by vm mutex */
- struct list_head mappings;
+ /* protected by vm mutex and spinlock */
struct list_head vm_status;
+ /* mappings for this bo_va */
+ struct list_head invalids;
+ struct list_head valids;
+
/* constant after initialization */
struct amdgpu_vm *vm;
struct amdgpu_bo *bo;
@@ -643,7 +658,7 @@ struct amdgpu_sa_bo {
struct amdgpu_sa_manager *manager;
unsigned soffset;
unsigned eoffset;
- struct amdgpu_fence *fence;
+ struct fence *fence;
};
/*
@@ -685,7 +700,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring,
struct amdgpu_semaphore *semaphore);
void amdgpu_semaphore_free(struct amdgpu_device *adev,
struct amdgpu_semaphore **semaphore,
- struct amdgpu_fence *fence);
+ struct fence *fence);
/*
* Synchronization
@@ -693,20 +708,23 @@ void amdgpu_semaphore_free(struct amdgpu_device *adev,
struct amdgpu_sync {
struct amdgpu_semaphore *semaphores[AMDGPU_NUM_SYNCS];
struct amdgpu_fence *sync_to[AMDGPU_MAX_RINGS];
- struct amdgpu_fence *last_vm_update;
+ DECLARE_HASHTABLE(fences, 4);
+ struct fence *last_vm_update;
};
void amdgpu_sync_create(struct amdgpu_sync *sync);
-void amdgpu_sync_fence(struct amdgpu_sync *sync,
- struct amdgpu_fence *fence);
+int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
+ struct fence *f);
int amdgpu_sync_resv(struct amdgpu_device *adev,
struct amdgpu_sync *sync,
struct reservation_object *resv,
void *owner);
int amdgpu_sync_rings(struct amdgpu_sync *sync,
struct amdgpu_ring *ring);
+struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
+int amdgpu_sync_wait(struct amdgpu_sync *sync);
void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync,
- struct amdgpu_fence *fence);
+ struct fence *fence);
/*
* GART structures, functions & helpers
@@ -821,7 +839,9 @@ struct amdgpu_flip_work {
uint64_t base;
struct drm_pending_vblank_event *event;
struct amdgpu_bo *old_rbo;
- struct fence *fence;
+ struct fence *excl;
+ unsigned shared_count;
+ struct fence **shared;
};
@@ -844,6 +864,8 @@ struct amdgpu_ib {
uint32_t gws_base, gws_size;
uint32_t oa_base, oa_size;
uint32_t flags;
+ /* resulting sequence number */
+ uint64_t sequence;
};
enum amdgpu_ring_type {
@@ -854,11 +876,23 @@ enum amdgpu_ring_type {
AMDGPU_RING_TYPE_VCE
};
+extern struct amd_sched_backend_ops amdgpu_sched_ops;
+
+int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring,
+ struct amdgpu_ib *ibs,
+ unsigned num_ibs,
+ int (*free_job)(struct amdgpu_job *),
+ void *owner,
+ struct fence **fence);
+
struct amdgpu_ring {
struct amdgpu_device *adev;
const struct amdgpu_ring_funcs *funcs;
struct amdgpu_fence_driver fence_drv;
+ struct amd_gpu_scheduler *scheduler;
+ spinlock_t fence_lock;
struct mutex *ring_lock;
struct amdgpu_bo *ring_obj;
volatile uint32_t *ring;
@@ -892,6 +926,7 @@ struct amdgpu_ring {
struct amdgpu_ctx *current_ctx;
enum amdgpu_ring_type type;
char name[16];
+ bool is_pte_ring;
};
/*
@@ -933,7 +968,7 @@ struct amdgpu_vm_id {
unsigned id;
uint64_t pd_gpu_addr;
/* last flushed PD/PT update */
- struct amdgpu_fence *flushed_updates;
+ struct fence *flushed_updates;
/* last use of vmid */
struct amdgpu_fence *last_id_use;
};
@@ -943,18 +978,22 @@ struct amdgpu_vm {
struct rb_root va;
- /* protecting invalidated and freed */
+ /* protecting invalidated */
spinlock_t status_lock;
/* BOs moved, but not yet updated in the PT */
struct list_head invalidated;
- /* BOs freed, but not yet updated in the PT */
+ /* BOs cleared in the PT because of a move */
+ struct list_head cleared;
+
+ /* BO mappings freed, but not yet updated in the PT */
struct list_head freed;
/* contains the page directory */
struct amdgpu_bo *page_directory;
unsigned max_pde_used;
+ struct fence *page_directory_fence;
/* array of page tables, one for each page directory entry */
struct amdgpu_vm_pt *page_tables;
@@ -983,27 +1022,47 @@ struct amdgpu_vm_manager {
* context related structures
*/
-struct amdgpu_ctx_state {
- uint64_t flags;
- uint32_t hangs;
+#define AMDGPU_CTX_MAX_CS_PENDING 16
+
+struct amdgpu_ctx_ring {
+ uint64_t sequence;
+ struct fence *fences[AMDGPU_CTX_MAX_CS_PENDING];
+ struct amd_sched_entity entity;
};
struct amdgpu_ctx {
- /* call kref_get()before CS start and kref_put() after CS fence signaled */
- struct kref refcount;
- struct amdgpu_fpriv *fpriv;
- struct amdgpu_ctx_state state;
- uint32_t id;
- unsigned reset_counter;
+ struct kref refcount;
+ struct amdgpu_device *adev;
+ unsigned reset_counter;
+ spinlock_t ring_lock;
+ struct amdgpu_ctx_ring rings[AMDGPU_MAX_RINGS];
};
struct amdgpu_ctx_mgr {
- struct amdgpu_device *adev;
- struct idr ctx_handles;
- /* lock for IDR system */
- struct mutex lock;
+ struct amdgpu_device *adev;
+ struct mutex lock;
+ /* protected by lock */
+ struct idr ctx_handles;
};
+int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
+ struct amdgpu_ctx *ctx);
+void amdgpu_ctx_fini(struct amdgpu_ctx *ctx);
+
+struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
+int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
+
+uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+ struct fence *fence);
+struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
+ struct amdgpu_ring *ring, uint64_t seq);
+
+int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+
+void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
+void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
+
/*
* file private structure
*/
@@ -1012,7 +1071,7 @@ struct amdgpu_fpriv {
struct amdgpu_vm vm;
struct mutex bo_list_lock;
struct idr bo_list_handles;
- struct amdgpu_ctx_mgr ctx_mgr;
+ struct amdgpu_ctx_mgr ctx_mgr;
};
/*
@@ -1160,6 +1219,7 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
void amdgpu_ring_free_size(struct amdgpu_ring *ring);
int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw);
+void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
void amdgpu_ring_commit(struct amdgpu_ring *ring);
void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring);
void amdgpu_ring_undo(struct amdgpu_ring *ring);
@@ -1207,6 +1267,16 @@ struct amdgpu_cs_parser {
struct amdgpu_user_fence uf;
};
+struct amdgpu_job {
+ struct amd_sched_job base;
+ struct amdgpu_device *adev;
+ struct amdgpu_ib *ibs;
+ uint32_t num_ibs;
+ struct mutex job_lock;
+ struct amdgpu_user_fence uf;
+ int (*free_job)(struct amdgpu_job *sched_job);
+};
+
static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx)
{
return p->ibs[ib_idx].ptr[idx];
@@ -1601,7 +1671,6 @@ struct amdgpu_uvd {
struct amdgpu_bo *vcpu_bo;
void *cpu_addr;
uint64_t gpu_addr;
- void *saved_bo;
atomic_t handles[AMDGPU_MAX_UVD_HANDLES];
struct drm_file *filp[AMDGPU_MAX_UVD_HANDLES];
struct delayed_work idle_work;
@@ -1645,6 +1714,7 @@ struct amdgpu_sdma {
uint32_t feature_version;
struct amdgpu_ring ring;
+ bool burst_nop;
};
/*
@@ -1849,17 +1919,12 @@ struct amdgpu_atcs {
struct amdgpu_atcs_functions functions;
};
-int amdgpu_ctx_alloc(struct amdgpu_device *adev,struct amdgpu_fpriv *fpriv,
- uint32_t *id,uint32_t flags);
-int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv,
- uint32_t id);
-
-void amdgpu_ctx_fini(struct amdgpu_fpriv *fpriv);
-struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
-int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
+/*
+ * CGS
+ */
+void *amdgpu_cgs_create_device(struct amdgpu_device *adev);
+void amdgpu_cgs_destroy_device(void *cgs_device);
-extern int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
- struct drm_file *filp);
/*
* Core structure, functions and helpers.
@@ -1883,7 +1948,7 @@ struct amdgpu_device {
struct rw_semaphore exclusive_lock;
/* ASIC */
- enum amdgpu_asic_type asic_type;
+ enum amd_asic_type asic_type;
uint32_t family;
uint32_t rev_id;
uint32_t external_rev_id;
@@ -1976,7 +2041,6 @@ struct amdgpu_device {
struct amdgpu_irq_src hpd_irq;
/* rings */
- wait_queue_head_t fence_queue;
unsigned fence_context;
struct mutex ring_lock;
unsigned num_rings;
@@ -1999,7 +2063,7 @@ struct amdgpu_device {
struct amdgpu_gfx gfx;
/* sdma */
- struct amdgpu_sdma sdma[2];
+ struct amdgpu_sdma sdma[AMDGPU_MAX_SDMA_INSTANCES];
struct amdgpu_irq_src sdma_trap_irq;
struct amdgpu_irq_src sdma_illegal_inst_irq;
@@ -2025,6 +2089,12 @@ struct amdgpu_device {
/* tracking pinned memory */
u64 vram_pin_size;
u64 gart_pin_size;
+
+ /* amdkfd interface */
+ struct kfd_dev *kfd;
+
+ /* kernel conext for IB submission */
+ struct amdgpu_ctx kernel_ctx;
};
bool amdgpu_device_is_px(struct drm_device *dev);
@@ -2132,6 +2202,21 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
ring->ring_free_dw--;
}
+static inline struct amdgpu_sdma * amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
+{
+ struct amdgpu_device *adev = ring->adev;
+ int i;
+
+ for (i = 0; i < AMDGPU_MAX_SDMA_INSTANCES; i++)
+ if (&adev->sdma[i].ring == ring)
+ break;
+
+ if (i < AMDGPU_MAX_SDMA_INSTANCES)
+ return &adev->sdma[i];
+ else
+ return NULL;
+}
+
/*
* ASICs macro.
*/
@@ -2183,8 +2268,8 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
#define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r))
#define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s))
#define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))
-#define amdgpu_emit_copy_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((r), (s), (d), (b))
-#define amdgpu_emit_fill_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((r), (s), (d), (b))
+#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b))
+#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
#define amdgpu_dpm_get_temperature(adev) (adev)->pm.funcs->get_temperature((adev))
#define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
#define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
@@ -2212,6 +2297,12 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev);
bool amdgpu_card_posted(struct amdgpu_device *adev);
void amdgpu_update_display_priority(struct amdgpu_device *adev);
bool amdgpu_boot_test_post_card(struct amdgpu_device *adev);
+struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev,
+ struct drm_file *filp,
+ struct amdgpu_ctx *ctx,
+ struct amdgpu_ib *ibs,
+ uint32_t num_ibs);
+
int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data);
int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
u32 ip_instance, u32 ring,
@@ -2275,11 +2366,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct list_head *head);
-struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
- struct amdgpu_vm *vm);
+int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
+ struct amdgpu_sync *sync);
void amdgpu_vm_flush(struct amdgpu_ring *ring,
struct amdgpu_vm *vm,
- struct amdgpu_fence *updates);
+ struct fence *updates);
void amdgpu_vm_fence(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_fence *fence);
@@ -2309,7 +2400,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
uint64_t addr);
void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va);
-
+int amdgpu_vm_free_job(struct amdgpu_job *job);
/*
* functions used by amdgpu_encoder.c
*/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
new file mode 100644
index 000000000000..496ed2192eba
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu_amdkfd.h"
+#include "amd_shared.h"
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include <linux/module.h>
+
+const struct kfd2kgd_calls *kfd2kgd;
+const struct kgd2kfd_calls *kgd2kfd;
+bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+
+bool amdgpu_amdkfd_init(void)
+{
+#if defined(CONFIG_HSA_AMD_MODULE)
+ bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+
+ kgd2kfd_init_p = symbol_request(kgd2kfd_init);
+
+ if (kgd2kfd_init_p == NULL)
+ return false;
+#endif
+ return true;
+}
+
+bool amdgpu_amdkfd_load_interface(struct amdgpu_device *rdev)
+{
+#if defined(CONFIG_HSA_AMD_MODULE)
+ bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+#endif
+
+ switch (rdev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_CIK
+ case CHIP_KAVERI:
+ kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions();
+ break;
+#endif
+ case CHIP_CARRIZO:
+ kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
+ break;
+ default:
+ return false;
+ }
+
+#if defined(CONFIG_HSA_AMD_MODULE)
+ kgd2kfd_init_p = symbol_request(kgd2kfd_init);
+
+ if (kgd2kfd_init_p == NULL) {
+ kfd2kgd = NULL;
+ return false;
+ }
+
+ if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd)) {
+ symbol_put(kgd2kfd_init);
+ kfd2kgd = NULL;
+ kgd2kfd = NULL;
+
+ return false;
+ }
+
+ return true;
+#elif defined(CONFIG_HSA_AMD)
+ if (!kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd)) {
+ kfd2kgd = NULL;
+ kgd2kfd = NULL;
+ return false;
+ }
+
+ return true;
+#else
+ kfd2kgd = NULL;
+ return false;
+#endif
+}
+
+void amdgpu_amdkfd_fini(void)
+{
+ if (kgd2kfd) {
+ kgd2kfd->exit();
+ symbol_put(kgd2kfd_init);
+ }
+}
+
+void amdgpu_amdkfd_device_probe(struct amdgpu_device *rdev)
+{
+ if (kgd2kfd)
+ rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev,
+ rdev->pdev, kfd2kgd);
+}
+
+void amdgpu_amdkfd_device_init(struct amdgpu_device *rdev)
+{
+ if (rdev->kfd) {
+ struct kgd2kfd_shared_resources gpu_resources = {
+ .compute_vmid_bitmap = 0xFF00,
+
+ .first_compute_pipe = 1,
+ .compute_pipe_count = 4 - 1,
+ };
+
+ amdgpu_doorbell_get_kfd_info(rdev,
+ &gpu_resources.doorbell_physical_address,
+ &gpu_resources.doorbell_aperture_size,
+ &gpu_resources.doorbell_start_offset);
+
+ kgd2kfd->device_init(rdev->kfd, &gpu_resources);
+ }
+}
+
+void amdgpu_amdkfd_device_fini(struct amdgpu_device *rdev)
+{
+ if (rdev->kfd) {
+ kgd2kfd->device_exit(rdev->kfd);
+ rdev->kfd = NULL;
+ }
+}
+
+void amdgpu_amdkfd_interrupt(struct amdgpu_device *rdev,
+ const void *ih_ring_entry)
+{
+ if (rdev->kfd)
+ kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
+}
+
+void amdgpu_amdkfd_suspend(struct amdgpu_device *rdev)
+{
+ if (rdev->kfd)
+ kgd2kfd->suspend(rdev->kfd);
+}
+
+int amdgpu_amdkfd_resume(struct amdgpu_device *rdev)
+{
+ int r = 0;
+
+ if (rdev->kfd)
+ r = kgd2kfd->resume(rdev->kfd);
+
+ return r;
+}
+
+u32 pool_to_domain(enum kgd_memory_pool p)
+{
+ switch (p) {
+ case KGD_POOL_FRAMEBUFFER: return AMDGPU_GEM_DOMAIN_VRAM;
+ default: return AMDGPU_GEM_DOMAIN_GTT;
+ }
+}
+
+int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
+ void **mem_obj, uint64_t *gpu_addr,
+ void **cpu_ptr)
+{
+ struct amdgpu_device *rdev = (struct amdgpu_device *)kgd;
+ struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
+ int r;
+
+ BUG_ON(kgd == NULL);
+ BUG_ON(gpu_addr == NULL);
+ BUG_ON(cpu_ptr == NULL);
+
+ *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
+ if ((*mem) == NULL)
+ return -ENOMEM;
+
+ r = amdgpu_bo_create(rdev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
+ AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, &(*mem)->bo);
+ if (r) {
+ dev_err(rdev->dev,
+ "failed to allocate BO for amdkfd (%d)\n", r);
+ return r;
+ }
+
+ /* map the buffer */
+ r = amdgpu_bo_reserve((*mem)->bo, true);
+ if (r) {
+ dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
+ goto allocate_mem_reserve_bo_failed;
+ }
+
+ r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT,
+ &(*mem)->gpu_addr);
+ if (r) {
+ dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r);
+ goto allocate_mem_pin_bo_failed;
+ }
+ *gpu_addr = (*mem)->gpu_addr;
+
+ r = amdgpu_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
+ if (r) {
+ dev_err(rdev->dev,
+ "(%d) failed to map bo to kernel for amdkfd\n", r);
+ goto allocate_mem_kmap_bo_failed;
+ }
+ *cpu_ptr = (*mem)->cpu_ptr;
+
+ amdgpu_bo_unreserve((*mem)->bo);
+
+ return 0;
+
+allocate_mem_kmap_bo_failed:
+ amdgpu_bo_unpin((*mem)->bo);
+allocate_mem_pin_bo_failed:
+ amdgpu_bo_unreserve((*mem)->bo);
+allocate_mem_reserve_bo_failed:
+ amdgpu_bo_unref(&(*mem)->bo);
+
+ return r;
+}
+
+void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
+{
+ struct kgd_mem *mem = (struct kgd_mem *) mem_obj;
+
+ BUG_ON(mem == NULL);
+
+ amdgpu_bo_reserve(mem->bo, true);
+ amdgpu_bo_kunmap(mem->bo);
+ amdgpu_bo_unpin(mem->bo);
+ amdgpu_bo_unreserve(mem->bo);
+ amdgpu_bo_unref(&(mem->bo));
+ kfree(mem);
+}
+
+uint64_t get_vmem_size(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *rdev =
+ (struct amdgpu_device *)kgd;
+
+ BUG_ON(kgd == NULL);
+
+ return rdev->mc.real_vram_size;
+}
+
+uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *rdev = (struct amdgpu_device *)kgd;
+
+ if (rdev->asic_funcs->get_gpu_clock_counter)
+ return rdev->asic_funcs->get_gpu_clock_counter(rdev);
+ return 0;
+}
+
+uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *rdev = (struct amdgpu_device *)kgd;
+
+ /* The sclk is in quantas of 10kHz */
+ return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
new file mode 100644
index 000000000000..a8be765542e6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
+
+#ifndef AMDGPU_AMDKFD_H_INCLUDED
+#define AMDGPU_AMDKFD_H_INCLUDED
+
+#include <linux/types.h>
+#include <kgd_kfd_interface.h>
+
+struct amdgpu_device;
+
+struct kgd_mem {
+ struct amdgpu_bo *bo;
+ uint64_t gpu_addr;
+ void *cpu_ptr;
+};
+
+bool amdgpu_amdkfd_init(void);
+void amdgpu_amdkfd_fini(void);
+
+bool amdgpu_amdkfd_load_interface(struct amdgpu_device *rdev);
+
+void amdgpu_amdkfd_suspend(struct amdgpu_device *rdev);
+int amdgpu_amdkfd_resume(struct amdgpu_device *rdev);
+void amdgpu_amdkfd_interrupt(struct amdgpu_device *rdev,
+ const void *ih_ring_entry);
+void amdgpu_amdkfd_device_probe(struct amdgpu_device *rdev);
+void amdgpu_amdkfd_device_init(struct amdgpu_device *rdev);
+void amdgpu_amdkfd_device_fini(struct amdgpu_device *rdev);
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
+
+/* Shared API */
+int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
+ void **mem_obj, uint64_t *gpu_addr,
+ void **cpu_ptr);
+void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
+uint64_t get_vmem_size(struct kgd_dev *kgd);
+uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
+
+uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
+
+#endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
new file mode 100644
index 000000000000..dd2037bc0b4a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -0,0 +1,670 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/fdtable.h>
+#include <linux/uaccess.h>
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+#include "cikd.h"
+#include "cik_sdma.h"
+#include "amdgpu_ucode.h"
+#include "gca/gfx_7_2_d.h"
+#include "gca/gfx_7_2_enum.h"
+#include "gca/gfx_7_2_sh_mask.h"
+#include "oss/oss_2_0_d.h"
+#include "oss/oss_2_0_sh_mask.h"
+#include "gmc/gmc_7_1_d.h"
+#include "gmc/gmc_7_1_sh_mask.h"
+#include "cik_structs.h"
+
+#define CIK_PIPE_PER_MEC (4)
+
+enum {
+ MAX_TRAPID = 8, /* 3 bits in the bitfield. */
+ MAX_WATCH_ADDRESSES = 4
+};
+
+enum {
+ ADDRESS_WATCH_REG_ADDR_HI = 0,
+ ADDRESS_WATCH_REG_ADDR_LO,
+ ADDRESS_WATCH_REG_CNTL,
+ ADDRESS_WATCH_REG_MAX
+};
+
+/* not defined in the CI/KV reg file */
+enum {
+ ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
+ ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
+ ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
+ /* extend the mask to 26 bits to match the low address field */
+ ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
+ ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
+};
+
+static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
+ mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL,
+ mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL,
+ mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL,
+ mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL
+};
+
+union TCP_WATCH_CNTL_BITS {
+ struct {
+ uint32_t mask:24;
+ uint32_t vmid:4;
+ uint32_t atc:1;
+ uint32_t mode:2;
+ uint32_t valid:1;
+ } bitfields, bits;
+ uint32_t u32All;
+ signed int i32All;
+ float f32All;
+};
+
+/*
+ * Register access functions
+ */
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
+ uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
+
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid);
+
+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+
+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout);
+static int kgd_address_watch_disable(struct kgd_dev *kgd);
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo);
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd);
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset);
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid);
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid);
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+
+static const struct kfd2kgd_calls kfd2kgd = {
+ .init_gtt_mem_allocation = alloc_gtt_mem,
+ .free_gtt_mem = free_gtt_mem,
+ .get_vmem_size = get_vmem_size,
+ .get_gpu_clock_counter = get_gpu_clock_counter,
+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+ .init_pipeline = kgd_init_pipeline,
+ .init_interrupts = kgd_init_interrupts,
+ .hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
+ .hqd_is_occupied = kgd_hqd_is_occupied,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+ .hqd_destroy = kgd_hqd_destroy,
+ .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
+ .address_watch_disable = kgd_address_watch_disable,
+ .address_watch_execute = kgd_address_watch_execute,
+ .wave_control_execute = kgd_wave_control_execute,
+ .address_watch_get_offset = kgd_address_watch_get_offset,
+ .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
+ .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
+ .write_vmid_invalidate_request = write_vmid_invalidate_request,
+ .get_fw_version = get_fw_version
+};
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions()
+{
+ return (struct kfd2kgd_calls *)&kfd2kgd;
+}
+
+static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+{
+ return (struct amdgpu_device *)kgd;
+}
+
+static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
+ uint32_t queue, uint32_t vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
+
+ mutex_lock(&adev->srbm_mutex);
+ WREG32(mmSRBM_GFX_CNTL, value);
+}
+
+static void unlock_srbm(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ WREG32(mmSRBM_GFX_CNTL, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
+ uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
+
+ lock_srbm(kgd, mec, pipe, queue_id, 0);
+}
+
+static void release_queue(struct kgd_dev *kgd)
+{
+ unlock_srbm(kgd);
+}
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config,
+ uint32_t sh_mem_ape1_base,
+ uint32_t sh_mem_ape1_limit,
+ uint32_t sh_mem_bases)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ lock_srbm(kgd, 0, 0, 0, vmid);
+
+ WREG32(mmSH_MEM_CONFIG, sh_mem_config);
+ WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base);
+ WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
+ WREG32(mmSH_MEM_BASES, sh_mem_bases);
+
+ unlock_srbm(kgd);
+}
+
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ /*
+ * We have to assume that there is no outstanding mapping.
+ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
+ * a mapping is in progress or because a mapping finished and the
+ * SW cleared it. So the protocol is to always wait & clear.
+ */
+ uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
+ ATC_VMID0_PASID_MAPPING__VALID_MASK;
+
+ WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping);
+
+ while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid)))
+ cpu_relax();
+ WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
+
+ /* Mapping vmid to pasid also for IH block */
+ WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping);
+
+ return 0;
+}
+
+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
+ uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
+
+ lock_srbm(kgd, mec, pipe, 0, 0);
+ WREG32(mmCP_HPD_EOP_BASE_ADDR, lower_32_bits(hpd_gpu_addr >> 8));
+ WREG32(mmCP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(hpd_gpu_addr >> 8));
+ WREG32(mmCP_HPD_EOP_VMID, 0);
+ WREG32(mmCP_HPD_EOP_CONTROL, hpd_size);
+ unlock_srbm(kgd);
+
+ return 0;
+}
+
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t mec;
+ uint32_t pipe;
+
+ mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;
+ pipe = (pipe_id % CIK_PIPE_PER_MEC);
+
+ lock_srbm(kgd, mec, pipe, 0, 0);
+
+ WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK |
+ CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK);
+
+ unlock_srbm(kgd);
+
+ return 0;
+}
+
+static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+{
+ uint32_t retval;
+
+ retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+ m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+
+ pr_debug("kfd: sdma base address: 0x%x\n", retval);
+
+ return retval;
+}
+
+static inline struct cik_mqd *get_mqd(void *mqd)
+{
+ return (struct cik_mqd *)mqd;
+}
+
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+ return (struct cik_sdma_rlc_registers *)mqd;
+}
+
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t wptr_shadow, is_wptr_shadow_valid;
+ struct cik_mqd *m;
+
+ m = get_mqd(mqd);
+
+ is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ WREG32(mmCP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
+ WREG32(mmCP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
+ WREG32(mmCP_MQD_CONTROL, m->cp_mqd_control);
+
+ WREG32(mmCP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
+ WREG32(mmCP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
+ WREG32(mmCP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
+
+ WREG32(mmCP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
+ WREG32(mmCP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
+ WREG32(mmCP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
+
+ WREG32(mmCP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
+
+ WREG32(mmCP_HQD_PERSISTENT_STATE, m->cp_hqd_persistent_state);
+ WREG32(mmCP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
+ WREG32(mmCP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
+
+ WREG32(mmCP_HQD_ATOMIC0_PREOP_LO, m->cp_hqd_atomic0_preop_lo);
+ WREG32(mmCP_HQD_ATOMIC0_PREOP_HI, m->cp_hqd_atomic0_preop_hi);
+ WREG32(mmCP_HQD_ATOMIC1_PREOP_LO, m->cp_hqd_atomic1_preop_lo);
+ WREG32(mmCP_HQD_ATOMIC1_PREOP_HI, m->cp_hqd_atomic1_preop_hi);
+
+ WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR, m->cp_hqd_pq_rptr_report_addr_lo);
+ WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+ m->cp_hqd_pq_rptr_report_addr_hi);
+
+ WREG32(mmCP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
+
+ WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, m->cp_hqd_pq_wptr_poll_addr_lo);
+ WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, m->cp_hqd_pq_wptr_poll_addr_hi);
+
+ WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, m->cp_hqd_pq_doorbell_control);
+
+ WREG32(mmCP_HQD_VMID, m->cp_hqd_vmid);
+
+ WREG32(mmCP_HQD_QUANTUM, m->cp_hqd_quantum);
+
+ WREG32(mmCP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
+ WREG32(mmCP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
+
+ WREG32(mmCP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
+
+ if (is_wptr_shadow_valid)
+ WREG32(mmCP_HQD_PQ_WPTR, wptr_shadow);
+
+ WREG32(mmCP_HQD_ACTIVE, m->cp_hqd_active);
+ release_queue(kgd);
+
+ return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
+ m->sdma_rlc_virtual_addr);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE,
+ m->sdma_rlc_rb_base);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+ m->sdma_rlc_rb_base_hi);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+ m->sdma_rlc_rb_rptr_addr_lo);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+ m->sdma_rlc_rb_rptr_addr_hi);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL,
+ m->sdma_rlc_doorbell);
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+ m->sdma_rlc_rb_cntl);
+
+ return 0;
+}
+
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t act;
+ bool retval = false;
+ uint32_t low, high;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ act = RREG32(mmCP_HQD_ACTIVE);
+ if (act) {
+ low = lower_32_bits(queue_address >> 8);
+ high = upper_32_bits(queue_address >> 8);
+
+ if (low == RREG32(mmCP_HQD_PQ_BASE) &&
+ high == RREG32(mmCP_HQD_PQ_BASE_HI))
+ retval = true;
+ }
+ release_queue(kgd);
+ return retval;
+}
+
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t sdma_rlc_rb_cntl;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+
+ if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
+ return true;
+
+ return false;
+}
+
+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t temp;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
+
+ WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+
+ while (true) {
+ temp = RREG32(mmCP_HQD_ACTIVE);
+ if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+ break;
+ if (timeout == 0) {
+ pr_err("kfd: cp queue preemption time out (%dms)\n",
+ temp);
+ release_queue(kgd);
+ return -ETIME;
+ }
+ msleep(20);
+ timeout -= 20;
+ }
+
+ release_queue(kgd);
+ return 0;
+}
+
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+ temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
+
+ while (true) {
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+ if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
+ break;
+ if (timeout == 0)
+ return -ETIME;
+ msleep(20);
+ timeout -= 20;
+ }
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0);
+
+ return 0;
+}
+
+static int kgd_address_watch_disable(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ union TCP_WATCH_CNTL_BITS cntl;
+ unsigned int i;
+
+ cntl.u32All = 0;
+
+ cntl.bitfields.valid = 0;
+ cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
+ cntl.bitfields.atc = 1;
+
+ /* Turning off this address until we set all the registers */
+ for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
+ WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX +
+ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+ return 0;
+}
+
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ union TCP_WATCH_CNTL_BITS cntl;
+
+ cntl.u32All = cntl_val;
+
+ /* Turning off this watch point until we set all the registers */
+ cntl.bitfields.valid = 0;
+ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
+ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
+ ADDRESS_WATCH_REG_ADDR_HI], addr_hi);
+
+ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
+ ADDRESS_WATCH_REG_ADDR_LO], addr_lo);
+
+ /* Enable the watch point */
+ cntl.bitfields.valid = 1;
+
+ WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
+ ADDRESS_WATCH_REG_CNTL], cntl.u32All);
+
+ return 0;
+}
+
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t data;
+
+ mutex_lock(&adev->grbm_idx_mutex);
+
+ WREG32(mmGRBM_GFX_INDEX, gfx_index_val);
+ WREG32(mmSQ_CMD, sq_cmd);
+
+ /* Restore the GRBM_GFX_INDEX register */
+
+ data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK |
+ GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
+ GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
+
+ WREG32(mmGRBM_GFX_INDEX, data);
+
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+ return 0;
+}
+
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset)
+{
+ return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
+}
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+}
+
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ const union amdgpu_firmware_header *hdr;
+
+ BUG_ON(kgd == NULL);
+
+ switch (type) {
+ case KGD_ENGINE_PFP:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.pfp_fw->data;
+ break;
+
+ case KGD_ENGINE_ME:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.me_fw->data;
+ break;
+
+ case KGD_ENGINE_CE:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.ce_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC1:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.mec_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC2:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.mec2_fw->data;
+ break;
+
+ case KGD_ENGINE_RLC:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.rlc_fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA1:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->sdma[0].fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA2:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->sdma[1].fw->data;
+ break;
+
+ default:
+ return 0;
+ }
+
+ if (hdr == NULL)
+ return 0;
+
+ /* Only 12 bit in use*/
+ return hdr->common.ucode_version;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
new file mode 100644
index 000000000000..dfd1d503bccf
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -0,0 +1,543 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/fdtable.h>
+#include <linux/uaccess.h>
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_amdkfd.h"
+#include "amdgpu_ucode.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_enum.h"
+#include "oss/oss_3_0_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "gmc/gmc_8_1_sh_mask.h"
+#include "gmc/gmc_8_1_d.h"
+#include "vi_structs.h"
+#include "vid.h"
+
+#define VI_PIPE_PER_MEC (4)
+
+struct cik_sdma_rlc_registers;
+
+/*
+ * Register access functions
+ */
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config,
+ uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
+ uint32_t sh_mem_bases);
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid);
+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr);
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout);
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+static int kgd_address_watch_disable(struct kgd_dev *kgd);
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo);
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd);
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset);
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ uint8_t vmid);
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid);
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+
+static const struct kfd2kgd_calls kfd2kgd = {
+ .init_gtt_mem_allocation = alloc_gtt_mem,
+ .free_gtt_mem = free_gtt_mem,
+ .get_vmem_size = get_vmem_size,
+ .get_gpu_clock_counter = get_gpu_clock_counter,
+ .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+ .program_sh_mem_settings = kgd_program_sh_mem_settings,
+ .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
+ .init_pipeline = kgd_init_pipeline,
+ .init_interrupts = kgd_init_interrupts,
+ .hqd_load = kgd_hqd_load,
+ .hqd_sdma_load = kgd_hqd_sdma_load,
+ .hqd_is_occupied = kgd_hqd_is_occupied,
+ .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
+ .hqd_destroy = kgd_hqd_destroy,
+ .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
+ .address_watch_disable = kgd_address_watch_disable,
+ .address_watch_execute = kgd_address_watch_execute,
+ .wave_control_execute = kgd_wave_control_execute,
+ .address_watch_get_offset = kgd_address_watch_get_offset,
+ .get_atc_vmid_pasid_mapping_pasid =
+ get_atc_vmid_pasid_mapping_pasid,
+ .get_atc_vmid_pasid_mapping_valid =
+ get_atc_vmid_pasid_mapping_valid,
+ .write_vmid_invalidate_request = write_vmid_invalidate_request,
+ .get_fw_version = get_fw_version
+};
+
+struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions()
+{
+ return (struct kfd2kgd_calls *)&kfd2kgd;
+}
+
+static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
+{
+ return (struct amdgpu_device *)kgd;
+}
+
+static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
+ uint32_t queue, uint32_t vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
+
+ mutex_lock(&adev->srbm_mutex);
+ WREG32(mmSRBM_GFX_CNTL, value);
+}
+
+static void unlock_srbm(struct kgd_dev *kgd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ WREG32(mmSRBM_GFX_CNTL, 0);
+ mutex_unlock(&adev->srbm_mutex);
+}
+
+static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ uint32_t mec = (++pipe_id / VI_PIPE_PER_MEC) + 1;
+ uint32_t pipe = (pipe_id % VI_PIPE_PER_MEC);
+
+ lock_srbm(kgd, mec, pipe, queue_id, 0);
+}
+
+static void release_queue(struct kgd_dev *kgd)
+{
+ unlock_srbm(kgd);
+}
+
+static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
+ uint32_t sh_mem_config,
+ uint32_t sh_mem_ape1_base,
+ uint32_t sh_mem_ape1_limit,
+ uint32_t sh_mem_bases)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ lock_srbm(kgd, 0, 0, 0, vmid);
+
+ WREG32(mmSH_MEM_CONFIG, sh_mem_config);
+ WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base);
+ WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
+ WREG32(mmSH_MEM_BASES, sh_mem_bases);
+
+ unlock_srbm(kgd);
+}
+
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+ unsigned int vmid)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ /*
+ * We have to assume that there is no outstanding mapping.
+ * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because
+ * a mapping is in progress or because a mapping finished
+ * and the SW cleared it.
+ * So the protocol is to always wait & clear.
+ */
+ uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
+ ATC_VMID0_PASID_MAPPING__VALID_MASK;
+
+ WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping);
+
+ while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid)))
+ cpu_relax();
+ WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
+
+ /* Mapping vmid to pasid also for IH block */
+ WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping);
+
+ return 0;
+}
+
+static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
+ uint32_t hpd_size, uint64_t hpd_gpu_addr)
+{
+ return 0;
+}
+
+static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t mec;
+ uint32_t pipe;
+
+ mec = (++pipe_id / VI_PIPE_PER_MEC) + 1;
+ pipe = (pipe_id % VI_PIPE_PER_MEC);
+
+ lock_srbm(kgd, mec, pipe, 0, 0);
+
+ WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK);
+
+ unlock_srbm(kgd);
+
+ return 0;
+}
+
+static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+{
+ return 0;
+}
+
+static inline struct vi_mqd *get_mqd(void *mqd)
+{
+ return (struct vi_mqd *)mqd;
+}
+
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+ return (struct cik_sdma_rlc_registers *)mqd;
+}
+
+static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
+ uint32_t queue_id, uint32_t __user *wptr)
+{
+ struct vi_mqd *m;
+ uint32_t shadow_wptr, valid_wptr;
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+
+ m = get_mqd(mqd);
+
+ valid_wptr = copy_from_user(&shadow_wptr, wptr, sizeof(shadow_wptr));
+ acquire_queue(kgd, pipe_id, queue_id);
+
+ WREG32(mmCP_MQD_CONTROL, m->cp_mqd_control);
+ WREG32(mmCP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
+ WREG32(mmCP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
+
+ WREG32(mmCP_HQD_VMID, m->cp_hqd_vmid);
+ WREG32(mmCP_HQD_PERSISTENT_STATE, m->cp_hqd_persistent_state);
+ WREG32(mmCP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
+ WREG32(mmCP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
+ WREG32(mmCP_HQD_QUANTUM, m->cp_hqd_quantum);
+ WREG32(mmCP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
+ WREG32(mmCP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
+ WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR, m->cp_hqd_pq_rptr_report_addr_lo);
+ WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
+ m->cp_hqd_pq_rptr_report_addr_hi);
+
+ if (valid_wptr > 0)
+ WREG32(mmCP_HQD_PQ_WPTR, shadow_wptr);
+
+ WREG32(mmCP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
+ WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, m->cp_hqd_pq_doorbell_control);
+
+ WREG32(mmCP_HQD_EOP_BASE_ADDR, m->cp_hqd_eop_base_addr_lo);
+ WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, m->cp_hqd_eop_base_addr_hi);
+ WREG32(mmCP_HQD_EOP_CONTROL, m->cp_hqd_eop_control);
+ WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr);
+ WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr);
+ WREG32(mmCP_HQD_EOP_EVENTS, m->cp_hqd_eop_done_events);
+
+ WREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO, m->cp_hqd_ctx_save_base_addr_lo);
+ WREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI, m->cp_hqd_ctx_save_base_addr_hi);
+ WREG32(mmCP_HQD_CTX_SAVE_CONTROL, m->cp_hqd_ctx_save_control);
+ WREG32(mmCP_HQD_CNTL_STACK_OFFSET, m->cp_hqd_cntl_stack_offset);
+ WREG32(mmCP_HQD_CNTL_STACK_SIZE, m->cp_hqd_cntl_stack_size);
+ WREG32(mmCP_HQD_WG_STATE_OFFSET, m->cp_hqd_wg_state_offset);
+ WREG32(mmCP_HQD_CTX_SAVE_SIZE, m->cp_hqd_ctx_save_size);
+
+ WREG32(mmCP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
+
+ WREG32(mmCP_HQD_DEQUEUE_REQUEST, m->cp_hqd_dequeue_request);
+ WREG32(mmCP_HQD_ERROR, m->cp_hqd_error);
+ WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem);
+ WREG32(mmCP_HQD_EOP_DONES, m->cp_hqd_eop_dones);
+
+ WREG32(mmCP_HQD_ACTIVE, m->cp_hqd_active);
+
+ release_queue(kgd);
+
+ return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+{
+ return 0;
+}
+
+static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
+ uint32_t pipe_id, uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t act;
+ bool retval = false;
+ uint32_t low, high;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+ act = RREG32(mmCP_HQD_ACTIVE);
+ if (act) {
+ low = lower_32_bits(queue_address >> 8);
+ high = upper_32_bits(queue_address >> 8);
+
+ if (low == RREG32(mmCP_HQD_PQ_BASE) &&
+ high == RREG32(mmCP_HQD_PQ_BASE_HI))
+ retval = true;
+ }
+ release_queue(kgd);
+ return retval;
+}
+
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t sdma_rlc_rb_cntl;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+
+ if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)
+ return true;
+
+ return false;
+}
+
+static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t temp;
+
+ acquire_queue(kgd, pipe_id, queue_id);
+
+ WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+
+ while (true) {
+ temp = RREG32(mmCP_HQD_ACTIVE);
+ if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+ break;
+ if (timeout == 0) {
+ pr_err("kfd: cp queue preemption time out (%dms)\n",
+ temp);
+ release_queue(kgd);
+ return -ETIME;
+ }
+ msleep(20);
+ timeout -= 20;
+ }
+
+ release_queue(kgd);
+ return 0;
+}
+
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+ unsigned int timeout)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ struct cik_sdma_rlc_registers *m;
+ uint32_t sdma_base_addr;
+ uint32_t temp;
+
+ m = get_sdma_mqd(mqd);
+ sdma_base_addr = get_sdma_base_addr(m);
+
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL);
+ temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK;
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp);
+
+ while (true) {
+ temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+ if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
+ break;
+ if (timeout == 0)
+ return -ETIME;
+ msleep(20);
+ timeout -= 20;
+ }
+
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
+ WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0);
+
+ return 0;
+}
+
+static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
+ uint8_t vmid)
+{
+ uint32_t reg;
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid);
+ return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK;
+}
+
+static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+ WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
+}
+
+static int kgd_address_watch_disable(struct kgd_dev *kgd)
+{
+ return 0;
+}
+
+static int kgd_address_watch_execute(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ uint32_t cntl_val,
+ uint32_t addr_hi,
+ uint32_t addr_lo)
+{
+ return 0;
+}
+
+static int kgd_wave_control_execute(struct kgd_dev *kgd,
+ uint32_t gfx_index_val,
+ uint32_t sq_cmd)
+{
+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
+ uint32_t data = 0;
+
+ mutex_lock(&adev->grbm_idx_mutex);
+
+ WREG32(mmGRBM_GFX_INDEX, gfx_index_val);
+ WREG32(mmSQ_CMD, sq_cmd);
+
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ INSTANCE_BROADCAST_WRITES, 1);
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ SH_BROADCAST_WRITES, 1);
+ data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
+ SE_BROADCAST_WRITES, 1);
+
+ WREG32(mmGRBM_GFX_INDEX, data);
+ mutex_unlock(&adev->grbm_idx_mutex);
+
+ return 0;
+}
+
+static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
+ unsigned int watch_point_id,
+ unsigned int reg_offset)
+{
+ return 0;
+}
+
+static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+ const union amdgpu_firmware_header *hdr;
+
+ BUG_ON(kgd == NULL);
+
+ switch (type) {
+ case KGD_ENGINE_PFP:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.pfp_fw->data;
+ break;
+
+ case KGD_ENGINE_ME:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.me_fw->data;
+ break;
+
+ case KGD_ENGINE_CE:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.ce_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC1:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.mec_fw->data;
+ break;
+
+ case KGD_ENGINE_MEC2:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.mec2_fw->data;
+ break;
+
+ case KGD_ENGINE_RLC:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->gfx.rlc_fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA1:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->sdma[0].fw->data;
+ break;
+
+ case KGD_ENGINE_SDMA2:
+ hdr = (const union amdgpu_firmware_header *)
+ adev->sdma[1].fw->data;
+ break;
+
+ default:
+ return 0;
+ }
+
+ if (hdr == NULL)
+ return 0;
+
+ /* Only 12 bit in use*/
+ return hdr->common.ucode_version;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 6a588371d54a..77f1d7c6ea3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -897,7 +897,7 @@ bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev,
if ((id == ASIC_INTERNAL_ENGINE_SS) ||
(id == ASIC_INTERNAL_MEMORY_SS))
ss->rate /= 100;
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
amdgpu_atombios_get_igp_ss_overrides(adev, ss, id);
return true;
}
@@ -1058,7 +1058,7 @@ void amdgpu_atombios_set_memory_clock(struct amdgpu_device *adev,
SET_MEMORY_CLOCK_PS_ALLOCATION args;
int index = GetIndexIntoMasterTable(COMMAND, SetMemoryClock);
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return;
args.ulTargetMemoryClock = cpu_to_le32(mem_clock); /* 10 khz */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 2742b9a35cbc..98d59ee640ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -33,7 +33,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
{
unsigned long start_jiffies;
unsigned long end_jiffies;
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
int i, r;
start_jiffies = jiffies;
@@ -42,17 +42,17 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence);
if (r)
goto exit_do_move;
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r)
goto exit_do_move;
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
}
end_jiffies = jiffies;
r = jiffies_to_msecs(end_jiffies - start_jiffies);
exit_do_move:
if (fence)
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
return r;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index ceb444f6d418..02add0a508cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -48,7 +48,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
resource_size_t vram_base;
resource_size_t size = 256 * 1024; /* ??? */
- if (!(adev->flags & AMDGPU_IS_APU))
+ if (!(adev->flags & AMD_IS_APU))
if (!amdgpu_card_posted(adev))
return false;
@@ -184,7 +184,7 @@ static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev)
bool found = false;
/* ATRM is for the discrete card only */
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return false;
while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) {
@@ -246,7 +246,7 @@ static inline bool amdgpu_atrm_get_bios(struct amdgpu_device *adev)
static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev)
{
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return igp_read_bios_from_vram(adev);
else
return amdgpu_asic_read_disabled_bios(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
new file mode 100644
index 000000000000..6b1243f9f86d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -0,0 +1,838 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <drm/drmP.h>
+#include <linux/firmware.h>
+#include <drm/amdgpu_drm.h>
+#include "amdgpu.h"
+#include "cgs_linux.h"
+#include "atom.h"
+#include "amdgpu_ucode.h"
+
+
+struct amdgpu_cgs_device {
+ struct cgs_device base;
+ struct amdgpu_device *adev;
+};
+
+#define CGS_FUNC_ADEV \
+ struct amdgpu_device *adev = \
+ ((struct amdgpu_cgs_device *)cgs_device)->adev
+
+static int amdgpu_cgs_gpu_mem_info(void *cgs_device, enum cgs_gpu_mem_type type,
+ uint64_t *mc_start, uint64_t *mc_size,
+ uint64_t *mem_size)
+{
+ CGS_FUNC_ADEV;
+ switch(type) {
+ case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB:
+ case CGS_GPU_MEM_TYPE__VISIBLE_FB:
+ *mc_start = 0;
+ *mc_size = adev->mc.visible_vram_size;
+ *mem_size = adev->mc.visible_vram_size - adev->vram_pin_size;
+ break;
+ case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
+ case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
+ *mc_start = adev->mc.visible_vram_size;
+ *mc_size = adev->mc.real_vram_size - adev->mc.visible_vram_size;
+ *mem_size = *mc_size;
+ break;
+ case CGS_GPU_MEM_TYPE__GART_CACHEABLE:
+ case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE:
+ *mc_start = adev->mc.gtt_start;
+ *mc_size = adev->mc.gtt_size;
+ *mem_size = adev->mc.gtt_size - adev->gart_pin_size;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int amdgpu_cgs_gmap_kmem(void *cgs_device, void *kmem,
+ uint64_t size,
+ uint64_t min_offset, uint64_t max_offset,
+ cgs_handle_t *kmem_handle, uint64_t *mcaddr)
+{
+ CGS_FUNC_ADEV;
+ int ret;
+ struct amdgpu_bo *bo;
+ struct page *kmem_page = vmalloc_to_page(kmem);
+ int npages = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT;
+
+ struct sg_table *sg = drm_prime_pages_to_sg(&kmem_page, npages);
+ ret = amdgpu_bo_create(adev, size, PAGE_SIZE, false,
+ AMDGPU_GEM_DOMAIN_GTT, 0, sg, &bo);
+ if (ret)
+ return ret;
+ ret = amdgpu_bo_reserve(bo, false);
+ if (unlikely(ret != 0))
+ return ret;
+
+ /* pin buffer into GTT */
+ ret = amdgpu_bo_pin_restricted(bo, AMDGPU_GEM_DOMAIN_GTT,
+ min_offset, max_offset, mcaddr);
+ amdgpu_bo_unreserve(bo);
+
+ *kmem_handle = (cgs_handle_t)bo;
+ return ret;
+}
+
+static int amdgpu_cgs_gunmap_kmem(void *cgs_device, cgs_handle_t kmem_handle)
+{
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)kmem_handle;
+
+ if (obj) {
+ int r = amdgpu_bo_reserve(obj, false);
+ if (likely(r == 0)) {
+ amdgpu_bo_unpin(obj);
+ amdgpu_bo_unreserve(obj);
+ }
+ amdgpu_bo_unref(&obj);
+
+ }
+ return 0;
+}
+
+static int amdgpu_cgs_alloc_gpu_mem(void *cgs_device,
+ enum cgs_gpu_mem_type type,
+ uint64_t size, uint64_t align,
+ uint64_t min_offset, uint64_t max_offset,
+ cgs_handle_t *handle)
+{
+ CGS_FUNC_ADEV;
+ uint16_t flags = 0;
+ int ret = 0;
+ uint32_t domain = 0;
+ struct amdgpu_bo *obj;
+ struct ttm_placement placement;
+ struct ttm_place place;
+
+ if (min_offset > max_offset) {
+ BUG_ON(1);
+ return -EINVAL;
+ }
+
+ /* fail if the alignment is not a power of 2 */
+ if (((align != 1) && (align & (align - 1)))
+ || size == 0 || align == 0)
+ return -EINVAL;
+
+
+ switch(type) {
+ case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB:
+ case CGS_GPU_MEM_TYPE__VISIBLE_FB:
+ flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+ domain = AMDGPU_GEM_DOMAIN_VRAM;
+ if (max_offset > adev->mc.real_vram_size)
+ return -EINVAL;
+ place.fpfn = min_offset >> PAGE_SHIFT;
+ place.lpfn = max_offset >> PAGE_SHIFT;
+ place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
+ TTM_PL_FLAG_VRAM;
+ break;
+ case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
+ case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
+ flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+ domain = AMDGPU_GEM_DOMAIN_VRAM;
+ if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
+ place.fpfn =
+ max(min_offset, adev->mc.visible_vram_size) >> PAGE_SHIFT;
+ place.lpfn =
+ min(max_offset, adev->mc.real_vram_size) >> PAGE_SHIFT;
+ place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
+ TTM_PL_FLAG_VRAM;
+ }
+
+ break;
+ case CGS_GPU_MEM_TYPE__GART_CACHEABLE:
+ domain = AMDGPU_GEM_DOMAIN_GTT;
+ place.fpfn = min_offset >> PAGE_SHIFT;
+ place.lpfn = max_offset >> PAGE_SHIFT;
+ place.flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
+ break;
+ case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE:
+ flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+ domain = AMDGPU_GEM_DOMAIN_GTT;
+ place.fpfn = min_offset >> PAGE_SHIFT;
+ place.lpfn = max_offset >> PAGE_SHIFT;
+ place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT |
+ TTM_PL_FLAG_UNCACHED;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+
+ *handle = 0;
+
+ placement.placement = &place;
+ placement.num_placement = 1;
+ placement.busy_placement = &place;
+ placement.num_busy_placement = 1;
+
+ ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE,
+ true, domain, flags,
+ NULL, &placement, &obj);
+ if (ret) {
+ DRM_ERROR("(%d) bo create failed\n", ret);
+ return ret;
+ }
+ *handle = (cgs_handle_t)obj;
+
+ return ret;
+}
+
+static int amdgpu_cgs_import_gpu_mem(void *cgs_device, int dmabuf_fd,
+ cgs_handle_t *handle)
+{
+ CGS_FUNC_ADEV;
+ int r;
+ uint32_t dma_handle;
+ struct drm_gem_object *obj;
+ struct amdgpu_bo *bo;
+ struct drm_device *dev = adev->ddev;
+ struct drm_file *file_priv = NULL, *priv;
+
+ mutex_lock(&dev->struct_mutex);
+ list_for_each_entry(priv, &dev->filelist, lhead) {
+ rcu_read_lock();
+ if (priv->pid == get_pid(task_pid(current)))
+ file_priv = priv;
+ rcu_read_unlock();
+ if (file_priv)
+ break;
+ }
+ mutex_unlock(&dev->struct_mutex);
+ r = dev->driver->prime_fd_to_handle(dev,
+ file_priv, dmabuf_fd,
+ &dma_handle);
+ spin_lock(&file_priv->table_lock);
+
+ /* Check if we currently have a reference on the object */
+ obj = idr_find(&file_priv->object_idr, dma_handle);
+ if (obj == NULL) {
+ spin_unlock(&file_priv->table_lock);
+ return -EINVAL;
+ }
+ spin_unlock(&file_priv->table_lock);
+ bo = gem_to_amdgpu_bo(obj);
+ *handle = (cgs_handle_t)bo;
+ return 0;
+}
+
+static int amdgpu_cgs_free_gpu_mem(void *cgs_device, cgs_handle_t handle)
+{
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
+
+ if (obj) {
+ int r = amdgpu_bo_reserve(obj, false);
+ if (likely(r == 0)) {
+ amdgpu_bo_kunmap(obj);
+ amdgpu_bo_unpin(obj);
+ amdgpu_bo_unreserve(obj);
+ }
+ amdgpu_bo_unref(&obj);
+
+ }
+ return 0;
+}
+
+static int amdgpu_cgs_gmap_gpu_mem(void *cgs_device, cgs_handle_t handle,
+ uint64_t *mcaddr)
+{
+ int r;
+ u64 min_offset, max_offset;
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
+
+ WARN_ON_ONCE(obj->placement.num_placement > 1);
+
+ min_offset = obj->placements[0].fpfn << PAGE_SHIFT;
+ max_offset = obj->placements[0].lpfn << PAGE_SHIFT;
+
+ r = amdgpu_bo_reserve(obj, false);
+ if (unlikely(r != 0))
+ return r;
+ r = amdgpu_bo_pin_restricted(obj, AMDGPU_GEM_DOMAIN_GTT,
+ min_offset, max_offset, mcaddr);
+ amdgpu_bo_unreserve(obj);
+ return r;
+}
+
+static int amdgpu_cgs_gunmap_gpu_mem(void *cgs_device, cgs_handle_t handle)
+{
+ int r;
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
+ r = amdgpu_bo_reserve(obj, false);
+ if (unlikely(r != 0))
+ return r;
+ r = amdgpu_bo_unpin(obj);
+ amdgpu_bo_unreserve(obj);
+ return r;
+}
+
+static int amdgpu_cgs_kmap_gpu_mem(void *cgs_device, cgs_handle_t handle,
+ void **map)
+{
+ int r;
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
+ r = amdgpu_bo_reserve(obj, false);
+ if (unlikely(r != 0))
+ return r;
+ r = amdgpu_bo_kmap(obj, map);
+ amdgpu_bo_unreserve(obj);
+ return r;
+}
+
+static int amdgpu_cgs_kunmap_gpu_mem(void *cgs_device, cgs_handle_t handle)
+{
+ int r;
+ struct amdgpu_bo *obj = (struct amdgpu_bo *)handle;
+ r = amdgpu_bo_reserve(obj, false);
+ if (unlikely(r != 0))
+ return r;
+ amdgpu_bo_kunmap(obj);
+ amdgpu_bo_unreserve(obj);
+ return r;
+}
+
+static uint32_t amdgpu_cgs_read_register(void *cgs_device, unsigned offset)
+{
+ CGS_FUNC_ADEV;
+ return RREG32(offset);
+}
+
+static void amdgpu_cgs_write_register(void *cgs_device, unsigned offset,
+ uint32_t value)
+{
+ CGS_FUNC_ADEV;
+ WREG32(offset, value);
+}
+
+static uint32_t amdgpu_cgs_read_ind_register(void *cgs_device,
+ enum cgs_ind_reg space,
+ unsigned index)
+{
+ CGS_FUNC_ADEV;
+ switch (space) {
+ case CGS_IND_REG__MMIO:
+ return RREG32_IDX(index);
+ case CGS_IND_REG__PCIE:
+ return RREG32_PCIE(index);
+ case CGS_IND_REG__SMC:
+ return RREG32_SMC(index);
+ case CGS_IND_REG__UVD_CTX:
+ return RREG32_UVD_CTX(index);
+ case CGS_IND_REG__DIDT:
+ return RREG32_DIDT(index);
+ case CGS_IND_REG__AUDIO_ENDPT:
+ DRM_ERROR("audio endpt register access not implemented.\n");
+ return 0;
+ }
+ WARN(1, "Invalid indirect register space");
+ return 0;
+}
+
+static void amdgpu_cgs_write_ind_register(void *cgs_device,
+ enum cgs_ind_reg space,
+ unsigned index, uint32_t value)
+{
+ CGS_FUNC_ADEV;
+ switch (space) {
+ case CGS_IND_REG__MMIO:
+ return WREG32_IDX(index, value);
+ case CGS_IND_REG__PCIE:
+ return WREG32_PCIE(index, value);
+ case CGS_IND_REG__SMC:
+ return WREG32_SMC(index, value);
+ case CGS_IND_REG__UVD_CTX:
+ return WREG32_UVD_CTX(index, value);
+ case CGS_IND_REG__DIDT:
+ return WREG32_DIDT(index, value);
+ case CGS_IND_REG__AUDIO_ENDPT:
+ DRM_ERROR("audio endpt register access not implemented.\n");
+ return;
+ }
+ WARN(1, "Invalid indirect register space");
+}
+
+static uint8_t amdgpu_cgs_read_pci_config_byte(void *cgs_device, unsigned addr)
+{
+ CGS_FUNC_ADEV;
+ uint8_t val;
+ int ret = pci_read_config_byte(adev->pdev, addr, &val);
+ if (WARN(ret, "pci_read_config_byte error"))
+ return 0;
+ return val;
+}
+
+static uint16_t amdgpu_cgs_read_pci_config_word(void *cgs_device, unsigned addr)
+{
+ CGS_FUNC_ADEV;
+ uint16_t val;
+ int ret = pci_read_config_word(adev->pdev, addr, &val);
+ if (WARN(ret, "pci_read_config_word error"))
+ return 0;
+ return val;
+}
+
+static uint32_t amdgpu_cgs_read_pci_config_dword(void *cgs_device,
+ unsigned addr)
+{
+ CGS_FUNC_ADEV;
+ uint32_t val;
+ int ret = pci_read_config_dword(adev->pdev, addr, &val);
+ if (WARN(ret, "pci_read_config_dword error"))
+ return 0;
+ return val;
+}
+
+static void amdgpu_cgs_write_pci_config_byte(void *cgs_device, unsigned addr,
+ uint8_t value)
+{
+ CGS_FUNC_ADEV;
+ int ret = pci_write_config_byte(adev->pdev, addr, value);
+ WARN(ret, "pci_write_config_byte error");
+}
+
+static void amdgpu_cgs_write_pci_config_word(void *cgs_device, unsigned addr,
+ uint16_t value)
+{
+ CGS_FUNC_ADEV;
+ int ret = pci_write_config_word(adev->pdev, addr, value);
+ WARN(ret, "pci_write_config_word error");
+}
+
+static void amdgpu_cgs_write_pci_config_dword(void *cgs_device, unsigned addr,
+ uint32_t value)
+{
+ CGS_FUNC_ADEV;
+ int ret = pci_write_config_dword(adev->pdev, addr, value);
+ WARN(ret, "pci_write_config_dword error");
+}
+
+static const void *amdgpu_cgs_atom_get_data_table(void *cgs_device,
+ unsigned table, uint16_t *size,
+ uint8_t *frev, uint8_t *crev)
+{
+ CGS_FUNC_ADEV;
+ uint16_t data_start;
+
+ if (amdgpu_atom_parse_data_header(
+ adev->mode_info.atom_context, table, size,
+ frev, crev, &data_start))
+ return (uint8_t*)adev->mode_info.atom_context->bios +
+ data_start;
+
+ return NULL;
+}
+
+static int amdgpu_cgs_atom_get_cmd_table_revs(void *cgs_device, unsigned table,
+ uint8_t *frev, uint8_t *crev)
+{
+ CGS_FUNC_ADEV;
+
+ if (amdgpu_atom_parse_cmd_header(
+ adev->mode_info.atom_context, table,
+ frev, crev))
+ return 0;
+
+ return -EINVAL;
+}
+
+static int amdgpu_cgs_atom_exec_cmd_table(void *cgs_device, unsigned table,
+ void *args)
+{
+ CGS_FUNC_ADEV;
+
+ return amdgpu_atom_execute_table(
+ adev->mode_info.atom_context, table, args);
+}
+
+static int amdgpu_cgs_create_pm_request(void *cgs_device, cgs_handle_t *request)
+{
+ /* TODO */
+ return 0;
+}
+
+static int amdgpu_cgs_destroy_pm_request(void *cgs_device, cgs_handle_t request)
+{
+ /* TODO */
+ return 0;
+}
+
+static int amdgpu_cgs_set_pm_request(void *cgs_device, cgs_handle_t request,
+ int active)
+{
+ /* TODO */
+ return 0;
+}
+
+static int amdgpu_cgs_pm_request_clock(void *cgs_device, cgs_handle_t request,
+ enum cgs_clock clock, unsigned freq)
+{
+ /* TODO */
+ return 0;
+}
+
+static int amdgpu_cgs_pm_request_engine(void *cgs_device, cgs_handle_t request,
+ enum cgs_engine engine, int powered)
+{
+ /* TODO */
+ return 0;
+}
+
+
+
+static int amdgpu_cgs_pm_query_clock_limits(void *cgs_device,
+ enum cgs_clock clock,
+ struct cgs_clock_limits *limits)
+{
+ /* TODO */
+ return 0;
+}
+
+static int amdgpu_cgs_set_camera_voltages(void *cgs_device, uint32_t mask,
+ const uint32_t *voltages)
+{
+ DRM_ERROR("not implemented");
+ return -EPERM;
+}
+
+struct cgs_irq_params {
+ unsigned src_id;
+ cgs_irq_source_set_func_t set;
+ cgs_irq_handler_func_t handler;
+ void *private_data;
+};
+
+static int cgs_set_irq_state(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *src,
+ unsigned type,
+ enum amdgpu_interrupt_state state)
+{
+ struct cgs_irq_params *irq_params =
+ (struct cgs_irq_params *)src->data;
+ if (!irq_params)
+ return -EINVAL;
+ if (!irq_params->set)
+ return -EINVAL;
+ return irq_params->set(irq_params->private_data,
+ irq_params->src_id,
+ type,
+ (int)state);
+}
+
+static int cgs_process_irq(struct amdgpu_device *adev,
+ struct amdgpu_irq_src *source,
+ struct amdgpu_iv_entry *entry)
+{
+ struct cgs_irq_params *irq_params =
+ (struct cgs_irq_params *)source->data;
+ if (!irq_params)
+ return -EINVAL;
+ if (!irq_params->handler)
+ return -EINVAL;
+ return irq_params->handler(irq_params->private_data,
+ irq_params->src_id,
+ entry->iv_entry);
+}
+
+static const struct amdgpu_irq_src_funcs cgs_irq_funcs = {
+ .set = cgs_set_irq_state,
+ .process = cgs_process_irq,
+};
+
+static int amdgpu_cgs_add_irq_source(void *cgs_device, unsigned src_id,
+ unsigned num_types,
+ cgs_irq_source_set_func_t set,
+ cgs_irq_handler_func_t handler,
+ void *private_data)
+{
+ CGS_FUNC_ADEV;
+ int ret = 0;
+ struct cgs_irq_params *irq_params;
+ struct amdgpu_irq_src *source =
+ kzalloc(sizeof(struct amdgpu_irq_src), GFP_KERNEL);
+ if (!source)
+ return -ENOMEM;
+ irq_params =
+ kzalloc(sizeof(struct cgs_irq_params), GFP_KERNEL);
+ if (!irq_params) {
+ kfree(source);
+ return -ENOMEM;
+ }
+ source->num_types = num_types;
+ source->funcs = &cgs_irq_funcs;
+ irq_params->src_id = src_id;
+ irq_params->set = set;
+ irq_params->handler = handler;
+ irq_params->private_data = private_data;
+ source->data = (void *)irq_params;
+ ret = amdgpu_irq_add_id(adev, src_id, source);
+ if (ret) {
+ kfree(irq_params);
+ kfree(source);
+ }
+
+ return ret;
+}
+
+static int amdgpu_cgs_irq_get(void *cgs_device, unsigned src_id, unsigned type)
+{
+ CGS_FUNC_ADEV;
+ return amdgpu_irq_get(adev, adev->irq.sources[src_id], type);
+}
+
+static int amdgpu_cgs_irq_put(void *cgs_device, unsigned src_id, unsigned type)
+{
+ CGS_FUNC_ADEV;
+ return amdgpu_irq_put(adev, adev->irq.sources[src_id], type);
+}
+
+int amdgpu_cgs_set_clockgating_state(void *cgs_device,
+ enum amd_ip_block_type block_type,
+ enum amd_clockgating_state state)
+{
+ CGS_FUNC_ADEV;
+ int i, r = -1;
+
+ for (i = 0; i < adev->num_ip_blocks; i++) {
+ if (!adev->ip_block_status[i].valid)
+ continue;
+
+ if (adev->ip_blocks[i].type == block_type) {
+ r = adev->ip_blocks[i].funcs->set_clockgating_state(
+ (void *)adev,
+ state);
+ break;
+ }
+ }
+ return r;
+}
+
+int amdgpu_cgs_set_powergating_state(void *cgs_device,
+ enum amd_ip_block_type block_type,
+ enum amd_powergating_state state)
+{
+ CGS_FUNC_ADEV;
+ int i, r = -1;
+
+ for (i = 0; i < adev->num_ip_blocks; i++) {
+ if (!adev->ip_block_status[i].valid)
+ continue;
+
+ if (adev->ip_blocks[i].type == block_type) {
+ r = adev->ip_blocks[i].funcs->set_powergating_state(
+ (void *)adev,
+ state);
+ break;
+ }
+ }
+ return r;
+}
+
+
+static uint32_t fw_type_convert(void *cgs_device, uint32_t fw_type)
+{
+ CGS_FUNC_ADEV;
+ enum AMDGPU_UCODE_ID result = AMDGPU_UCODE_ID_MAXIMUM;
+
+ switch (fw_type) {
+ case CGS_UCODE_ID_SDMA0:
+ result = AMDGPU_UCODE_ID_SDMA0;
+ break;
+ case CGS_UCODE_ID_SDMA1:
+ result = AMDGPU_UCODE_ID_SDMA1;
+ break;
+ case CGS_UCODE_ID_CP_CE:
+ result = AMDGPU_UCODE_ID_CP_CE;
+ break;
+ case CGS_UCODE_ID_CP_PFP:
+ result = AMDGPU_UCODE_ID_CP_PFP;
+ break;
+ case CGS_UCODE_ID_CP_ME:
+ result = AMDGPU_UCODE_ID_CP_ME;
+ break;
+ case CGS_UCODE_ID_CP_MEC:
+ case CGS_UCODE_ID_CP_MEC_JT1:
+ result = AMDGPU_UCODE_ID_CP_MEC1;
+ break;
+ case CGS_UCODE_ID_CP_MEC_JT2:
+ if (adev->asic_type == CHIP_TONGA)
+ result = AMDGPU_UCODE_ID_CP_MEC2;
+ else if (adev->asic_type == CHIP_CARRIZO)
+ result = AMDGPU_UCODE_ID_CP_MEC1;
+ break;
+ case CGS_UCODE_ID_RLC_G:
+ result = AMDGPU_UCODE_ID_RLC_G;
+ break;
+ default:
+ DRM_ERROR("Firmware type not supported\n");
+ }
+ return result;
+}
+
+static int amdgpu_cgs_get_firmware_info(void *cgs_device,
+ enum cgs_ucode_id type,
+ struct cgs_firmware_info *info)
+{
+ CGS_FUNC_ADEV;
+
+ if (CGS_UCODE_ID_SMU != type) {
+ uint64_t gpu_addr;
+ uint32_t data_size;
+ const struct gfx_firmware_header_v1_0 *header;
+ enum AMDGPU_UCODE_ID id;
+ struct amdgpu_firmware_info *ucode;
+
+ id = fw_type_convert(cgs_device, type);
+ ucode = &adev->firmware.ucode[id];
+ if (ucode->fw == NULL)
+ return -EINVAL;
+
+ gpu_addr = ucode->mc_addr;
+ header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
+ data_size = le32_to_cpu(header->header.ucode_size_bytes);
+
+ if ((type == CGS_UCODE_ID_CP_MEC_JT1) ||
+ (type == CGS_UCODE_ID_CP_MEC_JT2)) {
+ gpu_addr += le32_to_cpu(header->jt_offset) << 2;
+ data_size = le32_to_cpu(header->jt_size) << 2;
+ }
+ info->mc_addr = gpu_addr;
+ info->image_size = data_size;
+ info->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
+ info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version);
+ } else {
+ char fw_name[30] = {0};
+ int err = 0;
+ uint32_t ucode_size;
+ uint32_t ucode_start_address;
+ const uint8_t *src;
+ const struct smc_firmware_header_v1_0 *hdr;
+
+ switch (adev->asic_type) {
+ case CHIP_TONGA:
+ strcpy(fw_name, "amdgpu/tonga_smc.bin");
+ break;
+ default:
+ DRM_ERROR("SMC firmware not supported\n");
+ return -EINVAL;
+ }
+
+ err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
+ if (err) {
+ DRM_ERROR("Failed to request firmware\n");
+ return err;
+ }
+
+ err = amdgpu_ucode_validate(adev->pm.fw);
+ if (err) {
+ DRM_ERROR("Failed to load firmware \"%s\"", fw_name);
+ release_firmware(adev->pm.fw);
+ adev->pm.fw = NULL;
+ return err;
+ }
+
+ hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
+ adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
+ ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
+ ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
+ src = (const uint8_t *)(adev->pm.fw->data +
+ le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+ info->version = adev->pm.fw_version;
+ info->image_size = ucode_size;
+ info->kptr = (void *)src;
+ }
+ return 0;
+}
+
+static const struct cgs_ops amdgpu_cgs_ops = {
+ amdgpu_cgs_gpu_mem_info,
+ amdgpu_cgs_gmap_kmem,
+ amdgpu_cgs_gunmap_kmem,
+ amdgpu_cgs_alloc_gpu_mem,
+ amdgpu_cgs_free_gpu_mem,
+ amdgpu_cgs_gmap_gpu_mem,
+ amdgpu_cgs_gunmap_gpu_mem,
+ amdgpu_cgs_kmap_gpu_mem,
+ amdgpu_cgs_kunmap_gpu_mem,
+ amdgpu_cgs_read_register,
+ amdgpu_cgs_write_register,
+ amdgpu_cgs_read_ind_register,
+ amdgpu_cgs_write_ind_register,
+ amdgpu_cgs_read_pci_config_byte,
+ amdgpu_cgs_read_pci_config_word,
+ amdgpu_cgs_read_pci_config_dword,
+ amdgpu_cgs_write_pci_config_byte,
+ amdgpu_cgs_write_pci_config_word,
+ amdgpu_cgs_write_pci_config_dword,
+ amdgpu_cgs_atom_get_data_table,
+ amdgpu_cgs_atom_get_cmd_table_revs,
+ amdgpu_cgs_atom_exec_cmd_table,
+ amdgpu_cgs_create_pm_request,
+ amdgpu_cgs_destroy_pm_request,
+ amdgpu_cgs_set_pm_request,
+ amdgpu_cgs_pm_request_clock,
+ amdgpu_cgs_pm_request_engine,
+ amdgpu_cgs_pm_query_clock_limits,
+ amdgpu_cgs_set_camera_voltages,
+ amdgpu_cgs_get_firmware_info,
+ amdgpu_cgs_set_powergating_state,
+ amdgpu_cgs_set_clockgating_state
+};
+
+static const struct cgs_os_ops amdgpu_cgs_os_ops = {
+ amdgpu_cgs_import_gpu_mem,
+ amdgpu_cgs_add_irq_source,
+ amdgpu_cgs_irq_get,
+ amdgpu_cgs_irq_put
+};
+
+void *amdgpu_cgs_create_device(struct amdgpu_device *adev)
+{
+ struct amdgpu_cgs_device *cgs_device =
+ kmalloc(sizeof(*cgs_device), GFP_KERNEL);
+
+ if (!cgs_device) {
+ DRM_ERROR("Couldn't allocate CGS device structure\n");
+ return NULL;
+ }
+
+ cgs_device->base.ops = &amdgpu_cgs_ops;
+ cgs_device->base.os_ops = &amdgpu_cgs_os_ops;
+ cgs_device->adev = adev;
+
+ return cgs_device;
+}
+
+void amdgpu_cgs_destroy_device(void *cgs_device)
+{
+ kfree(cgs_device);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 27df17a0e620..89c3dd62ba21 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -75,6 +75,11 @@ void amdgpu_connector_hotplug(struct drm_connector *connector)
if (!amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) {
drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
} else if (amdgpu_atombios_dp_needs_link_train(amdgpu_connector)) {
+ /* Don't try to start link training before we
+ * have the dpcd */
+ if (!amdgpu_atombios_dp_get_dpcd(amdgpu_connector))
+ return;
+
/* set it to OFF so that drm_helper_connector_dpms()
* won't return immediately since the current state
* is ON at this point.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 1f040d85ac47..3b355aeb62fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -126,6 +126,30 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
return 0;
}
+struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev,
+ struct drm_file *filp,
+ struct amdgpu_ctx *ctx,
+ struct amdgpu_ib *ibs,
+ uint32_t num_ibs)
+{
+ struct amdgpu_cs_parser *parser;
+ int i;
+
+ parser = kzalloc(sizeof(struct amdgpu_cs_parser), GFP_KERNEL);
+ if (!parser)
+ return NULL;
+
+ parser->adev = adev;
+ parser->filp = filp;
+ parser->ctx = ctx;
+ parser->ibs = ibs;
+ parser->num_ibs = num_ibs;
+ for (i = 0; i < num_ibs; i++)
+ ibs[i].ctx = ctx;
+
+ return parser;
+}
+
int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
{
union drm_amdgpu_cs *cs = data;
@@ -147,13 +171,13 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
/* get chunks */
INIT_LIST_HEAD(&p->validated);
- chunk_array = kcalloc(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
+ chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
if (chunk_array == NULL) {
r = -ENOMEM;
goto out;
}
- chunk_array_user = (uint64_t *)(unsigned long)(cs->in.chunks);
+ chunk_array_user = (uint64_t __user *)(cs->in.chunks);
if (copy_from_user(chunk_array, chunk_array_user,
sizeof(uint64_t)*cs->in.num_chunks)) {
r = -EFAULT;
@@ -161,7 +185,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
}
p->nchunks = cs->in.num_chunks;
- p->chunks = kcalloc(p->nchunks, sizeof(struct amdgpu_cs_chunk),
+ p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
GFP_KERNEL);
if (p->chunks == NULL) {
r = -ENOMEM;
@@ -173,7 +197,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
struct drm_amdgpu_cs_chunk user_chunk;
uint32_t __user *cdata;
- chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
+ chunk_ptr = (void __user *)chunk_array[i];
if (copy_from_user(&user_chunk, chunk_ptr,
sizeof(struct drm_amdgpu_cs_chunk))) {
r = -EFAULT;
@@ -183,7 +207,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
p->chunks[i].length_dw = user_chunk.length_dw;
size = p->chunks[i].length_dw;
- cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
+ cdata = (void __user *)user_chunk.chunk_data;
p->chunks[i].user_ptr = cdata;
p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
@@ -235,11 +259,10 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
}
}
+
p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL);
- if (!p->ibs) {
+ if (!p->ibs)
r = -ENOMEM;
- goto out;
- }
out:
kfree(chunk_array);
@@ -331,7 +354,7 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p)
* into account. We don't want to disallow buffer moves
* completely.
*/
- if (current_domain != AMDGPU_GEM_DOMAIN_CPU &&
+ if ((lobj->allowed_domains & current_domain) != 0 &&
(domain & current_domain) == 0 && /* will be moved */
bytes_moved > bytes_moved_threshold) {
/* don't move it */
@@ -415,18 +438,8 @@ static int cmp_size_smaller_first(void *priv, struct list_head *a,
return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
}
-/**
- * cs_parser_fini() - clean parser states
- * @parser: parser structure holding parsing context.
- * @error: error number
- *
- * If error is set than unvalidate buffer, otherwise just free memory
- * used by parsing context.
- **/
-static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
+static void amdgpu_cs_parser_fini_early(struct amdgpu_cs_parser *parser, int error, bool backoff)
{
- unsigned i;
-
if (!error) {
/* Sort the buffer list from the smallest to largest buffer,
* which affects the order of buffers in the LRU list.
@@ -447,21 +460,45 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo
ttm_eu_backoff_reservation(&parser->ticket,
&parser->validated);
}
+}
+static void amdgpu_cs_parser_fini_late(struct amdgpu_cs_parser *parser)
+{
+ unsigned i;
if (parser->ctx)
amdgpu_ctx_put(parser->ctx);
if (parser->bo_list)
amdgpu_bo_list_put(parser->bo_list);
+
drm_free_large(parser->vm_bos);
for (i = 0; i < parser->nchunks; i++)
drm_free_large(parser->chunks[i].kdata);
kfree(parser->chunks);
- if (parser->ibs)
- for (i = 0; i < parser->num_ibs; i++)
- amdgpu_ib_free(parser->adev, &parser->ibs[i]);
- kfree(parser->ibs);
- if (parser->uf.bo)
- drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base);
+ if (!amdgpu_enable_scheduler)
+ {
+ if (parser->ibs)
+ for (i = 0; i < parser->num_ibs; i++)
+ amdgpu_ib_free(parser->adev, &parser->ibs[i]);
+ kfree(parser->ibs);
+ if (parser->uf.bo)
+ drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base);
+ }
+
+ kfree(parser);
+}
+
+/**
+ * cs_parser_fini() - clean parser states
+ * @parser: parser structure holding parsing context.
+ * @error: error number
+ *
+ * If error is set than unvalidate buffer, otherwise just free memory
+ * used by parsing context.
+ **/
+static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
+{
+ amdgpu_cs_parser_fini_early(parser, error, backoff);
+ amdgpu_cs_parser_fini_late(parser);
}
static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
@@ -476,12 +513,18 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
if (r)
return r;
+ r = amdgpu_sync_fence(adev, &p->ibs[0].sync, vm->page_directory_fence);
+ if (r)
+ return r;
+
r = amdgpu_vm_clear_freed(adev, vm);
if (r)
return r;
if (p->bo_list) {
for (i = 0; i < p->bo_list->num_entries; i++) {
+ struct fence *f;
+
/* ignore duplicates */
bo = p->bo_list->array[i].robj;
if (!bo)
@@ -495,7 +538,10 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
if (r)
return r;
- amdgpu_sync_fence(&p->ibs[0].sync, bo_va->last_pt_update);
+ f = bo_va->last_pt_update;
+ r = amdgpu_sync_fence(adev, &p->ibs[0].sync, f);
+ if (r)
+ return r;
}
}
@@ -529,9 +575,9 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
goto out;
}
amdgpu_cs_sync_rings(parser);
-
- r = amdgpu_ib_schedule(adev, parser->num_ibs, parser->ibs,
- parser->filp);
+ if (!amdgpu_enable_scheduler)
+ r = amdgpu_ib_schedule(adev, parser->num_ibs, parser->ibs,
+ parser->filp);
out:
mutex_unlock(&vm->mutex);
@@ -650,7 +696,6 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
ib->oa_size = amdgpu_bo_size(oa);
}
}
-
/* wrap the last IB with user fence */
if (parser->uf.bo) {
struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1];
@@ -693,9 +738,9 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
sizeof(struct drm_amdgpu_cs_chunk_dep);
for (j = 0; j < num_deps; ++j) {
- struct amdgpu_fence *fence;
struct amdgpu_ring *ring;
struct amdgpu_ctx *ctx;
+ struct fence *fence;
r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
deps[j].ip_instance,
@@ -707,85 +752,137 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
if (ctx == NULL)
return -EINVAL;
- r = amdgpu_fence_recreate(ring, p->filp,
- deps[j].handle,
- &fence);
- if (r) {
+ fence = amdgpu_ctx_get_fence(ctx, ring,
+ deps[j].handle);
+ if (IS_ERR(fence)) {
+ r = PTR_ERR(fence);
amdgpu_ctx_put(ctx);
return r;
- }
- amdgpu_sync_fence(&ib->sync, fence);
- amdgpu_fence_unref(&fence);
- amdgpu_ctx_put(ctx);
+ } else if (fence) {
+ r = amdgpu_sync_fence(adev, &ib->sync, fence);
+ fence_put(fence);
+ amdgpu_ctx_put(ctx);
+ if (r)
+ return r;
+ }
}
}
return 0;
}
+static int amdgpu_cs_free_job(struct amdgpu_job *sched_job)
+{
+ int i;
+ if (sched_job->ibs)
+ for (i = 0; i < sched_job->num_ibs; i++)
+ amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]);
+ kfree(sched_job->ibs);
+ if (sched_job->uf.bo)
+ drm_gem_object_unreference_unlocked(&sched_job->uf.bo->gem_base);
+ return 0;
+}
+
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
struct amdgpu_device *adev = dev->dev_private;
union drm_amdgpu_cs *cs = data;
- struct amdgpu_cs_parser parser;
- int r, i;
+ struct amdgpu_cs_parser *parser;
bool reserved_buffers = false;
+ int i, r;
down_read(&adev->exclusive_lock);
if (!adev->accel_working) {
up_read(&adev->exclusive_lock);
return -EBUSY;
}
- /* initialize parser */
- memset(&parser, 0, sizeof(struct amdgpu_cs_parser));
- parser.filp = filp;
- parser.adev = adev;
- r = amdgpu_cs_parser_init(&parser, data);
+
+ parser = amdgpu_cs_parser_create(adev, filp, NULL, NULL, 0);
+ if (!parser)
+ return -ENOMEM;
+ r = amdgpu_cs_parser_init(parser, data);
if (r) {
DRM_ERROR("Failed to initialize parser !\n");
- amdgpu_cs_parser_fini(&parser, r, false);
+ amdgpu_cs_parser_fini(parser, r, false);
up_read(&adev->exclusive_lock);
r = amdgpu_cs_handle_lockup(adev, r);
return r;
}
- r = amdgpu_cs_parser_relocs(&parser);
- if (r) {
- if (r != -ERESTARTSYS) {
- if (r == -ENOMEM)
- DRM_ERROR("Not enough memory for command submission!\n");
- else
- DRM_ERROR("Failed to process the buffer list %d!\n", r);
- }
+ r = amdgpu_cs_parser_relocs(parser);
+ if (r == -ENOMEM)
+ DRM_ERROR("Not enough memory for command submission!\n");
+ else if (r && r != -ERESTARTSYS)
+ DRM_ERROR("Failed to process the buffer list %d!\n", r);
+ else if (!r) {
+ reserved_buffers = true;
+ r = amdgpu_cs_ib_fill(adev, parser);
}
if (!r) {
- reserved_buffers = true;
- r = amdgpu_cs_ib_fill(adev, &parser);
+ r = amdgpu_cs_dependencies(adev, parser);
+ if (r)
+ DRM_ERROR("Failed in the dependencies handling %d!\n", r);
}
- if (!r)
- r = amdgpu_cs_dependencies(adev, &parser);
-
- if (r) {
- amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
- up_read(&adev->exclusive_lock);
- r = amdgpu_cs_handle_lockup(adev, r);
- return r;
- }
+ if (r)
+ goto out;
- for (i = 0; i < parser.num_ibs; i++)
- trace_amdgpu_cs(&parser, i);
+ for (i = 0; i < parser->num_ibs; i++)
+ trace_amdgpu_cs(parser, i);
- r = amdgpu_cs_ib_vm_chunk(adev, &parser);
- if (r) {
+ r = amdgpu_cs_ib_vm_chunk(adev, parser);
+ if (r)
goto out;
+
+ if (amdgpu_enable_scheduler && parser->num_ibs) {
+ struct amdgpu_job *job;
+ struct amdgpu_ring * ring = parser->ibs->ring;
+ job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL);
+ if (!job)
+ return -ENOMEM;
+ job->base.sched = ring->scheduler;
+ job->base.s_entity = &parser->ctx->rings[ring->idx].entity;
+ job->adev = parser->adev;
+ job->ibs = parser->ibs;
+ job->num_ibs = parser->num_ibs;
+ job->base.owner = parser->filp;
+ mutex_init(&job->job_lock);
+ if (job->ibs[job->num_ibs - 1].user) {
+ memcpy(&job->uf, &parser->uf,
+ sizeof(struct amdgpu_user_fence));
+ job->ibs[job->num_ibs - 1].user = &job->uf;
+ }
+
+ job->free_job = amdgpu_cs_free_job;
+ mutex_lock(&job->job_lock);
+ r = amd_sched_entity_push_job((struct amd_sched_job *)job);
+ if (r) {
+ mutex_unlock(&job->job_lock);
+ amdgpu_cs_free_job(job);
+ kfree(job);
+ goto out;
+ }
+ cs->out.handle =
+ amdgpu_ctx_add_fence(parser->ctx, ring,
+ &job->base.s_fence->base);
+ parser->ibs[parser->num_ibs - 1].sequence = cs->out.handle;
+
+ list_sort(NULL, &parser->validated, cmp_size_smaller_first);
+ ttm_eu_fence_buffer_objects(&parser->ticket,
+ &parser->validated,
+ &job->base.s_fence->base);
+
+ mutex_unlock(&job->job_lock);
+ amdgpu_cs_parser_fini_late(parser);
+ up_read(&adev->exclusive_lock);
+ return 0;
}
- cs->out.handle = parser.ibs[parser.num_ibs - 1].fence->seq;
+ cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence;
out:
- amdgpu_cs_parser_fini(&parser, r, true);
+ amdgpu_cs_parser_fini(parser, r, reserved_buffers);
up_read(&adev->exclusive_lock);
r = amdgpu_cs_handle_lockup(adev, r);
return r;
@@ -806,30 +903,29 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
union drm_amdgpu_wait_cs *wait = data;
struct amdgpu_device *adev = dev->dev_private;
unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
- struct amdgpu_fence *fence = NULL;
struct amdgpu_ring *ring = NULL;
struct amdgpu_ctx *ctx;
+ struct fence *fence;
long r;
- ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
- if (ctx == NULL)
- return -EINVAL;
-
r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
wait->in.ring, &ring);
- if (r) {
- amdgpu_ctx_put(ctx);
+ if (r)
return r;
- }
- r = amdgpu_fence_recreate(ring, filp, wait->in.handle, &fence);
- if (r) {
- amdgpu_ctx_put(ctx);
- return r;
- }
+ ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
+ if (ctx == NULL)
+ return -EINVAL;
+
+ fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
+ if (IS_ERR(fence))
+ r = PTR_ERR(fence);
+ else if (fence) {
+ r = fence_wait_timeout(fence, true, timeout);
+ fence_put(fence);
+ } else
+ r = 1;
- r = fence_wait_timeout(&fence->base, true, timeout);
- amdgpu_fence_unref(&fence);
amdgpu_ctx_put(ctx);
if (r < 0)
return r;
@@ -864,7 +960,16 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
if (!reloc->bo_va)
continue;
- list_for_each_entry(mapping, &reloc->bo_va->mappings, list) {
+ list_for_each_entry(mapping, &reloc->bo_va->valids, list) {
+ if (mapping->it.start > addr ||
+ addr > mapping->it.last)
+ continue;
+
+ *bo = reloc->bo_va->bo;
+ return mapping;
+ }
+
+ list_for_each_entry(mapping, &reloc->bo_va->invalids, list) {
if (mapping->it.start > addr ||
addr > mapping->it.last)
continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 6c66ac8a1891..20cbc4eb5a6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -25,54 +25,107 @@
#include <drm/drmP.h>
#include "amdgpu.h"
-static void amdgpu_ctx_do_release(struct kref *ref)
+int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel,
+ struct amdgpu_ctx *ctx)
{
- struct amdgpu_ctx *ctx;
- struct amdgpu_ctx_mgr *mgr;
+ unsigned i, j;
+ int r;
- ctx = container_of(ref, struct amdgpu_ctx, refcount);
- mgr = &ctx->fpriv->ctx_mgr;
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->adev = adev;
+ kref_init(&ctx->refcount);
+ spin_lock_init(&ctx->ring_lock);
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
+ ctx->rings[i].sequence = 1;
- idr_remove(&mgr->ctx_handles, ctx->id);
- kfree(ctx);
+ if (amdgpu_enable_scheduler) {
+ /* create context entity for each ring */
+ for (i = 0; i < adev->num_rings; i++) {
+ struct amd_sched_rq *rq;
+ if (kernel)
+ rq = &adev->rings[i]->scheduler->kernel_rq;
+ else
+ rq = &adev->rings[i]->scheduler->sched_rq;
+ r = amd_sched_entity_init(adev->rings[i]->scheduler,
+ &ctx->rings[i].entity,
+ rq, amdgpu_sched_jobs);
+ if (r)
+ break;
+ }
+
+ if (i < adev->num_rings) {
+ for (j = 0; j < i; j++)
+ amd_sched_entity_fini(adev->rings[j]->scheduler,
+ &ctx->rings[j].entity);
+ kfree(ctx);
+ return r;
+ }
+ }
+ return 0;
}
-int amdgpu_ctx_alloc(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t *id, uint32_t flags)
+void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
+{
+ struct amdgpu_device *adev = ctx->adev;
+ unsigned i, j;
+
+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
+ for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j)
+ fence_put(ctx->rings[i].fences[j]);
+
+ if (amdgpu_enable_scheduler) {
+ for (i = 0; i < adev->num_rings; i++)
+ amd_sched_entity_fini(adev->rings[i]->scheduler,
+ &ctx->rings[i].entity);
+ }
+}
+
+static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
+ struct amdgpu_fpriv *fpriv,
+ uint32_t *id)
{
- int r;
- struct amdgpu_ctx *ctx;
struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
+ struct amdgpu_ctx *ctx;
+ int r;
ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
mutex_lock(&mgr->lock);
- r = idr_alloc(&mgr->ctx_handles, ctx, 0, 0, GFP_KERNEL);
+ r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
if (r < 0) {
mutex_unlock(&mgr->lock);
kfree(ctx);
return r;
}
*id = (uint32_t)r;
-
- memset(ctx, 0, sizeof(*ctx));
- ctx->id = *id;
- ctx->fpriv = fpriv;
- kref_init(&ctx->refcount);
+ r = amdgpu_ctx_init(adev, false, ctx);
mutex_unlock(&mgr->lock);
- return 0;
+ return r;
}
-int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t id)
+static void amdgpu_ctx_do_release(struct kref *ref)
{
struct amdgpu_ctx *ctx;
+
+ ctx = container_of(ref, struct amdgpu_ctx, refcount);
+
+ amdgpu_ctx_fini(ctx);
+
+ kfree(ctx);
+}
+
+static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
+{
struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
+ struct amdgpu_ctx *ctx;
mutex_lock(&mgr->lock);
ctx = idr_find(&mgr->ctx_handles, id);
if (ctx) {
+ idr_remove(&mgr->ctx_handles, id);
kref_put(&ctx->refcount, amdgpu_ctx_do_release);
mutex_unlock(&mgr->lock);
return 0;
@@ -86,9 +139,13 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
union drm_amdgpu_ctx_out *out)
{
struct amdgpu_ctx *ctx;
- struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
+ struct amdgpu_ctx_mgr *mgr;
unsigned reset_counter;
+ if (!fpriv)
+ return -EINVAL;
+
+ mgr = &fpriv->ctx_mgr;
mutex_lock(&mgr->lock);
ctx = idr_find(&mgr->ctx_handles, id);
if (!ctx) {
@@ -97,8 +154,8 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
}
/* TODO: these two are always zero */
- out->state.flags = ctx->state.flags;
- out->state.hangs = ctx->state.hangs;
+ out->state.flags = 0x0;
+ out->state.hangs = 0x0;
/* determine if a GPU reset has occured since the last call */
reset_counter = atomic_read(&adev->gpu_reset_counter);
@@ -113,28 +170,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
return 0;
}
-void amdgpu_ctx_fini(struct amdgpu_fpriv *fpriv)
-{
- struct idr *idp;
- struct amdgpu_ctx *ctx;
- uint32_t id;
- struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
- idp = &mgr->ctx_handles;
-
- idr_for_each_entry(idp,ctx,id) {
- if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1)
- DRM_ERROR("ctx (id=%ul) is still alive\n",ctx->id);
- }
-
- mutex_destroy(&mgr->lock);
-}
-
int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp)
{
int r;
uint32_t id;
- uint32_t flags;
union drm_amdgpu_ctx *args = data;
struct amdgpu_device *adev = dev->dev_private;
@@ -142,15 +182,14 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
r = 0;
id = args->in.ctx_id;
- flags = args->in.flags;
switch (args->in.op) {
case AMDGPU_CTX_OP_ALLOC_CTX:
- r = amdgpu_ctx_alloc(adev, fpriv, &id, flags);
+ r = amdgpu_ctx_alloc(adev, fpriv, &id);
args->out.alloc.ctx_id = id;
break;
case AMDGPU_CTX_OP_FREE_CTX:
- r = amdgpu_ctx_free(adev, fpriv, id);
+ r = amdgpu_ctx_free(fpriv, id);
break;
case AMDGPU_CTX_OP_QUERY_STATE:
r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
@@ -165,7 +204,12 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
{
struct amdgpu_ctx *ctx;
- struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
+ struct amdgpu_ctx_mgr *mgr;
+
+ if (!fpriv)
+ return NULL;
+
+ mgr = &fpriv->ctx_mgr;
mutex_lock(&mgr->lock);
ctx = idr_find(&mgr->ctx_handles, id);
@@ -177,17 +221,86 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
{
- struct amdgpu_fpriv *fpriv;
- struct amdgpu_ctx_mgr *mgr;
-
if (ctx == NULL)
return -EINVAL;
- fpriv = ctx->fpriv;
- mgr = &fpriv->ctx_mgr;
- mutex_lock(&mgr->lock);
kref_put(&ctx->refcount, amdgpu_ctx_do_release);
- mutex_unlock(&mgr->lock);
-
return 0;
}
+
+uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
+ struct fence *fence)
+{
+ struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
+ uint64_t seq = cring->sequence;
+ unsigned idx = 0;
+ struct fence *other = NULL;
+
+ idx = seq % AMDGPU_CTX_MAX_CS_PENDING;
+ other = cring->fences[idx];
+ if (other) {
+ signed long r;
+ r = fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
+ if (r < 0)
+ DRM_ERROR("Error (%ld) waiting for fence!\n", r);
+ }
+
+ fence_get(fence);
+
+ spin_lock(&ctx->ring_lock);
+ cring->fences[idx] = fence;
+ cring->sequence++;
+ spin_unlock(&ctx->ring_lock);
+
+ fence_put(other);
+
+ return seq;
+}
+
+struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
+ struct amdgpu_ring *ring, uint64_t seq)
+{
+ struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
+ struct fence *fence;
+
+ spin_lock(&ctx->ring_lock);
+
+ if (seq >= cring->sequence) {
+ spin_unlock(&ctx->ring_lock);
+ return ERR_PTR(-EINVAL);
+ }
+
+
+ if (seq + AMDGPU_CTX_MAX_CS_PENDING < cring->sequence) {
+ spin_unlock(&ctx->ring_lock);
+ return NULL;
+ }
+
+ fence = fence_get(cring->fences[seq % AMDGPU_CTX_MAX_CS_PENDING]);
+ spin_unlock(&ctx->ring_lock);
+
+ return fence;
+}
+
+void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
+{
+ mutex_init(&mgr->lock);
+ idr_init(&mgr->ctx_handles);
+}
+
+void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
+{
+ struct amdgpu_ctx *ctx;
+ struct idr *idp;
+ uint32_t id;
+
+ idp = &mgr->ctx_handles;
+
+ idr_for_each_entry(idp, ctx, id) {
+ if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1)
+ DRM_ERROR("ctx %p is still alive\n", ctx);
+ }
+
+ idr_destroy(&mgr->ctx_handles);
+ mutex_destroy(&mgr->lock);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 99f158e1baff..6ff6ae945794 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -55,6 +55,7 @@ static const char *amdgpu_asic_name[] = {
"MULLINS",
"TOPAZ",
"TONGA",
+ "FIJI",
"CARRIZO",
"LAST",
};
@@ -63,7 +64,7 @@ bool amdgpu_device_is_px(struct drm_device *dev)
{
struct amdgpu_device *adev = dev->dev_private;
- if (adev->flags & AMDGPU_IS_PX)
+ if (adev->flags & AMD_IS_PX)
return true;
return false;
}
@@ -243,7 +244,8 @@ static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
if (adev->vram_scratch.robj == NULL) {
r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
- PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, 0,
+ PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
NULL, &adev->vram_scratch.robj);
if (r) {
return r;
@@ -1160,6 +1162,7 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
switch (adev->asic_type) {
case CHIP_TOPAZ:
case CHIP_TONGA:
+ case CHIP_FIJI:
case CHIP_CARRIZO:
if (adev->asic_type == CHIP_CARRIZO)
adev->family = AMDGPU_FAMILY_CZ;
@@ -1377,7 +1380,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
adev->ddev = ddev;
adev->pdev = pdev;
adev->flags = flags;
- adev->asic_type = flags & AMDGPU_ASIC_MASK;
+ adev->asic_type = flags & AMD_ASIC_MASK;
adev->is_atom_bios = false;
adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
adev->mc.gtt_size = 512 * 1024 * 1024;
@@ -1523,6 +1526,11 @@ int amdgpu_device_init(struct amdgpu_device *adev,
return r;
}
+ r = amdgpu_ctx_init(adev, true, &adev->kernel_ctx);
+ if (r) {
+ dev_err(adev->dev, "failed to create kernel context (%d).\n", r);
+ return r;
+ }
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
@@ -1584,6 +1592,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
adev->shutdown = true;
/* evict vram memory */
amdgpu_bo_evict_vram(adev);
+ amdgpu_ctx_fini(&adev->kernel_ctx);
amdgpu_ib_pool_fini(adev);
amdgpu_fence_driver_fini(adev);
amdgpu_fbdev_fini(adev);
@@ -1627,8 +1636,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
struct amdgpu_device *adev;
struct drm_crtc *crtc;
struct drm_connector *connector;
- int i, r;
- bool force_completion = false;
+ int r;
if (dev == NULL || dev->dev_private == NULL) {
return -ENODEV;
@@ -1667,21 +1675,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
/* evict vram memory */
amdgpu_bo_evict_vram(adev);
- /* wait for gpu to finish processing current batch */
- for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
- struct amdgpu_ring *ring = adev->rings[i];
- if (!ring)
- continue;
-
- r = amdgpu_fence_wait_empty(ring);
- if (r) {
- /* delay GPU reset to resume */
- force_completion = true;
- }
- }
- if (force_completion) {
- amdgpu_fence_driver_force_completion(adev);
- }
+ amdgpu_fence_driver_suspend(adev);
r = amdgpu_suspend(adev);
@@ -1739,6 +1733,8 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
r = amdgpu_resume(adev);
+ amdgpu_fence_driver_resume(adev);
+
r = amdgpu_ib_ring_tests(adev);
if (r)
DRM_ERROR("ib ring test failed (%d).\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index b16b9256883e..e3d70772b531 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -35,6 +35,36 @@
#include <drm/drm_crtc_helper.h>
#include <drm/drm_edid.h>
+static void amdgpu_flip_wait_fence(struct amdgpu_device *adev,
+ struct fence **f)
+{
+ struct amdgpu_fence *fence;
+ long r;
+
+ if (*f == NULL)
+ return;
+
+ fence = to_amdgpu_fence(*f);
+ if (fence) {
+ r = fence_wait(&fence->base, false);
+ if (r == -EDEADLK) {
+ up_read(&adev->exclusive_lock);
+ r = amdgpu_gpu_reset(adev);
+ down_read(&adev->exclusive_lock);
+ }
+ } else
+ r = fence_wait(*f, false);
+
+ if (r)
+ DRM_ERROR("failed to wait on page flip fence (%ld)!\n", r);
+
+ /* We continue with the page flip even if we failed to wait on
+ * the fence, otherwise the DRM core and userspace will be
+ * confused about which BO the CRTC is scanning out
+ */
+ fence_put(*f);
+ *f = NULL;
+}
static void amdgpu_flip_work_func(struct work_struct *__work)
{
@@ -44,34 +74,13 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
struct amdgpu_crtc *amdgpuCrtc = adev->mode_info.crtcs[work->crtc_id];
struct drm_crtc *crtc = &amdgpuCrtc->base;
- struct amdgpu_fence *fence;
unsigned long flags;
- int r;
+ unsigned i;
down_read(&adev->exclusive_lock);
- if (work->fence) {
- fence = to_amdgpu_fence(work->fence);
- if (fence) {
- r = amdgpu_fence_wait(fence, false);
- if (r == -EDEADLK) {
- up_read(&adev->exclusive_lock);
- r = amdgpu_gpu_reset(adev);
- down_read(&adev->exclusive_lock);
- }
- } else
- r = fence_wait(work->fence, false);
-
- if (r)
- DRM_ERROR("failed to wait on page flip fence (%d)!\n", r);
-
- /* We continue with the page flip even if we failed to wait on
- * the fence, otherwise the DRM core and userspace will be
- * confused about which BO the CRTC is scanning out
- */
-
- fence_put(work->fence);
- work->fence = NULL;
- }
+ amdgpu_flip_wait_fence(adev, &work->excl);
+ for (i = 0; i < work->shared_count; ++i)
+ amdgpu_flip_wait_fence(adev, &work->shared[i]);
/* We borrow the event spin lock for protecting flip_status */
spin_lock_irqsave(&crtc->dev->event_lock, flags);
@@ -108,6 +117,7 @@ static void amdgpu_unpin_work_func(struct work_struct *__work)
DRM_ERROR("failed to reserve buffer after flip\n");
drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
+ kfree(work->shared);
kfree(work);
}
@@ -127,7 +137,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
unsigned long flags;
u64 tiling_flags;
u64 base;
- int r;
+ int i, r;
work = kzalloc(sizeof *work, GFP_KERNEL);
if (work == NULL)
@@ -167,7 +177,19 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
goto cleanup;
}
- work->fence = fence_get(reservation_object_get_excl(new_rbo->tbo.resv));
+ r = reservation_object_get_fences_rcu(new_rbo->tbo.resv, &work->excl,
+ &work->shared_count,
+ &work->shared);
+ if (unlikely(r != 0)) {
+ amdgpu_bo_unreserve(new_rbo);
+ DRM_ERROR("failed to get fences for buffer\n");
+ goto cleanup;
+ }
+
+ fence_get(work->excl);
+ for (i = 0; i < work->shared_count; ++i)
+ fence_get(work->shared[i]);
+
amdgpu_bo_get_tiling_flags(new_rbo, &tiling_flags);
amdgpu_bo_unreserve(new_rbo);
@@ -212,7 +234,10 @@ pflip_cleanup:
cleanup:
drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
- fence_put(work->fence);
+ fence_put(work->excl);
+ for (i = 0; i < work->shared_count; ++i)
+ fence_put(work->shared[i]);
+ kfree(work->shared);
kfree(work);
return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 56da962231fc..0fcc0bd1622c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -44,12 +44,15 @@
#include "amdgpu.h"
#include "amdgpu_irq.h"
+#include "amdgpu_amdkfd.h"
+
/*
* KMS wrapper.
* - 3.0.0 - initial driver
+ * - 3.1.0 - allow reading more status registers (GRBM, SRBM, SDMA, CP)
*/
#define KMS_DRIVER_MAJOR 3
-#define KMS_DRIVER_MINOR 0
+#define KMS_DRIVER_MINOR 1
#define KMS_DRIVER_PATCHLEVEL 0
int amdgpu_vram_limit = 0;
@@ -61,7 +64,7 @@ int amdgpu_disp_priority = 0;
int amdgpu_hw_i2c = 0;
int amdgpu_pcie_gen2 = -1;
int amdgpu_msi = -1;
-int amdgpu_lockup_timeout = 10000;
+int amdgpu_lockup_timeout = 0;
int amdgpu_dpm = -1;
int amdgpu_smc_load_fw = 1;
int amdgpu_aspm = -1;
@@ -73,6 +76,9 @@ int amdgpu_deep_color = 0;
int amdgpu_vm_size = 8;
int amdgpu_vm_block_size = -1;
int amdgpu_exp_hw_support = 0;
+int amdgpu_enable_scheduler = 0;
+int amdgpu_sched_jobs = 16;
+int amdgpu_sched_hw_submission = 2;
MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -101,7 +107,7 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(msi, amdgpu_msi, int, 0444);
-MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (defaul 10000 = 10 seconds, 0 = disable)");
+MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default 0 = disable)");
module_param_named(lockup_timeout, amdgpu_lockup_timeout, int, 0444);
MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)");
@@ -137,36 +143,45 @@ module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444);
MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
+MODULE_PARM_DESC(enable_scheduler, "enable SW GPU scheduler (1 = enable, 0 = disable ((default))");
+module_param_named(enable_scheduler, amdgpu_enable_scheduler, int, 0444);
+
+MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 16)");
+module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444);
+
+MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)");
+module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
+
static struct pci_device_id pciidlist[] = {
#ifdef CONFIG_DRM_AMDGPU_CIK
/* Kaveri */
- {0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x1305, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1306, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x1307, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1309, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x130F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1311, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1313, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1315, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1316, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x1317, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x1318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x131B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x131C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
- {0x1002, 0x131D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU},
+ {0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x1305, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1306, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x1307, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1309, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x130F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1311, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1313, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1315, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1316, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x1317, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x1318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x131B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x131C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
+ {0x1002, 0x131D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU},
/* Bonaire */
- {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY},
- {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY},
- {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY},
- {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY},
+ {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
+ {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
+ {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
+ {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
{0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
{0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
{0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
@@ -188,39 +203,39 @@ static struct pci_device_id pciidlist[] = {
{0x1002, 0x67BA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII},
{0x1002, 0x67BE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII},
/* Kabini */
- {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
- {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU},
+ {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
+ {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU},
/* mullins */
- {0x1002, 0x9850, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9851, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9852, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9853, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9854, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9855, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9856, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9857, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9858, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x9859, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
- {0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU},
+ {0x1002, 0x9850, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9851, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9852, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9853, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9854, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9855, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9856, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9857, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9858, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x9859, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
+ {0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU},
#endif
/* topaz */
{0x1002, 0x6900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ},
@@ -238,12 +253,14 @@ static struct pci_device_id pciidlist[] = {
{0x1002, 0x6930, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
{0x1002, 0x6938, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
{0x1002, 0x6939, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
+ /* fiji */
+ {0x1002, 0x7300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI},
/* carrizo */
- {0x1002, 0x9870, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU},
- {0x1002, 0x9874, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU},
- {0x1002, 0x9875, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU},
- {0x1002, 0x9876, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU},
- {0x1002, 0x9877, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU},
+ {0x1002, 0x9870, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
+ {0x1002, 0x9874, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
+ {0x1002, 0x9875, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
+ {0x1002, 0x9876, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
+ {0x1002, 0x9877, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
{0, 0, 0}
};
@@ -279,7 +296,7 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
unsigned long flags = ent->driver_data;
int ret;
- if ((flags & AMDGPU_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) {
+ if ((flags & AMD_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) {
DRM_INFO("This hardware requires experimental hardware support.\n"
"See modparam exp_hw_support\n");
return -ENODEV;
@@ -527,12 +544,15 @@ static int __init amdgpu_init(void)
driver->num_ioctls = amdgpu_max_kms_ioctl;
amdgpu_register_atpx_handler();
+ amdgpu_amdkfd_init();
+
/* let modprobe override vga console setting */
return drm_pci_init(driver, pdriver);
}
static void __exit amdgpu_exit(void)
{
+ amdgpu_amdkfd_fini();
drm_pci_exit(driver, pdriver);
amdgpu_unregister_atpx_handler();
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
index cceeb33c447a..e3a4f7048042 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h
@@ -31,7 +31,7 @@
#include <linux/firmware.h>
#include <linux/platform_device.h>
-#include "amdgpu_family.h"
+#include "amd_shared.h"
/* General customization:
*/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index c1645d21f8e2..8a122b1b7786 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -53,9 +53,9 @@ static struct fb_ops amdgpufb_ops = {
.owner = THIS_MODULE,
.fb_check_var = drm_fb_helper_check_var,
.fb_set_par = drm_fb_helper_set_par,
- .fb_fillrect = cfb_fillrect,
- .fb_copyarea = cfb_copyarea,
- .fb_imageblit = cfb_imageblit,
+ .fb_fillrect = drm_fb_helper_cfb_fillrect,
+ .fb_copyarea = drm_fb_helper_cfb_copyarea,
+ .fb_imageblit = drm_fb_helper_cfb_imageblit,
.fb_pan_display = drm_fb_helper_pan_display,
.fb_blank = drm_fb_helper_blank,
.fb_setcmap = drm_fb_helper_setcmap,
@@ -126,8 +126,8 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
aligned_size = ALIGN(size, PAGE_SIZE);
ret = amdgpu_gem_object_create(adev, aligned_size, 0,
AMDGPU_GEM_DOMAIN_VRAM,
- 0, true,
- &gobj);
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ true, &gobj);
if (ret) {
printk(KERN_ERR "failed to allocate framebuffer (%d)\n",
aligned_size);
@@ -179,7 +179,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
struct drm_mode_fb_cmd2 mode_cmd;
struct drm_gem_object *gobj = NULL;
struct amdgpu_bo *rbo = NULL;
- struct device *device = &adev->pdev->dev;
int ret;
unsigned long tmp;
@@ -201,9 +200,9 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
rbo = gem_to_amdgpu_bo(gobj);
/* okay we have an object now allocate the framebuffer */
- info = framebuffer_alloc(0, device);
- if (info == NULL) {
- ret = -ENOMEM;
+ info = drm_fb_helper_alloc_fbi(helper);
+ if (IS_ERR(info)) {
+ ret = PTR_ERR(info);
goto out_unref;
}
@@ -212,14 +211,13 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
ret = amdgpu_framebuffer_init(adev->ddev, &rfbdev->rfb, &mode_cmd, gobj);
if (ret) {
DRM_ERROR("failed to initialize framebuffer %d\n", ret);
- goto out_unref;
+ goto out_destroy_fbi;
}
fb = &rfbdev->rfb.base;
/* setup helper */
rfbdev->helper.fb = fb;
- rfbdev->helper.fbdev = info;
memset_io(rbo->kptr, 0x0, amdgpu_bo_size(rbo));
@@ -239,11 +237,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height);
/* setup aperture base/size for vesafb takeover */
- info->apertures = alloc_apertures(1);
- if (!info->apertures) {
- ret = -ENOMEM;
- goto out_unref;
- }
info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base;
info->apertures->ranges[0].size = adev->mc.aper_size;
@@ -251,13 +244,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
if (info->screen_base == NULL) {
ret = -ENOSPC;
- goto out_unref;
- }
-
- ret = fb_alloc_cmap(&info->cmap, 256, 0);
- if (ret) {
- ret = -ENOMEM;
- goto out_unref;
+ goto out_destroy_fbi;
}
DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start);
@@ -269,6 +256,8 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
vga_switcheroo_client_fb_set(adev->ddev->pdev, info);
return 0;
+out_destroy_fbi:
+ drm_fb_helper_release_fbi(helper);
out_unref:
if (rbo) {
@@ -290,17 +279,10 @@ void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev)
static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev)
{
- struct fb_info *info;
struct amdgpu_framebuffer *rfb = &rfbdev->rfb;
- if (rfbdev->helper.fbdev) {
- info = rfbdev->helper.fbdev;
-
- unregister_framebuffer(info);
- if (info->cmap.len)
- fb_dealloc_cmap(&info->cmap);
- framebuffer_release(info);
- }
+ drm_fb_helper_unregister_fbi(&rfbdev->helper);
+ drm_fb_helper_release_fbi(&rfbdev->helper);
if (rfb->obj) {
amdgpufb_destroy_pinned_object(rfb->obj);
@@ -395,7 +377,8 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev)
void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state)
{
if (adev->mode_info.rfbdev)
- fb_set_suspend(adev->mode_info.rfbdev->helper.fbdev, state);
+ drm_fb_helper_set_suspend(&adev->mode_info.rfbdev->helper,
+ state);
}
int amdgpu_fbdev_total_size(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index a7189a1fa6a1..1be2bd6d07ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -126,7 +126,8 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,
(*fence)->ring = ring;
(*fence)->owner = owner;
fence_init(&(*fence)->base, &amdgpu_fence_ops,
- &adev->fence_queue.lock, adev->fence_context + ring->idx,
+ &ring->fence_drv.fence_queue.lock,
+ adev->fence_context + ring->idx,
(*fence)->seq);
amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
(*fence)->seq,
@@ -136,38 +137,6 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,
}
/**
- * amdgpu_fence_recreate - recreate a fence from an user fence
- *
- * @ring: ring the fence is associated with
- * @owner: creator of the fence
- * @seq: user fence sequence number
- * @fence: resulting amdgpu fence object
- *
- * Recreates a fence command from the user fence sequence number (all asics).
- * Returns 0 on success, -ENOMEM on failure.
- */
-int amdgpu_fence_recreate(struct amdgpu_ring *ring, void *owner,
- uint64_t seq, struct amdgpu_fence **fence)
-{
- struct amdgpu_device *adev = ring->adev;
-
- if (seq > ring->fence_drv.sync_seq[ring->idx])
- return -EINVAL;
-
- *fence = kmalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
- if ((*fence) == NULL)
- return -ENOMEM;
-
- (*fence)->seq = seq;
- (*fence)->ring = ring;
- (*fence)->owner = owner;
- fence_init(&(*fence)->base, &amdgpu_fence_ops,
- &adev->fence_queue.lock, adev->fence_context + ring->idx,
- (*fence)->seq);
- return 0;
-}
-
-/**
* amdgpu_fence_check_signaled - callback from fence_queue
*
* this function is called with fence_queue lock held, which is also used
@@ -196,9 +165,7 @@ static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int fl
else
FENCE_TRACE(&fence->base, "was already signaled\n");
- amdgpu_irq_put(adev, fence->ring->fence_drv.irq_src,
- fence->ring->fence_drv.irq_type);
- __remove_wait_queue(&adev->fence_queue, &fence->fence_wake);
+ __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);
fence_put(&fence->base);
} else
FENCE_TRACE(&fence->base, "pending\n");
@@ -299,14 +266,9 @@ static void amdgpu_fence_check_lockup(struct work_struct *work)
return;
}
- if (fence_drv->delayed_irq && ring->adev->ddev->irq_enabled) {
- fence_drv->delayed_irq = false;
- amdgpu_irq_update(ring->adev, fence_drv->irq_src,
- fence_drv->irq_type);
+ if (amdgpu_fence_activity(ring)) {
+ wake_up_all(&ring->fence_drv.fence_queue);
}
-
- if (amdgpu_fence_activity(ring))
- wake_up_all(&ring->adev->fence_queue);
else if (amdgpu_ring_is_lockup(ring)) {
/* good news we believe it's a lockup */
dev_warn(ring->adev->dev, "GPU lockup (current fence id "
@@ -316,7 +278,7 @@ static void amdgpu_fence_check_lockup(struct work_struct *work)
/* remember that we need an reset */
ring->adev->needs_reset = true;
- wake_up_all(&ring->adev->fence_queue);
+ wake_up_all(&ring->fence_drv.fence_queue);
}
up_read(&ring->adev->exclusive_lock);
}
@@ -332,62 +294,8 @@ static void amdgpu_fence_check_lockup(struct work_struct *work)
*/
void amdgpu_fence_process(struct amdgpu_ring *ring)
{
- uint64_t seq, last_seq, last_emitted;
- unsigned count_loop = 0;
- bool wake = false;
-
- /* Note there is a scenario here for an infinite loop but it's
- * very unlikely to happen. For it to happen, the current polling
- * process need to be interrupted by another process and another
- * process needs to update the last_seq btw the atomic read and
- * xchg of the current process.
- *
- * More over for this to go in infinite loop there need to be
- * continuously new fence signaled ie amdgpu_fence_read needs
- * to return a different value each time for both the currently
- * polling process and the other process that xchg the last_seq
- * btw atomic read and xchg of the current process. And the
- * value the other process set as last seq must be higher than
- * the seq value we just read. Which means that current process
- * need to be interrupted after amdgpu_fence_read and before
- * atomic xchg.
- *
- * To be even more safe we count the number of time we loop and
- * we bail after 10 loop just accepting the fact that we might
- * have temporarly set the last_seq not to the true real last
- * seq but to an older one.
- */
- last_seq = atomic64_read(&ring->fence_drv.last_seq);
- do {
- last_emitted = ring->fence_drv.sync_seq[ring->idx];
- seq = amdgpu_fence_read(ring);
- seq |= last_seq & 0xffffffff00000000LL;
- if (seq < last_seq) {
- seq &= 0xffffffff;
- seq |= last_emitted & 0xffffffff00000000LL;
- }
-
- if (seq <= last_seq || seq > last_emitted) {
- break;
- }
- /* If we loop over we don't want to return without
- * checking if a fence is signaled as it means that the
- * seq we just read is different from the previous on.
- */
- wake = true;
- last_seq = seq;
- if ((count_loop++) > 10) {
- /* We looped over too many time leave with the
- * fact that we might have set an older fence
- * seq then the current real last seq as signaled
- * by the hw.
- */
- break;
- }
- } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq);
-
- if (wake)
- wake_up_all(&ring->adev->fence_queue);
+ if (amdgpu_fence_activity(ring))
+ wake_up_all(&ring->fence_drv.fence_queue);
}
/**
@@ -447,284 +355,49 @@ static bool amdgpu_fence_enable_signaling(struct fence *f)
{
struct amdgpu_fence *fence = to_amdgpu_fence(f);
struct amdgpu_ring *ring = fence->ring;
- struct amdgpu_device *adev = ring->adev;
if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)
return false;
- if (down_read_trylock(&adev->exclusive_lock)) {
- amdgpu_irq_get(adev, ring->fence_drv.irq_src,
- ring->fence_drv.irq_type);
- if (amdgpu_fence_activity(ring))
- wake_up_all_locked(&adev->fence_queue);
-
- /* did fence get signaled after we enabled the sw irq? */
- if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) {
- amdgpu_irq_put(adev, ring->fence_drv.irq_src,
- ring->fence_drv.irq_type);
- up_read(&adev->exclusive_lock);
- return false;
- }
-
- up_read(&adev->exclusive_lock);
- } else {
- /* we're probably in a lockup, lets not fiddle too much */
- if (amdgpu_irq_get_delayed(adev, ring->fence_drv.irq_src,
- ring->fence_drv.irq_type))
- ring->fence_drv.delayed_irq = true;
- amdgpu_fence_schedule_check(ring);
- }
-
fence->fence_wake.flags = 0;
fence->fence_wake.private = NULL;
fence->fence_wake.func = amdgpu_fence_check_signaled;
- __add_wait_queue(&adev->fence_queue, &fence->fence_wake);
+ __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);
fence_get(f);
FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
return true;
}
-/**
- * amdgpu_fence_signaled - check if a fence has signaled
- *
- * @fence: amdgpu fence object
- *
- * Check if the requested fence has signaled (all asics).
- * Returns true if the fence has signaled or false if it has not.
- */
-bool amdgpu_fence_signaled(struct amdgpu_fence *fence)
-{
- if (!fence)
- return true;
-
- if (amdgpu_fence_seq_signaled(fence->ring, fence->seq)) {
- if (!fence_signal(&fence->base))
- FENCE_TRACE(&fence->base, "signaled from amdgpu_fence_signaled\n");
- return true;
- }
-
- return false;
-}
-
-/**
- * amdgpu_fence_any_seq_signaled - check if any sequence number is signaled
- *
- * @adev: amdgpu device pointer
- * @seq: sequence numbers
- *
- * Check if the last signaled fence sequnce number is >= the requested
- * sequence number (all asics).
- * Returns true if any has signaled (current value is >= requested value)
- * or false if it has not. Helper function for amdgpu_fence_wait_seq.
- */
-static bool amdgpu_fence_any_seq_signaled(struct amdgpu_device *adev, u64 *seq)
-{
- unsigned i;
-
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- if (!adev->rings[i] || !seq[i])
- continue;
-
- if (amdgpu_fence_seq_signaled(adev->rings[i], seq[i]))
- return true;
- }
-
- return false;
-}
-
-/**
- * amdgpu_fence_wait_seq_timeout - wait for a specific sequence numbers
- *
- * @adev: amdgpu device pointer
- * @target_seq: sequence number(s) we want to wait for
- * @intr: use interruptable sleep
- * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait
- *
- * Wait for the requested sequence number(s) to be written by any ring
- * (all asics). Sequnce number array is indexed by ring id.
- * @intr selects whether to use interruptable (true) or non-interruptable
- * (false) sleep when waiting for the sequence number. Helper function
- * for amdgpu_fence_wait_*().
- * Returns remaining time if the sequence number has passed, 0 when
- * the wait timeout, or an error for all other cases.
- * -EDEADLK is returned when a GPU lockup has been detected.
- */
-static long amdgpu_fence_wait_seq_timeout(struct amdgpu_device *adev,
- u64 *target_seq, bool intr,
- long timeout)
-{
- uint64_t last_seq[AMDGPU_MAX_RINGS];
- bool signaled;
- int i;
- long r;
-
- if (timeout == 0) {
- return amdgpu_fence_any_seq_signaled(adev, target_seq);
- }
-
- while (!amdgpu_fence_any_seq_signaled(adev, target_seq)) {
-
- /* Save current sequence values, used to check for GPU lockups */
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_ring *ring = adev->rings[i];
-
- if (!ring || !target_seq[i])
- continue;
-
- last_seq[i] = atomic64_read(&ring->fence_drv.last_seq);
- trace_amdgpu_fence_wait_begin(adev->ddev, i, target_seq[i]);
- amdgpu_irq_get(adev, ring->fence_drv.irq_src,
- ring->fence_drv.irq_type);
- }
-
- if (intr) {
- r = wait_event_interruptible_timeout(adev->fence_queue, (
- (signaled = amdgpu_fence_any_seq_signaled(adev, target_seq))
- || adev->needs_reset), AMDGPU_FENCE_JIFFIES_TIMEOUT);
- } else {
- r = wait_event_timeout(adev->fence_queue, (
- (signaled = amdgpu_fence_any_seq_signaled(adev, target_seq))
- || adev->needs_reset), AMDGPU_FENCE_JIFFIES_TIMEOUT);
- }
-
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_ring *ring = adev->rings[i];
-
- if (!ring || !target_seq[i])
- continue;
-
- amdgpu_irq_put(adev, ring->fence_drv.irq_src,
- ring->fence_drv.irq_type);
- trace_amdgpu_fence_wait_end(adev->ddev, i, target_seq[i]);
- }
-
- if (unlikely(r < 0))
- return r;
-
- if (unlikely(!signaled)) {
-
- if (adev->needs_reset)
- return -EDEADLK;
-
- /* we were interrupted for some reason and fence
- * isn't signaled yet, resume waiting */
- if (r)
- continue;
-
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_ring *ring = adev->rings[i];
-
- if (!ring || !target_seq[i])
- continue;
-
- if (last_seq[i] != atomic64_read(&ring->fence_drv.last_seq))
- break;
- }
-
- if (i != AMDGPU_MAX_RINGS)
- continue;
-
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- if (!adev->rings[i] || !target_seq[i])
- continue;
-
- if (amdgpu_ring_is_lockup(adev->rings[i]))
- break;
- }
-
- if (i < AMDGPU_MAX_RINGS) {
- /* good news we believe it's a lockup */
- dev_warn(adev->dev, "GPU lockup (waiting for "
- "0x%016llx last fence id 0x%016llx on"
- " ring %d)\n",
- target_seq[i], last_seq[i], i);
-
- /* remember that we need an reset */
- adev->needs_reset = true;
- wake_up_all(&adev->fence_queue);
- return -EDEADLK;
- }
-
- if (timeout < MAX_SCHEDULE_TIMEOUT) {
- timeout -= AMDGPU_FENCE_JIFFIES_TIMEOUT;
- if (timeout <= 0) {
- return 0;
- }
- }
- }
- }
- return timeout;
-}
-
-/**
- * amdgpu_fence_wait - wait for a fence to signal
- *
- * @fence: amdgpu fence object
- * @intr: use interruptable sleep
- *
- * Wait for the requested fence to signal (all asics).
- * @intr selects whether to use interruptable (true) or non-interruptable
- * (false) sleep when waiting for the fence.
- * Returns 0 if the fence has passed, error for all other cases.
- */
-int amdgpu_fence_wait(struct amdgpu_fence *fence, bool intr)
-{
- uint64_t seq[AMDGPU_MAX_RINGS] = {};
- long r;
-
- seq[fence->ring->idx] = fence->seq;
- r = amdgpu_fence_wait_seq_timeout(fence->ring->adev, seq, intr, MAX_SCHEDULE_TIMEOUT);
- if (r < 0) {
- return r;
- }
-
- r = fence_signal(&fence->base);
- if (!r)
- FENCE_TRACE(&fence->base, "signaled from fence_wait\n");
- return 0;
-}
-
-/**
- * amdgpu_fence_wait_any - wait for a fence to signal on any ring
- *
- * @adev: amdgpu device pointer
- * @fences: amdgpu fence object(s)
- * @intr: use interruptable sleep
+/*
+ * amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal
+ * @ring: ring to wait on for the seq number
+ * @seq: seq number wait for
*
- * Wait for any requested fence to signal (all asics). Fence
- * array is indexed by ring id. @intr selects whether to use
- * interruptable (true) or non-interruptable (false) sleep when
- * waiting for the fences. Used by the suballocator.
- * Returns 0 if any fence has passed, error for all other cases.
+ * return value:
+ * 0: seq signaled, and gpu not hang
+ * -EDEADL: GPU hang detected
+ * -EINVAL: some paramter is not valid
*/
-int amdgpu_fence_wait_any(struct amdgpu_device *adev,
- struct amdgpu_fence **fences,
- bool intr)
+static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq)
{
- uint64_t seq[AMDGPU_MAX_RINGS];
- unsigned i, num_rings = 0;
- long r;
-
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- seq[i] = 0;
+ struct amdgpu_device *adev = ring->adev;
+ bool signaled = false;
- if (!fences[i]) {
- continue;
- }
+ BUG_ON(!ring);
+ if (seq > ring->fence_drv.sync_seq[ring->idx])
+ return -EINVAL;
- seq[i] = fences[i]->seq;
- ++num_rings;
- }
+ if (atomic64_read(&ring->fence_drv.last_seq) >= seq)
+ return 0;
- /* nothing to wait for ? */
- if (num_rings == 0)
- return -ENOENT;
+ wait_event(ring->fence_drv.fence_queue, (
+ (signaled = amdgpu_fence_seq_signaled(ring, seq))
+ || adev->needs_reset));
- r = amdgpu_fence_wait_seq_timeout(adev, seq, intr, MAX_SCHEDULE_TIMEOUT);
- if (r < 0) {
- return r;
- }
- return 0;
+ if (signaled)
+ return 0;
+ else
+ return -EDEADLK;
}
/**
@@ -739,19 +412,12 @@ int amdgpu_fence_wait_any(struct amdgpu_device *adev,
*/
int amdgpu_fence_wait_next(struct amdgpu_ring *ring)
{
- uint64_t seq[AMDGPU_MAX_RINGS] = {};
- long r;
+ uint64_t seq = atomic64_read(&ring->fence_drv.last_seq) + 1ULL;
- seq[ring->idx] = atomic64_read(&ring->fence_drv.last_seq) + 1ULL;
- if (seq[ring->idx] >= ring->fence_drv.sync_seq[ring->idx]) {
- /* nothing to wait for, last_seq is
- already the last emited fence */
+ if (seq >= ring->fence_drv.sync_seq[ring->idx])
return -ENOENT;
- }
- r = amdgpu_fence_wait_seq_timeout(ring->adev, seq, false, MAX_SCHEDULE_TIMEOUT);
- if (r < 0)
- return r;
- return 0;
+
+ return amdgpu_fence_ring_wait_seq(ring, seq);
}
/**
@@ -766,23 +432,12 @@ int amdgpu_fence_wait_next(struct amdgpu_ring *ring)
*/
int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
{
- struct amdgpu_device *adev = ring->adev;
- uint64_t seq[AMDGPU_MAX_RINGS] = {};
- long r;
+ uint64_t seq = ring->fence_drv.sync_seq[ring->idx];
- seq[ring->idx] = ring->fence_drv.sync_seq[ring->idx];
- if (!seq[ring->idx])
+ if (!seq)
return 0;
- r = amdgpu_fence_wait_seq_timeout(adev, seq, false, MAX_SCHEDULE_TIMEOUT);
- if (r < 0) {
- if (r == -EDEADLK)
- return -EDEADLK;
-
- dev_err(adev->dev, "error waiting for ring[%d] to become idle (%ld)\n",
- ring->idx, r);
- }
- return 0;
+ return amdgpu_fence_ring_wait_seq(ring, seq);
}
/**
@@ -933,9 +588,12 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;
}
amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq));
- ring->fence_drv.initialized = true;
+ amdgpu_irq_get(adev, irq_src, irq_type);
+
ring->fence_drv.irq_src = irq_src;
ring->fence_drv.irq_type = irq_type;
+ ring->fence_drv.initialized = true;
+
dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
"cpu addr 0x%p\n", ring->idx,
ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
@@ -966,6 +624,16 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
INIT_DELAYED_WORK(&ring->fence_drv.lockup_work,
amdgpu_fence_check_lockup);
ring->fence_drv.ring = ring;
+
+ if (amdgpu_enable_scheduler) {
+ ring->scheduler = amd_sched_create(&amdgpu_sched_ops,
+ ring->idx,
+ amdgpu_sched_hw_submission,
+ (void *)ring->adev);
+ if (!ring->scheduler)
+ DRM_ERROR("Failed to create scheduler on ring %d.\n",
+ ring->idx);
+ }
}
/**
@@ -982,7 +650,6 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
*/
int amdgpu_fence_driver_init(struct amdgpu_device *adev)
{
- init_waitqueue_head(&adev->fence_queue);
if (amdgpu_debugfs_fence_init(adev))
dev_err(adev->dev, "fence debugfs file creation failed\n");
@@ -1011,13 +678,78 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
/* no need to trigger GPU reset as we are unloading */
amdgpu_fence_driver_force_completion(adev);
}
- wake_up_all(&adev->fence_queue);
+ wake_up_all(&ring->fence_drv.fence_queue);
+ amdgpu_irq_put(adev, ring->fence_drv.irq_src,
+ ring->fence_drv.irq_type);
+ if (ring->scheduler)
+ amd_sched_destroy(ring->scheduler);
ring->fence_drv.initialized = false;
}
mutex_unlock(&adev->ring_lock);
}
/**
+ * amdgpu_fence_driver_suspend - suspend the fence driver
+ * for all possible rings.
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Suspend the fence driver for all possible rings (all asics).
+ */
+void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
+{
+ int i, r;
+
+ mutex_lock(&adev->ring_lock);
+ for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+ struct amdgpu_ring *ring = adev->rings[i];
+ if (!ring || !ring->fence_drv.initialized)
+ continue;
+
+ /* wait for gpu to finish processing current batch */
+ r = amdgpu_fence_wait_empty(ring);
+ if (r) {
+ /* delay GPU reset to resume */
+ amdgpu_fence_driver_force_completion(adev);
+ }
+
+ /* disable the interrupt */
+ amdgpu_irq_put(adev, ring->fence_drv.irq_src,
+ ring->fence_drv.irq_type);
+ }
+ mutex_unlock(&adev->ring_lock);
+}
+
+/**
+ * amdgpu_fence_driver_resume - resume the fence driver
+ * for all possible rings.
+ *
+ * @adev: amdgpu device pointer
+ *
+ * Resume the fence driver for all possible rings (all asics).
+ * Not all asics have all rings, so each asic will only
+ * start the fence driver on the rings it has using
+ * amdgpu_fence_driver_start_ring().
+ * Returns 0 for success.
+ */
+void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
+{
+ int i;
+
+ mutex_lock(&adev->ring_lock);
+ for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+ struct amdgpu_ring *ring = adev->rings[i];
+ if (!ring || !ring->fence_drv.initialized)
+ continue;
+
+ /* enable the interrupt */
+ amdgpu_irq_get(adev, ring->fence_drv.irq_src,
+ ring->fence_drv.irq_type);
+ }
+ mutex_unlock(&adev->ring_lock);
+}
+
+/**
* amdgpu_fence_driver_force_completion - force all fence waiter to complete
*
* @adev: amdgpu device pointer
@@ -1104,6 +836,21 @@ static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence)
return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);
}
+static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count)
+{
+ int idx;
+ struct fence *fence;
+
+ for (idx = 0; idx < count; ++idx) {
+ fence = fences[idx];
+ if (fence) {
+ if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+ return true;
+ }
+ }
+ return false;
+}
+
struct amdgpu_wait_cb {
struct fence_cb base;
struct task_struct *task;
@@ -1121,12 +868,48 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr,
{
struct amdgpu_fence *fence = to_amdgpu_fence(f);
struct amdgpu_device *adev = fence->ring->adev;
- struct amdgpu_wait_cb cb;
- cb.task = current;
+ return amdgpu_fence_wait_any(adev, &f, 1, intr, t);
+}
- if (fence_add_callback(f, &cb.base, amdgpu_fence_wait_cb))
- return t;
+/**
+ * Wait the fence array with timeout
+ *
+ * @adev: amdgpu device
+ * @array: the fence array with amdgpu fence pointer
+ * @count: the number of the fence array
+ * @intr: when sleep, set the current task interruptable or not
+ * @t: timeout to wait
+ *
+ * It will return when any fence is signaled or timeout.
+ */
+signed long amdgpu_fence_wait_any(struct amdgpu_device *adev,
+ struct fence **array, uint32_t count,
+ bool intr, signed long t)
+{
+ struct amdgpu_wait_cb *cb;
+ struct fence *fence;
+ unsigned idx;
+
+ BUG_ON(!array);
+
+ cb = kcalloc(count, sizeof(struct amdgpu_wait_cb), GFP_KERNEL);
+ if (cb == NULL) {
+ t = -ENOMEM;
+ goto err_free_cb;
+ }
+
+ for (idx = 0; idx < count; ++idx) {
+ fence = array[idx];
+ if (fence) {
+ cb[idx].task = current;
+ if (fence_add_callback(fence,
+ &cb[idx].base, amdgpu_fence_wait_cb)) {
+ /* The fence is already signaled */
+ goto fence_rm_cb;
+ }
+ }
+ }
while (t > 0) {
if (intr)
@@ -1135,10 +918,10 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr,
set_current_state(TASK_UNINTERRUPTIBLE);
/*
- * amdgpu_test_signaled must be called after
+ * amdgpu_test_signaled_any must be called after
* set_current_state to prevent a race with wake_up_process
*/
- if (amdgpu_test_signaled(fence))
+ if (amdgpu_test_signaled_any(array, count))
break;
if (adev->needs_reset) {
@@ -1153,7 +936,16 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr,
}
__set_current_state(TASK_RUNNING);
- fence_remove_callback(f, &cb.base);
+
+fence_rm_cb:
+ for (idx = 0; idx < count; ++idx) {
+ fence = array[idx];
+ if (fence && cb[idx].base.func)
+ fence_remove_callback(fence, &cb[idx].base);
+ }
+
+err_free_cb:
+ kfree(cb);
return t;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index e02db0b2e839..cbd3a486c5c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -125,7 +125,8 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
if (adev->gart.robj == NULL) {
r = amdgpu_bo_create(adev, adev->gart.table_size,
- PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, 0,
+ PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
NULL, &adev->gart.robj);
if (r) {
return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 4afc507820c0..5839fab374bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -615,6 +615,7 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
info.domains = robj->initial_domain;
info.domain_flags = robj->flags;
+ amdgpu_bo_unreserve(robj);
if (copy_to_user(out, &info, sizeof(info)))
r = -EFAULT;
break;
@@ -622,17 +623,19 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
case AMDGPU_GEM_OP_SET_PLACEMENT:
if (amdgpu_ttm_tt_has_userptr(robj->tbo.ttm)) {
r = -EPERM;
+ amdgpu_bo_unreserve(robj);
break;
}
robj->initial_domain = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
AMDGPU_GEM_DOMAIN_GTT |
AMDGPU_GEM_DOMAIN_CPU);
+ amdgpu_bo_unreserve(robj);
break;
default:
+ amdgpu_bo_unreserve(robj);
r = -EINVAL;
}
- amdgpu_bo_unreserve(robj);
out:
drm_gem_object_unreference_unlocked(gobj);
return r;
@@ -653,7 +656,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
r = amdgpu_gem_object_create(adev, args->size, 0,
AMDGPU_GEM_DOMAIN_VRAM,
- 0, ttm_bo_type_device,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ ttm_bo_type_device,
&gobj);
if (r)
return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index bc0fac618a3f..c439735ee670 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -73,28 +73,12 @@ int amdgpu_ib_get(struct amdgpu_ring *ring, struct amdgpu_vm *vm,
if (!vm)
ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
- else
- ib->gpu_addr = 0;
-
- } else {
- ib->sa_bo = NULL;
- ib->ptr = NULL;
- ib->gpu_addr = 0;
}
amdgpu_sync_create(&ib->sync);
ib->ring = ring;
- ib->fence = NULL;
- ib->user = NULL;
ib->vm = vm;
- ib->gds_base = 0;
- ib->gds_size = 0;
- ib->gws_base = 0;
- ib->gws_size = 0;
- ib->oa_base = 0;
- ib->oa_size = 0;
- ib->flags = 0;
return 0;
}
@@ -109,8 +93,8 @@ int amdgpu_ib_get(struct amdgpu_ring *ring, struct amdgpu_vm *vm,
*/
void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib)
{
- amdgpu_sync_free(adev, &ib->sync, ib->fence);
- amdgpu_sa_bo_free(adev, &ib->sa_bo, ib->fence);
+ amdgpu_sync_free(adev, &ib->sync, &ib->fence->base);
+ amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base);
amdgpu_fence_unref(&ib->fence);
}
@@ -156,7 +140,11 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs,
dev_err(adev->dev, "couldn't schedule ib\n");
return -EINVAL;
}
-
+ r = amdgpu_sync_wait(&ibs->sync);
+ if (r) {
+ dev_err(adev->dev, "IB sync failed (%d).\n", r);
+ return r;
+ }
r = amdgpu_ring_lock(ring, (256 + AMDGPU_NUM_SYNCS * 8) * num_ibs);
if (r) {
dev_err(adev->dev, "scheduling IB failed (%d).\n", r);
@@ -165,9 +153,11 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs,
if (vm) {
/* grab a vm id if necessary */
- struct amdgpu_fence *vm_id_fence = NULL;
- vm_id_fence = amdgpu_vm_grab_id(ibs->ring, ibs->vm);
- amdgpu_sync_fence(&ibs->sync, vm_id_fence);
+ r = amdgpu_vm_grab_id(ibs->vm, ibs->ring, &ibs->sync);
+ if (r) {
+ amdgpu_ring_unlock_undo(ring);
+ return r;
+ }
}
r = amdgpu_sync_rings(&ibs->sync, ring);
@@ -212,11 +202,15 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs,
return r;
}
+ if (!amdgpu_enable_scheduler && ib->ctx)
+ ib->sequence = amdgpu_ctx_add_fence(ib->ctx, ring,
+ &ib->fence->base);
+
/* wrap the last IB with fence */
if (ib->user) {
uint64_t addr = amdgpu_bo_gpu_offset(ib->user->bo);
addr += ib->user->offset;
- amdgpu_ring_emit_fence(ring, addr, ib->fence->seq,
+ amdgpu_ring_emit_fence(ring, addr, ib->sequence,
AMDGPU_FENCE_FLAG_64BIT);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index db5422e65ec5..5c8a803acedc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -24,6 +24,7 @@
#include <drm/drmP.h>
#include "amdgpu.h"
#include "amdgpu_ih.h"
+#include "amdgpu_amdkfd.h"
/**
* amdgpu_ih_ring_alloc - allocate memory for the IH ring
@@ -97,18 +98,12 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
/* add 8 bytes for the rptr/wptr shadows and
* add them to the end of the ring allocation.
*/
- adev->irq.ih.ring = kzalloc(adev->irq.ih.ring_size + 8, GFP_KERNEL);
+ adev->irq.ih.ring = pci_alloc_consistent(adev->pdev,
+ adev->irq.ih.ring_size + 8,
+ &adev->irq.ih.rb_dma_addr);
if (adev->irq.ih.ring == NULL)
return -ENOMEM;
- adev->irq.ih.rb_dma_addr = pci_map_single(adev->pdev,
- (void *)adev->irq.ih.ring,
- adev->irq.ih.ring_size,
- PCI_DMA_BIDIRECTIONAL);
- if (pci_dma_mapping_error(adev->pdev, adev->irq.ih.rb_dma_addr)) {
- dev_err(&adev->pdev->dev, "Failed to DMA MAP the IH RB page\n");
- kfree((void *)adev->irq.ih.ring);
- return -ENOMEM;
- }
+ memset((void *)adev->irq.ih.ring, 0, adev->irq.ih.ring_size + 8);
adev->irq.ih.wptr_offs = (adev->irq.ih.ring_size / 4) + 0;
adev->irq.ih.rptr_offs = (adev->irq.ih.ring_size / 4) + 1;
}
@@ -148,9 +143,9 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev)
/* add 8 bytes for the rptr/wptr shadows and
* add them to the end of the ring allocation.
*/
- pci_unmap_single(adev->pdev, adev->irq.ih.rb_dma_addr,
- adev->irq.ih.ring_size + 8, PCI_DMA_BIDIRECTIONAL);
- kfree((void *)adev->irq.ih.ring);
+ pci_free_consistent(adev->pdev, adev->irq.ih.ring_size + 8,
+ (void *)adev->irq.ih.ring,
+ adev->irq.ih.rb_dma_addr);
adev->irq.ih.ring = NULL;
}
} else {
@@ -199,6 +194,14 @@ restart_ih:
rmb();
while (adev->irq.ih.rptr != wptr) {
+ u32 ring_index = adev->irq.ih.rptr >> 2;
+
+ /* Before dispatching irq to IP blocks, send it to amdkfd */
+ amdgpu_amdkfd_interrupt(adev,
+ (const void *) &adev->irq.ih.ring[ring_index]);
+
+ entry.iv_entry = (const uint32_t *)
+ &adev->irq.ih.ring[ring_index];
amdgpu_ih_decode_iv(adev, &entry);
adev->irq.ih.rptr &= adev->irq.ih.ptr_mask;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index c62b09e555d6..ba38ae6a1463 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -52,6 +52,7 @@ struct amdgpu_iv_entry {
unsigned ring_id;
unsigned vm_id;
unsigned pas_id;
+ const uint32_t *iv_entry;
};
int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index b4d36f0f2153..0aba8e9bc8a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -272,6 +272,11 @@ void amdgpu_irq_fini(struct amdgpu_device *adev)
kfree(src->enabled_types);
src->enabled_types = NULL;
+ if (src->data) {
+ kfree(src->data);
+ kfree(src);
+ adev->irq.sources[i] = NULL;
+ }
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index 8299795f2b2d..17b01aef4278 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -40,6 +40,7 @@ struct amdgpu_irq_src {
unsigned num_types;
atomic_t *enabled_types;
const struct amdgpu_irq_src_funcs *funcs;
+ void *data;
};
/* provided by interrupt generating IP blocks */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 3bfe67de8349..22367939ebf1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -34,6 +34,7 @@
#include <linux/vga_switcheroo.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
+#include "amdgpu_amdkfd.h"
#if defined(CONFIG_VGA_SWITCHEROO)
bool amdgpu_has_atpx(void);
@@ -61,6 +62,8 @@ int amdgpu_driver_unload_kms(struct drm_device *dev)
pm_runtime_get_sync(dev->dev);
+ amdgpu_amdkfd_device_fini(adev);
+
amdgpu_acpi_fini(adev);
amdgpu_device_fini(adev);
@@ -93,8 +96,8 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
if ((amdgpu_runtime_pm != 0) &&
amdgpu_has_atpx() &&
- ((flags & AMDGPU_IS_APU) == 0))
- flags |= AMDGPU_IS_PX;
+ ((flags & AMD_IS_APU) == 0))
+ flags |= AMD_IS_PX;
/* amdgpu_device_init should report only fatal error
* like memory allocation failure or iomapping failure,
@@ -118,6 +121,10 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
"Error during ACPI methods call\n");
}
+ amdgpu_amdkfd_load_interface(adev);
+ amdgpu_amdkfd_device_probe(adev);
+ amdgpu_amdkfd_device_init(adev);
+
if (amdgpu_device_is_px(dev)) {
pm_runtime_use_autosuspend(dev->dev);
pm_runtime_set_autosuspend_delay(dev->dev, 5000);
@@ -444,11 +451,11 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
dev_info.num_hw_gfx_contexts = adev->gfx.config.max_hw_contexts;
dev_info._pad = 0;
dev_info.ids_flags = 0;
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
- dev_info.virtual_address_alignment = max(PAGE_SIZE, 0x10000UL);
+ dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
dev_info.pte_fragment_size = (1 << AMDGPU_LOG2_PAGES_PER_FRAG) *
AMDGPU_GPU_PAGE_SIZE;
dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE;
@@ -520,10 +527,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
mutex_init(&fpriv->bo_list_lock);
idr_init(&fpriv->bo_list_handles);
- /* init context manager */
- mutex_init(&fpriv->ctx_mgr.lock);
- idr_init(&fpriv->ctx_mgr.ctx_handles);
- fpriv->ctx_mgr.adev = adev;
+ amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
file_priv->driver_priv = fpriv;
@@ -556,6 +560,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
if (!fpriv)
return;
+ amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
+
amdgpu_vm_fini(adev, &fpriv->vm);
idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
@@ -564,9 +570,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
idr_destroy(&fpriv->bo_list_handles);
mutex_destroy(&fpriv->bo_list_lock);
- /* release context */
- amdgpu_ctx_fini(fpriv);
-
kfree(fpriv);
file_priv->driver_priv = NULL;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 8da64245b31b..08b09d55b96f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -127,7 +127,7 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
placements[c].fpfn =
adev->mc.visible_vram_size >> PAGE_SHIFT;
placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
- TTM_PL_FLAG_VRAM;
+ TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN;
}
placements[c].fpfn = 0;
placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
@@ -223,18 +223,6 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
size_t acc_size;
int r;
- /* VI has a hw bug where VM PTEs have to be allocated in groups of 8.
- * do this as a temporary workaround
- */
- if (!(domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA))) {
- if (adev->asic_type >= CHIP_TOPAZ) {
- if (byte_align & 0x7fff)
- byte_align = ALIGN(byte_align, 0x8000);
- if (size & 0x7fff)
- size = ALIGN(size, 0x8000);
- }
- }
-
page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
size = ALIGN(size, PAGE_SIZE);
@@ -462,7 +450,7 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
{
/* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
- if (0 && (adev->flags & AMDGPU_IS_APU)) {
+ if (0 && (adev->flags & AMD_IS_APU)) {
/* Useless to evict on IGP chips */
return 0;
}
@@ -478,7 +466,6 @@ void amdgpu_bo_force_delete(struct amdgpu_device *adev)
}
dev_err(adev->dev, "Userspace still has active objects !\n");
list_for_each_entry_safe(bo, n, &adev->gem.objects, list) {
- mutex_lock(&adev->ddev->struct_mutex);
dev_err(adev->dev, "%p %p %lu %lu force free\n",
&bo->gem_base, bo, (unsigned long)bo->gem_base.size,
*((unsigned long *)&bo->gem_base.refcount));
@@ -486,8 +473,7 @@ void amdgpu_bo_force_delete(struct amdgpu_device *adev)
list_del_init(&bo->list);
mutex_unlock(&bo->adev->gem.mutex);
/* this should unref the ttm bo */
- drm_gem_object_unreference(&bo->gem_base);
- mutex_unlock(&adev->ddev->struct_mutex);
+ drm_gem_object_unreference_unlocked(&bo->gem_base);
}
}
@@ -658,13 +644,13 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
* @shared: true if fence should be added shared
*
*/
-void amdgpu_bo_fence(struct amdgpu_bo *bo, struct amdgpu_fence *fence,
+void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
bool shared)
{
struct reservation_object *resv = bo->tbo.resv;
if (shared)
- reservation_object_add_shared_fence(resv, &fence->base);
+ reservation_object_add_shared_fence(resv, fence);
else
- reservation_object_add_excl_fence(resv, &fence->base);
+ reservation_object_add_excl_fence(resv, fence);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 675bdc30e41d..6ea18dcec561 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -161,7 +161,7 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
struct ttm_mem_reg *new_mem);
int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
-void amdgpu_bo_fence(struct amdgpu_bo *bo, struct amdgpu_fence *fence,
+void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
bool shared);
/*
@@ -193,7 +193,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev,
unsigned size, unsigned align);
void amdgpu_sa_bo_free(struct amdgpu_device *adev,
struct amdgpu_sa_bo **sa_bo,
- struct amdgpu_fence *fence);
+ struct fence *fence);
#if defined(CONFIG_DEBUG_FS)
void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
struct seq_file *m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index ed13baa7c976..efed11509f4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -82,7 +82,7 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev,
mutex_unlock(&adev->pm.mutex);
/* Can't set dpm state when the card is off */
- if (!(adev->flags & AMDGPU_IS_PX) ||
+ if (!(adev->flags & AMD_IS_PX) ||
(ddev->switch_power_state == DRM_SWITCH_POWER_ON))
amdgpu_pm_compute_clocks(adev);
fail:
@@ -538,7 +538,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
/* vce just modifies an existing state so force a change */
if (ps->vce_active != adev->pm.dpm.vce_active)
goto force;
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
/* for APUs if the num crtcs changed but state is the same,
* all we need to do is update the display configuration.
*/
@@ -580,7 +580,6 @@ force:
amdgpu_dpm_print_power_state(adev, adev->pm.dpm.requested_ps);
}
- mutex_lock(&adev->ddev->struct_mutex);
mutex_lock(&adev->ring_lock);
/* update whether vce is active */
@@ -628,7 +627,6 @@ force:
done:
mutex_unlock(&adev->ring_lock);
- mutex_unlock(&adev->ddev->struct_mutex);
}
void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 855e2196657a..9bec91484c24 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -131,6 +131,21 @@ int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw)
return 0;
}
+/** amdgpu_ring_insert_nop - insert NOP packets
+ *
+ * @ring: amdgpu_ring structure holding ring information
+ * @count: the number of NOP packets to insert
+ *
+ * This is the generic insert_nop function for rings except SDMA
+ */
+void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ int i;
+
+ for (i = 0; i < count; i++)
+ amdgpu_ring_write(ring, ring->nop);
+}
+
/**
* amdgpu_ring_commit - tell the GPU to execute the new
* commands on the ring buffer
@@ -143,10 +158,13 @@ int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw)
*/
void amdgpu_ring_commit(struct amdgpu_ring *ring)
{
+ uint32_t count;
+
/* We pad to match fetch size */
- while (ring->wptr & ring->align_mask) {
- amdgpu_ring_write(ring, ring->nop);
- }
+ count = ring->align_mask + 1 - (ring->wptr & ring->align_mask);
+ count %= ring->align_mask + 1;
+ ring->funcs->insert_nop(ring, count);
+
mb();
amdgpu_ring_set_wptr(ring);
}
@@ -342,6 +360,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
amdgpu_fence_driver_init_ring(ring);
}
+ init_waitqueue_head(&ring->fence_drv.fence_queue);
+
r = amdgpu_wb_get(adev, &ring->rptr_offs);
if (r) {
dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
@@ -367,7 +387,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
}
ring->next_rptr_gpu_addr = adev->wb.gpu_addr + (ring->next_rptr_offs * 4);
ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs];
-
+ spin_lock_init(&ring->fence_lock);
r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);
if (r) {
dev_err(adev->dev, "failed initializing fences (%d).\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index eb20987ce18d..74dad270362c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -139,6 +139,20 @@ int amdgpu_sa_bo_manager_suspend(struct amdgpu_device *adev,
return r;
}
+static uint32_t amdgpu_sa_get_ring_from_fence(struct fence *f)
+{
+ struct amdgpu_fence *a_fence;
+ struct amd_sched_fence *s_fence;
+
+ s_fence = to_amd_sched_fence(f);
+ if (s_fence)
+ return s_fence->scheduler->ring_id;
+ a_fence = to_amdgpu_fence(f);
+ if (a_fence)
+ return a_fence->ring->idx;
+ return 0;
+}
+
static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
{
struct amdgpu_sa_manager *sa_manager = sa_bo->manager;
@@ -147,7 +161,7 @@ static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
}
list_del_init(&sa_bo->olist);
list_del_init(&sa_bo->flist);
- amdgpu_fence_unref(&sa_bo->fence);
+ fence_put(sa_bo->fence);
kfree(sa_bo);
}
@@ -160,7 +174,8 @@ static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager)
sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist);
list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
- if (sa_bo->fence == NULL || !amdgpu_fence_signaled(sa_bo->fence)) {
+ if (sa_bo->fence == NULL ||
+ !fence_is_signaled(sa_bo->fence)) {
return;
}
amdgpu_sa_bo_remove_locked(sa_bo);
@@ -245,7 +260,7 @@ static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager,
}
static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
- struct amdgpu_fence **fences,
+ struct fence **fences,
unsigned *tries)
{
struct amdgpu_sa_bo *best_bo = NULL;
@@ -274,7 +289,7 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
sa_bo = list_first_entry(&sa_manager->flist[i],
struct amdgpu_sa_bo, flist);
- if (!amdgpu_fence_signaled(sa_bo->fence)) {
+ if (!fence_is_signaled(sa_bo->fence)) {
fences[i] = sa_bo->fence;
continue;
}
@@ -298,7 +313,8 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
}
if (best_bo) {
- ++tries[best_bo->fence->ring->idx];
+ uint32_t idx = amdgpu_sa_get_ring_from_fence(best_bo->fence);
+ ++tries[idx];
sa_manager->hole = best_bo->olist.prev;
/* we knew that this one is signaled,
@@ -314,9 +330,10 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev,
struct amdgpu_sa_bo **sa_bo,
unsigned size, unsigned align)
{
- struct amdgpu_fence *fences[AMDGPU_MAX_RINGS];
+ struct fence *fences[AMDGPU_MAX_RINGS];
unsigned tries[AMDGPU_MAX_RINGS];
int i, r;
+ signed long t;
BUG_ON(align > sa_manager->align);
BUG_ON(size > sa_manager->size);
@@ -350,7 +367,9 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev,
} while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries));
spin_unlock(&sa_manager->wq.lock);
- r = amdgpu_fence_wait_any(adev, fences, false);
+ t = amdgpu_fence_wait_any(adev, fences, AMDGPU_MAX_RINGS,
+ false, MAX_SCHEDULE_TIMEOUT);
+ r = (t > 0) ? 0 : t;
spin_lock(&sa_manager->wq.lock);
/* if we have nothing to wait for block */
if (r == -ENOENT) {
@@ -369,7 +388,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev,
}
void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
- struct amdgpu_fence *fence)
+ struct fence *fence)
{
struct amdgpu_sa_manager *sa_manager;
@@ -379,10 +398,11 @@ void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
sa_manager = (*sa_bo)->manager;
spin_lock(&sa_manager->wq.lock);
- if (fence && !amdgpu_fence_signaled(fence)) {
- (*sa_bo)->fence = amdgpu_fence_ref(fence);
- list_add_tail(&(*sa_bo)->flist,
- &sa_manager->flist[fence->ring->idx]);
+ if (fence && !fence_is_signaled(fence)) {
+ uint32_t idx;
+ (*sa_bo)->fence = fence_get(fence);
+ idx = amdgpu_sa_get_ring_from_fence(fence);
+ list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
} else {
amdgpu_sa_bo_remove_locked(*sa_bo);
}
@@ -409,8 +429,16 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
seq_printf(m, "[0x%010llx 0x%010llx] size %8lld",
soffset, eoffset, eoffset - soffset);
if (i->fence) {
- seq_printf(m, " protected by 0x%016llx on ring %d",
- i->fence->seq, i->fence->ring->idx);
+ struct amdgpu_fence *a_fence = to_amdgpu_fence(i->fence);
+ struct amd_sched_fence *s_fence = to_amd_sched_fence(i->fence);
+ if (a_fence)
+ seq_printf(m, " protected by 0x%016llx on ring %d",
+ a_fence->seq, a_fence->ring->idx);
+ if (s_fence)
+ seq_printf(m, " protected by 0x%016x on ring %d",
+ s_fence->base.seqno,
+ s_fence->scheduler->ring_id);
+
}
seq_printf(m, "\n");
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
new file mode 100644
index 000000000000..de98fbd2971e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+
+static struct fence *amdgpu_sched_dependency(struct amd_sched_job *job)
+{
+ struct amdgpu_job *sched_job = (struct amdgpu_job *)job;
+ return amdgpu_sync_get_fence(&sched_job->ibs->sync);
+}
+
+static struct fence *amdgpu_sched_run_job(struct amd_sched_job *job)
+{
+ struct amdgpu_job *sched_job;
+ struct amdgpu_fence *fence;
+ int r;
+
+ if (!job) {
+ DRM_ERROR("job is null\n");
+ return NULL;
+ }
+ sched_job = (struct amdgpu_job *)job;
+ mutex_lock(&sched_job->job_lock);
+ r = amdgpu_ib_schedule(sched_job->adev,
+ sched_job->num_ibs,
+ sched_job->ibs,
+ sched_job->base.owner);
+ if (r)
+ goto err;
+ fence = amdgpu_fence_ref(sched_job->ibs[sched_job->num_ibs - 1].fence);
+
+ if (sched_job->free_job)
+ sched_job->free_job(sched_job);
+
+ mutex_unlock(&sched_job->job_lock);
+ return &fence->base;
+
+err:
+ DRM_ERROR("Run job error\n");
+ mutex_unlock(&sched_job->job_lock);
+ job->sched->ops->process_job(job);
+ return NULL;
+}
+
+static void amdgpu_sched_process_job(struct amd_sched_job *job)
+{
+ struct amdgpu_job *sched_job;
+
+ if (!job) {
+ DRM_ERROR("job is null\n");
+ return;
+ }
+ sched_job = (struct amdgpu_job *)job;
+ /* after processing job, free memory */
+ fence_put(&sched_job->base.s_fence->base);
+ kfree(sched_job);
+}
+
+struct amd_sched_backend_ops amdgpu_sched_ops = {
+ .dependency = amdgpu_sched_dependency,
+ .run_job = amdgpu_sched_run_job,
+ .process_job = amdgpu_sched_process_job
+};
+
+int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev,
+ struct amdgpu_ring *ring,
+ struct amdgpu_ib *ibs,
+ unsigned num_ibs,
+ int (*free_job)(struct amdgpu_job *),
+ void *owner,
+ struct fence **f)
+{
+ int r = 0;
+ if (amdgpu_enable_scheduler) {
+ struct amdgpu_job *job =
+ kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL);
+ if (!job)
+ return -ENOMEM;
+ job->base.sched = ring->scheduler;
+ job->base.s_entity = &adev->kernel_ctx.rings[ring->idx].entity;
+ job->adev = adev;
+ job->ibs = ibs;
+ job->num_ibs = num_ibs;
+ job->base.owner = owner;
+ mutex_init(&job->job_lock);
+ job->free_job = free_job;
+ mutex_lock(&job->job_lock);
+ r = amd_sched_entity_push_job((struct amd_sched_job *)job);
+ if (r) {
+ mutex_unlock(&job->job_lock);
+ kfree(job);
+ return r;
+ }
+ *f = fence_get(&job->base.s_fence->base);
+ mutex_unlock(&job->job_lock);
+ } else {
+ r = amdgpu_ib_schedule(adev, num_ibs, ibs, owner);
+ if (r)
+ return r;
+ *f = fence_get(&ibs[num_ibs - 1].fence->base);
+ }
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c
index d6d41a42ab65..ff3ca52ec6fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c
@@ -87,7 +87,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring,
void amdgpu_semaphore_free(struct amdgpu_device *adev,
struct amdgpu_semaphore **semaphore,
- struct amdgpu_fence *fence)
+ struct fence *fence)
{
if (semaphore == NULL || *semaphore == NULL) {
return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index 21accbdd0a1a..068aeaff7183 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -32,6 +32,11 @@
#include "amdgpu.h"
#include "amdgpu_trace.h"
+struct amdgpu_sync_entry {
+ struct hlist_node node;
+ struct fence *fence;
+};
+
/**
* amdgpu_sync_create - zero init sync object
*
@@ -49,36 +54,104 @@ void amdgpu_sync_create(struct amdgpu_sync *sync)
for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
sync->sync_to[i] = NULL;
+ hash_init(sync->fences);
sync->last_vm_update = NULL;
}
+static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f)
+{
+ struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
+ struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+
+ if (a_fence)
+ return a_fence->ring->adev == adev;
+ if (s_fence)
+ return (struct amdgpu_device *)s_fence->scheduler->priv == adev;
+ return false;
+}
+
+static bool amdgpu_sync_test_owner(struct fence *f, void *owner)
+{
+ struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
+ struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+ if (s_fence)
+ return s_fence->owner == owner;
+ if (a_fence)
+ return a_fence->owner == owner;
+ return false;
+}
+
/**
- * amdgpu_sync_fence - use the semaphore to sync to a fence
+ * amdgpu_sync_fence - remember to sync to this fence
*
* @sync: sync object to add fence to
* @fence: fence to sync to
*
- * Sync to the fence using the semaphore objects
*/
-void amdgpu_sync_fence(struct amdgpu_sync *sync,
- struct amdgpu_fence *fence)
+int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
+ struct fence *f)
{
+ struct amdgpu_sync_entry *e;
+ struct amdgpu_fence *fence;
struct amdgpu_fence *other;
+ struct fence *tmp, *later;
- if (!fence)
- return;
+ if (!f)
+ return 0;
+
+ if (amdgpu_sync_same_dev(adev, f) &&
+ amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) {
+ if (sync->last_vm_update) {
+ tmp = sync->last_vm_update;
+ BUG_ON(f->context != tmp->context);
+ later = (f->seqno - tmp->seqno <= INT_MAX) ? f : tmp;
+ sync->last_vm_update = fence_get(later);
+ fence_put(tmp);
+ } else
+ sync->last_vm_update = fence_get(f);
+ }
+
+ fence = to_amdgpu_fence(f);
+ if (!fence || fence->ring->adev != adev) {
+ hash_for_each_possible(sync->fences, e, node, f->context) {
+ struct fence *new;
+ if (unlikely(e->fence->context != f->context))
+ continue;
+ new = fence_get(fence_later(e->fence, f));
+ if (new) {
+ fence_put(e->fence);
+ e->fence = new;
+ }
+ return 0;
+ }
+
+ e = kmalloc(sizeof(struct amdgpu_sync_entry), GFP_KERNEL);
+ if (!e)
+ return -ENOMEM;
+
+ hash_add(sync->fences, &e->node, f->context);
+ e->fence = fence_get(f);
+ return 0;
+ }
other = sync->sync_to[fence->ring->idx];
sync->sync_to[fence->ring->idx] = amdgpu_fence_ref(
amdgpu_fence_later(fence, other));
amdgpu_fence_unref(&other);
- if (fence->owner == AMDGPU_FENCE_OWNER_VM) {
- other = sync->last_vm_update;
- sync->last_vm_update = amdgpu_fence_ref(
- amdgpu_fence_later(fence, other));
- amdgpu_fence_unref(&other);
- }
+ return 0;
+}
+
+static void *amdgpu_sync_get_owner(struct fence *f)
+{
+ struct amdgpu_fence *a_fence = to_amdgpu_fence(f);
+ struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+
+ if (s_fence)
+ return s_fence->owner;
+ else if (a_fence)
+ return a_fence->owner;
+ return AMDGPU_FENCE_OWNER_UNDEFINED;
}
/**
@@ -97,7 +170,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
{
struct reservation_object_list *flist;
struct fence *f;
- struct amdgpu_fence *fence;
+ void *fence_owner;
unsigned i;
int r = 0;
@@ -106,11 +179,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
/* always sync to the exclusive fence */
f = reservation_object_get_excl(resv);
- fence = f ? to_amdgpu_fence(f) : NULL;
- if (fence && fence->ring->adev == adev)
- amdgpu_sync_fence(sync, fence);
- else if (f)
- r = fence_wait(f, true);
+ r = amdgpu_sync_fence(adev, sync, f);
flist = reservation_object_get_list(resv);
if (!flist || r)
@@ -119,20 +188,72 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
for (i = 0; i < flist->shared_count; ++i) {
f = rcu_dereference_protected(flist->shared[i],
reservation_object_held(resv));
- fence = f ? to_amdgpu_fence(f) : NULL;
- if (fence && fence->ring->adev == adev) {
- if (fence->owner != owner ||
- fence->owner == AMDGPU_FENCE_OWNER_UNDEFINED)
- amdgpu_sync_fence(sync, fence);
- } else if (f) {
- r = fence_wait(f, true);
- if (r)
- break;
+ if (amdgpu_sync_same_dev(adev, f)) {
+ /* VM updates are only interesting
+ * for other VM updates and moves.
+ */
+ fence_owner = amdgpu_sync_get_owner(f);
+ if ((owner != AMDGPU_FENCE_OWNER_MOVE) &&
+ (fence_owner != AMDGPU_FENCE_OWNER_MOVE) &&
+ ((owner == AMDGPU_FENCE_OWNER_VM) !=
+ (fence_owner == AMDGPU_FENCE_OWNER_VM)))
+ continue;
+
+ /* Ignore fence from the same owner as
+ * long as it isn't undefined.
+ */
+ if (owner != AMDGPU_FENCE_OWNER_UNDEFINED &&
+ fence_owner == owner)
+ continue;
}
+
+ r = amdgpu_sync_fence(adev, sync, f);
+ if (r)
+ break;
}
return r;
}
+struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
+{
+ struct amdgpu_sync_entry *e;
+ struct hlist_node *tmp;
+ struct fence *f;
+ int i;
+
+ hash_for_each_safe(sync->fences, i, tmp, e, node) {
+
+ f = e->fence;
+
+ hash_del(&e->node);
+ kfree(e);
+
+ if (!fence_is_signaled(f))
+ return f;
+
+ fence_put(f);
+ }
+ return NULL;
+}
+
+int amdgpu_sync_wait(struct amdgpu_sync *sync)
+{
+ struct amdgpu_sync_entry *e;
+ struct hlist_node *tmp;
+ int i, r;
+
+ hash_for_each_safe(sync->fences, i, tmp, e, node) {
+ r = fence_wait(e->fence, false);
+ if (r)
+ return r;
+
+ hash_del(&e->node);
+ fence_put(e->fence);
+ kfree(e);
+ }
+ return 0;
+}
+
/**
* amdgpu_sync_rings - sync ring to all registered fences
*
@@ -164,9 +285,9 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
return -EINVAL;
}
- if (count >= AMDGPU_NUM_SYNCS) {
+ if (amdgpu_enable_scheduler || (count >= AMDGPU_NUM_SYNCS)) {
/* not enough room, wait manually */
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(&fence->base, false);
if (r)
return r;
continue;
@@ -186,7 +307,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
if (!amdgpu_semaphore_emit_signal(other, semaphore)) {
/* signaling wasn't successful wait manually */
amdgpu_ring_undo(other);
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(&fence->base, false);
if (r)
return r;
continue;
@@ -196,7 +317,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
if (!amdgpu_semaphore_emit_wait(ring, semaphore)) {
/* waiting wasn't successful wait manually */
amdgpu_ring_undo(other);
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(&fence->base, false);
if (r)
return r;
continue;
@@ -220,15 +341,23 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync,
*/
void amdgpu_sync_free(struct amdgpu_device *adev,
struct amdgpu_sync *sync,
- struct amdgpu_fence *fence)
+ struct fence *fence)
{
+ struct amdgpu_sync_entry *e;
+ struct hlist_node *tmp;
unsigned i;
+ hash_for_each_safe(sync->fences, i, tmp, e, node) {
+ hash_del(&e->node);
+ fence_put(e->fence);
+ kfree(e);
+ }
+
for (i = 0; i < AMDGPU_NUM_SYNCS; ++i)
amdgpu_semaphore_free(adev, &sync->semaphores[i], fence);
for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
amdgpu_fence_unref(&sync->sync_to[i]);
- amdgpu_fence_unref(&sync->last_vm_update);
+ fence_put(sync->last_vm_update);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index df202999fbfe..f80b1a43be8a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -77,7 +77,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
void *gtt_map, *vram_map;
void **gtt_start, **gtt_end;
void **vram_start, **vram_end;
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
AMDGPU_GEM_DOMAIN_GTT, 0, NULL, gtt_obj + i);
@@ -116,13 +116,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
goto out_lclean_unpin;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i);
goto out_lclean_unpin;
}
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
r = amdgpu_bo_kmap(vram_obj, &vram_map);
if (r) {
@@ -161,13 +161,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
goto out_lclean_unpin;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i);
goto out_lclean_unpin;
}
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
r = amdgpu_bo_kmap(gtt_obj[i], &gtt_map);
if (r) {
@@ -214,7 +214,7 @@ out_lclean:
amdgpu_bo_unref(&gtt_obj[i]);
}
if (fence)
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
break;
}
@@ -238,7 +238,7 @@ void amdgpu_test_moves(struct amdgpu_device *adev)
static int amdgpu_test_create_and_emit_fence(struct amdgpu_device *adev,
struct amdgpu_ring *ring,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
uint32_t handle = ring->idx ^ 0xdeafbeef;
int r;
@@ -269,15 +269,16 @@ static int amdgpu_test_create_and_emit_fence(struct amdgpu_device *adev,
DRM_ERROR("Failed to get dummy destroy msg\n");
return r;
}
-
} else {
+ struct amdgpu_fence *a_fence = NULL;
r = amdgpu_ring_lock(ring, 64);
if (r) {
DRM_ERROR("Failed to lock ring A %d\n", ring->idx);
return r;
}
- amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_UNDEFINED, fence);
+ amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_UNDEFINED, &a_fence);
amdgpu_ring_unlock_commit(ring);
+ *fence = &a_fence->base;
}
return 0;
}
@@ -286,7 +287,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev,
struct amdgpu_ring *ringA,
struct amdgpu_ring *ringB)
{
- struct amdgpu_fence *fence1 = NULL, *fence2 = NULL;
+ struct fence *fence1 = NULL, *fence2 = NULL;
struct amdgpu_semaphore *semaphore = NULL;
int r;
@@ -322,7 +323,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev,
mdelay(1000);
- if (amdgpu_fence_signaled(fence1)) {
+ if (fence_is_signaled(fence1)) {
DRM_ERROR("Fence 1 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
@@ -335,7 +336,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev,
amdgpu_semaphore_emit_signal(ringB, semaphore);
amdgpu_ring_unlock_commit(ringB);
- r = amdgpu_fence_wait(fence1, false);
+ r = fence_wait(fence1, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
@@ -343,7 +344,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev,
mdelay(1000);
- if (amdgpu_fence_signaled(fence2)) {
+ if (fence_is_signaled(fence2)) {
DRM_ERROR("Fence 2 signaled without waiting for semaphore.\n");
goto out_cleanup;
}
@@ -356,7 +357,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev,
amdgpu_semaphore_emit_signal(ringB, semaphore);
amdgpu_ring_unlock_commit(ringB);
- r = amdgpu_fence_wait(fence2, false);
+ r = fence_wait(fence2, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence 1\n");
goto out_cleanup;
@@ -366,10 +367,10 @@ out_cleanup:
amdgpu_semaphore_free(adev, &semaphore, NULL);
if (fence1)
- amdgpu_fence_unref(&fence1);
+ fence_put(fence1);
if (fence2)
- amdgpu_fence_unref(&fence2);
+ fence_put(fence2);
if (r)
printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
@@ -380,7 +381,7 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev,
struct amdgpu_ring *ringB,
struct amdgpu_ring *ringC)
{
- struct amdgpu_fence *fenceA = NULL, *fenceB = NULL;
+ struct fence *fenceA = NULL, *fenceB = NULL;
struct amdgpu_semaphore *semaphore = NULL;
bool sigA, sigB;
int i, r;
@@ -416,11 +417,11 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev,
mdelay(1000);
- if (amdgpu_fence_signaled(fenceA)) {
+ if (fence_is_signaled(fenceA)) {
DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
goto out_cleanup;
}
- if (amdgpu_fence_signaled(fenceB)) {
+ if (fence_is_signaled(fenceB)) {
DRM_ERROR("Fence B signaled without waiting for semaphore.\n");
goto out_cleanup;
}
@@ -435,8 +436,8 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev,
for (i = 0; i < 30; ++i) {
mdelay(100);
- sigA = amdgpu_fence_signaled(fenceA);
- sigB = amdgpu_fence_signaled(fenceB);
+ sigA = fence_is_signaled(fenceA);
+ sigB = fence_is_signaled(fenceB);
if (sigA || sigB)
break;
}
@@ -461,12 +462,12 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev,
mdelay(1000);
- r = amdgpu_fence_wait(fenceA, false);
+ r = fence_wait(fenceA, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence A\n");
goto out_cleanup;
}
- r = amdgpu_fence_wait(fenceB, false);
+ r = fence_wait(fenceB, false);
if (r) {
DRM_ERROR("Failed to wait for sync fence B\n");
goto out_cleanup;
@@ -476,10 +477,10 @@ out_cleanup:
amdgpu_semaphore_free(adev, &semaphore, NULL);
if (fenceA)
- amdgpu_fence_unref(&fenceA);
+ fence_put(fenceA);
if (fenceB)
- amdgpu_fence_unref(&fenceB);
+ fence_put(fenceB);
if (r)
printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index dd3415d2e45d..b5abd5cde413 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -228,7 +228,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
struct amdgpu_device *adev;
struct amdgpu_ring *ring;
uint64_t old_start, new_start;
- struct amdgpu_fence *fence;
+ struct fence *fence;
int r;
adev = amdgpu_get_adev(bo->bdev);
@@ -269,9 +269,9 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
new_mem->num_pages * PAGE_SIZE, /* bytes */
bo->resv, &fence);
/* FIXME: handle copy error */
- r = ttm_bo_move_accel_cleanup(bo, &fence->base,
+ r = ttm_bo_move_accel_cleanup(bo, fence,
evict, no_wait_gpu, new_mem);
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
return r;
}
@@ -859,7 +859,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
NULL, &adev->stollen_vga_memory);
if (r) {
return r;
@@ -987,46 +988,48 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
uint64_t dst_offset,
uint32_t byte_count,
struct reservation_object *resv,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
struct amdgpu_device *adev = ring->adev;
- struct amdgpu_sync sync;
uint32_t max_bytes;
unsigned num_loops, num_dw;
+ struct amdgpu_ib *ib;
unsigned i;
int r;
- /* sync other rings */
- amdgpu_sync_create(&sync);
- if (resv) {
- r = amdgpu_sync_resv(adev, &sync, resv, false);
- if (r) {
- DRM_ERROR("sync failed (%d).\n", r);
- amdgpu_sync_free(adev, &sync, NULL);
- return r;
- }
- }
-
max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
num_loops = DIV_ROUND_UP(byte_count, max_bytes);
num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
- /* for fence and sync */
- num_dw += 64 + AMDGPU_NUM_SYNCS * 8;
+ /* for IB padding */
+ while (num_dw & 0x7)
+ num_dw++;
+
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
+ return -ENOMEM;
- r = amdgpu_ring_lock(ring, num_dw);
+ r = amdgpu_ib_get(ring, NULL, num_dw * 4, ib);
if (r) {
- DRM_ERROR("ring lock failed (%d).\n", r);
- amdgpu_sync_free(adev, &sync, NULL);
+ kfree(ib);
return r;
}
- amdgpu_sync_rings(&sync, ring);
+ ib->length_dw = 0;
+
+ if (resv) {
+ r = amdgpu_sync_resv(adev, &ib->sync, resv,
+ AMDGPU_FENCE_OWNER_UNDEFINED);
+ if (r) {
+ DRM_ERROR("sync failed (%d).\n", r);
+ goto error_free;
+ }
+ }
for (i = 0; i < num_loops; i++) {
uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
- amdgpu_emit_copy_buffer(adev, ring, src_offset, dst_offset,
+ amdgpu_emit_copy_buffer(adev, ib, src_offset, dst_offset,
cur_size_in_bytes);
src_offset += cur_size_in_bytes;
@@ -1034,17 +1037,24 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
byte_count -= cur_size_in_bytes;
}
- r = amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_MOVE, fence);
- if (r) {
- amdgpu_ring_unlock_undo(ring);
- amdgpu_sync_free(adev, &sync, NULL);
- return r;
- }
-
- amdgpu_ring_unlock_commit(ring);
- amdgpu_sync_free(adev, &sync, *fence);
+ amdgpu_vm_pad_ib(adev, ib);
+ WARN_ON(ib->length_dw > num_dw);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vm_free_job,
+ AMDGPU_FENCE_OWNER_MOVE,
+ fence);
+ if (r)
+ goto error_free;
+ if (!amdgpu_enable_scheduler) {
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
+ }
return 0;
+error_free:
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
+ return r;
}
#if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index f5c22556ec2c..2cf6c6b06e3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -52,6 +52,7 @@
#endif
#define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin"
#define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin"
+#define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin"
/**
* amdgpu_uvd_cs_ctx - Command submission parser context
@@ -81,6 +82,7 @@ MODULE_FIRMWARE(FIRMWARE_MULLINS);
#endif
MODULE_FIRMWARE(FIRMWARE_TONGA);
MODULE_FIRMWARE(FIRMWARE_CARRIZO);
+MODULE_FIRMWARE(FIRMWARE_FIJI);
static void amdgpu_uvd_note_usage(struct amdgpu_device *adev);
static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
@@ -116,6 +118,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
case CHIP_TONGA:
fw_name = FIRMWARE_TONGA;
break;
+ case CHIP_FIJI:
+ fw_name = FIRMWARE_FIJI;
+ break;
case CHIP_CARRIZO:
fw_name = FIRMWARE_CARRIZO;
break;
@@ -149,7 +154,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8)
+ AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE;
r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->uvd.vcpu_bo);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &adev->uvd.vcpu_bo);
if (r) {
dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r);
return r;
@@ -216,31 +223,32 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
int amdgpu_uvd_suspend(struct amdgpu_device *adev)
{
- unsigned size;
- void *ptr;
- const struct common_firmware_header *hdr;
- int i;
+ struct amdgpu_ring *ring = &adev->uvd.ring;
+ int i, r;
if (adev->uvd.vcpu_bo == NULL)
return 0;
- for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i)
- if (atomic_read(&adev->uvd.handles[i]))
- break;
-
- if (i == AMDGPU_MAX_UVD_HANDLES)
- return 0;
+ for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
+ uint32_t handle = atomic_read(&adev->uvd.handles[i]);
+ if (handle != 0) {
+ struct fence *fence;
- hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
+ amdgpu_uvd_note_usage(adev);
- size = amdgpu_bo_size(adev->uvd.vcpu_bo);
- size -= le32_to_cpu(hdr->ucode_size_bytes);
+ r = amdgpu_uvd_get_destroy_msg(ring, handle, &fence);
+ if (r) {
+ DRM_ERROR("Error destroying UVD (%d)!\n", r);
+ continue;
+ }
- ptr = adev->uvd.cpu_addr;
- ptr += le32_to_cpu(hdr->ucode_size_bytes);
+ fence_wait(fence, false);
+ fence_put(fence);
- adev->uvd.saved_bo = kmalloc(size, GFP_KERNEL);
- memcpy(adev->uvd.saved_bo, ptr, size);
+ adev->uvd.filp[i] = NULL;
+ atomic_set(&adev->uvd.handles[i], 0);
+ }
+ }
return 0;
}
@@ -265,12 +273,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
ptr = adev->uvd.cpu_addr;
ptr += le32_to_cpu(hdr->ucode_size_bytes);
- if (adev->uvd.saved_bo != NULL) {
- memcpy(ptr, adev->uvd.saved_bo, size);
- kfree(adev->uvd.saved_bo);
- adev->uvd.saved_bo = NULL;
- } else
- memset(ptr, 0, size);
+ memset(ptr, 0, size);
return 0;
}
@@ -283,7 +286,7 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) {
uint32_t handle = atomic_read(&adev->uvd.handles[i]);
if (handle != 0 && adev->uvd.filp[i] == filp) {
- struct amdgpu_fence *fence;
+ struct fence *fence;
amdgpu_uvd_note_usage(adev);
@@ -293,8 +296,8 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
continue;
}
- amdgpu_fence_wait(fence, false);
- amdgpu_fence_unref(&fence);
+ fence_wait(fence, false);
+ fence_put(fence);
adev->uvd.filp[i] = NULL;
atomic_set(&adev->uvd.handles[i], 0);
@@ -374,7 +377,8 @@ static int amdgpu_uvd_cs_msg_decode(uint32_t *msg, unsigned buf_sizes[])
unsigned height_in_mb = ALIGN(height / 16, 2);
unsigned fs_in_mb = width_in_mb * height_in_mb;
- unsigned image_size, tmp, min_dpb_size, num_dpb_buffer, min_ctx_size;
+ unsigned image_size, tmp, min_dpb_size, num_dpb_buffer;
+ unsigned min_ctx_size = 0;
image_size = width * height;
image_size += image_size / 2;
@@ -507,28 +511,25 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
{
struct amdgpu_device *adev = ctx->parser->adev;
int32_t *msg, msg_type, handle;
- struct fence *f;
void *ptr;
-
- int i, r;
+ long r;
+ int i;
if (offset & 0x3F) {
DRM_ERROR("UVD messages must be 64 byte aligned!\n");
return -EINVAL;
}
- f = reservation_object_get_excl(bo->tbo.resv);
- if (f) {
- r = amdgpu_fence_wait((struct amdgpu_fence *)f, false);
- if (r) {
- DRM_ERROR("Failed waiting for UVD message (%d)!\n", r);
- return r;
- }
+ r = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false,
+ MAX_SCHEDULE_TIMEOUT);
+ if (r < 0) {
+ DRM_ERROR("Failed waiting for UVD message (%ld)!\n", r);
+ return r;
}
r = amdgpu_bo_kmap(bo, &ptr);
if (r) {
- DRM_ERROR("Failed mapping the UVD message (%d)!\n", r);
+ DRM_ERROR("Failed mapping the UVD message (%ld)!\n", r);
return r;
}
@@ -803,14 +804,24 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
return 0;
}
+static int amdgpu_uvd_free_job(
+ struct amdgpu_job *sched_job)
+{
+ amdgpu_ib_free(sched_job->adev, sched_job->ibs);
+ kfree(sched_job->ibs);
+ return 0;
+}
+
static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring,
struct amdgpu_bo *bo,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
struct ttm_validate_buffer tv;
struct ww_acquire_ctx ticket;
struct list_head head;
- struct amdgpu_ib ib;
+ struct amdgpu_ib *ib = NULL;
+ struct fence *f = NULL;
+ struct amdgpu_device *adev = ring->adev;
uint64_t addr;
int i, r;
@@ -832,34 +843,49 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring,
r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
if (r)
goto err;
-
- r = amdgpu_ib_get(ring, NULL, 64, &ib);
- if (r)
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib) {
+ r = -ENOMEM;
goto err;
+ }
+ r = amdgpu_ib_get(ring, NULL, 64, ib);
+ if (r)
+ goto err1;
addr = amdgpu_bo_gpu_offset(bo);
- ib.ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0);
- ib.ptr[1] = addr;
- ib.ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0);
- ib.ptr[3] = addr >> 32;
- ib.ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0);
- ib.ptr[5] = 0;
+ ib->ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0);
+ ib->ptr[1] = addr;
+ ib->ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0);
+ ib->ptr[3] = addr >> 32;
+ ib->ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0);
+ ib->ptr[5] = 0;
for (i = 6; i < 16; ++i)
- ib.ptr[i] = PACKET2(0);
- ib.length_dw = 16;
+ ib->ptr[i] = PACKET2(0);
+ ib->length_dw = 16;
- r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_uvd_free_job,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
if (r)
- goto err;
- ttm_eu_fence_buffer_objects(&ticket, &head, &ib.fence->base);
+ goto err2;
- if (fence)
- *fence = amdgpu_fence_ref(ib.fence);
+ ttm_eu_fence_buffer_objects(&ticket, &head, f);
- amdgpu_ib_free(ring->adev, &ib);
+ if (fence)
+ *fence = fence_get(f);
amdgpu_bo_unref(&bo);
- return 0;
+ fence_put(f);
+ if (amdgpu_enable_scheduler)
+ return 0;
+ amdgpu_ib_free(ring->adev, ib);
+ kfree(ib);
+ return 0;
+err2:
+ amdgpu_ib_free(ring->adev, ib);
+err1:
+ kfree(ib);
err:
ttm_eu_backoff_reservation(&ticket, &head);
return r;
@@ -869,7 +895,7 @@ err:
crash the vcpu so just try to emmit a dummy create/destroy msg to
avoid this */
int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_bo *bo;
@@ -877,7 +903,9 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
int r, i;
r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &bo);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &bo);
if (r)
return r;
@@ -916,7 +944,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
}
int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_bo *bo;
@@ -924,7 +952,9 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
int r, i;
r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &bo);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &bo);
if (r)
return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 2255aa710e33..1724c2c86151 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -29,9 +29,9 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev);
int amdgpu_uvd_suspend(struct amdgpu_device *adev);
int amdgpu_uvd_resume(struct amdgpu_device *adev);
int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence);
+ struct fence **fence);
int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence);
+ struct fence **fence);
void amdgpu_uvd_free_handles(struct amdgpu_device *adev,
struct drm_file *filp);
int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index d3ca73090e39..3cab96c42aa8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -48,6 +48,7 @@
#endif
#define FIRMWARE_TONGA "amdgpu/tonga_vce.bin"
#define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin"
+#define FIRMWARE_FIJI "amdgpu/fiji_vce.bin"
#ifdef CONFIG_DRM_AMDGPU_CIK
MODULE_FIRMWARE(FIRMWARE_BONAIRE);
@@ -58,6 +59,7 @@ MODULE_FIRMWARE(FIRMWARE_MULLINS);
#endif
MODULE_FIRMWARE(FIRMWARE_TONGA);
MODULE_FIRMWARE(FIRMWARE_CARRIZO);
+MODULE_FIRMWARE(FIRMWARE_FIJI);
static void amdgpu_vce_idle_work_handler(struct work_struct *work);
@@ -101,6 +103,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
case CHIP_CARRIZO:
fw_name = FIRMWARE_CARRIZO;
break;
+ case CHIP_FIJI:
+ fw_name = FIRMWARE_FIJI;
+ break;
default:
return -EINVAL;
@@ -136,7 +141,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
/* allocate firmware, stack and heap BO */
r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->vce.vcpu_bo);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &adev->vce.vcpu_bo);
if (r) {
dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
return r;
@@ -334,6 +341,14 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
}
}
+static int amdgpu_vce_free_job(
+ struct amdgpu_job *sched_job)
+{
+ amdgpu_ib_free(sched_job->adev, sched_job->ibs);
+ kfree(sched_job->ibs);
+ return 0;
+}
+
/**
* amdgpu_vce_get_create_msg - generate a VCE create msg
*
@@ -345,59 +360,69 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
* Open up a stream for HW test
*/
int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
const unsigned ib_size_dw = 1024;
- struct amdgpu_ib ib;
+ struct amdgpu_ib *ib = NULL;
+ struct fence *f = NULL;
+ struct amdgpu_device *adev = ring->adev;
uint64_t dummy;
int i, r;
- r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, &ib);
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
+ return -ENOMEM;
+ r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, ib);
if (r) {
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+ kfree(ib);
return r;
}
- dummy = ib.gpu_addr + 1024;
+ dummy = ib->gpu_addr + 1024;
/* stitch together an VCE create msg */
- ib.length_dw = 0;
- ib.ptr[ib.length_dw++] = 0x0000000c; /* len */
- ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */
- ib.ptr[ib.length_dw++] = handle;
-
- ib.ptr[ib.length_dw++] = 0x00000030; /* len */
- ib.ptr[ib.length_dw++] = 0x01000001; /* create cmd */
- ib.ptr[ib.length_dw++] = 0x00000000;
- ib.ptr[ib.length_dw++] = 0x00000042;
- ib.ptr[ib.length_dw++] = 0x0000000a;
- ib.ptr[ib.length_dw++] = 0x00000001;
- ib.ptr[ib.length_dw++] = 0x00000080;
- ib.ptr[ib.length_dw++] = 0x00000060;
- ib.ptr[ib.length_dw++] = 0x00000100;
- ib.ptr[ib.length_dw++] = 0x00000100;
- ib.ptr[ib.length_dw++] = 0x0000000c;
- ib.ptr[ib.length_dw++] = 0x00000000;
-
- ib.ptr[ib.length_dw++] = 0x00000014; /* len */
- ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */
- ib.ptr[ib.length_dw++] = upper_32_bits(dummy);
- ib.ptr[ib.length_dw++] = dummy;
- ib.ptr[ib.length_dw++] = 0x00000001;
-
- for (i = ib.length_dw; i < ib_size_dw; ++i)
- ib.ptr[i] = 0x0;
-
- r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- }
-
+ ib->length_dw = 0;
+ ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
+ ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
+ ib->ptr[ib->length_dw++] = handle;
+
+ ib->ptr[ib->length_dw++] = 0x00000030; /* len */
+ ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */
+ ib->ptr[ib->length_dw++] = 0x00000000;
+ ib->ptr[ib->length_dw++] = 0x00000042;
+ ib->ptr[ib->length_dw++] = 0x0000000a;
+ ib->ptr[ib->length_dw++] = 0x00000001;
+ ib->ptr[ib->length_dw++] = 0x00000080;
+ ib->ptr[ib->length_dw++] = 0x00000060;
+ ib->ptr[ib->length_dw++] = 0x00000100;
+ ib->ptr[ib->length_dw++] = 0x00000100;
+ ib->ptr[ib->length_dw++] = 0x0000000c;
+ ib->ptr[ib->length_dw++] = 0x00000000;
+
+ ib->ptr[ib->length_dw++] = 0x00000014; /* len */
+ ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
+ ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+ ib->ptr[ib->length_dw++] = dummy;
+ ib->ptr[ib->length_dw++] = 0x00000001;
+
+ for (i = ib->length_dw; i < ib_size_dw; ++i)
+ ib->ptr[i] = 0x0;
+
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vce_free_job,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err;
if (fence)
- *fence = amdgpu_fence_ref(ib.fence);
-
- amdgpu_ib_free(ring->adev, &ib);
-
+ *fence = fence_get(f);
+ fence_put(f);
+ if (amdgpu_enable_scheduler)
+ return 0;
+err:
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
return r;
}
@@ -412,49 +437,59 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
* Close up a stream for HW test or if userspace failed to do so
*/
int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
const unsigned ib_size_dw = 1024;
- struct amdgpu_ib ib;
+ struct amdgpu_ib *ib = NULL;
+ struct fence *f = NULL;
+ struct amdgpu_device *adev = ring->adev;
uint64_t dummy;
int i, r;
- r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, &ib);
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
+ return -ENOMEM;
+
+ r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, ib);
if (r) {
+ kfree(ib);
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
return r;
}
- dummy = ib.gpu_addr + 1024;
+ dummy = ib->gpu_addr + 1024;
/* stitch together an VCE destroy msg */
- ib.length_dw = 0;
- ib.ptr[ib.length_dw++] = 0x0000000c; /* len */
- ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */
- ib.ptr[ib.length_dw++] = handle;
-
- ib.ptr[ib.length_dw++] = 0x00000014; /* len */
- ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */
- ib.ptr[ib.length_dw++] = upper_32_bits(dummy);
- ib.ptr[ib.length_dw++] = dummy;
- ib.ptr[ib.length_dw++] = 0x00000001;
-
- ib.ptr[ib.length_dw++] = 0x00000008; /* len */
- ib.ptr[ib.length_dw++] = 0x02000001; /* destroy cmd */
-
- for (i = ib.length_dw; i < ib_size_dw; ++i)
- ib.ptr[i] = 0x0;
-
- r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- }
-
+ ib->length_dw = 0;
+ ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
+ ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
+ ib->ptr[ib->length_dw++] = handle;
+
+ ib->ptr[ib->length_dw++] = 0x00000014; /* len */
+ ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
+ ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
+ ib->ptr[ib->length_dw++] = dummy;
+ ib->ptr[ib->length_dw++] = 0x00000001;
+
+ ib->ptr[ib->length_dw++] = 0x00000008; /* len */
+ ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
+
+ for (i = ib->length_dw; i < ib_size_dw; ++i)
+ ib->ptr[i] = 0x0;
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vce_free_job,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err;
if (fence)
- *fence = amdgpu_fence_ref(ib.fence);
-
- amdgpu_ib_free(ring->adev, &ib);
-
+ *fence = fence_get(f);
+ fence_put(f);
+ if (amdgpu_enable_scheduler)
+ return 0;
+err:
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
return r;
}
@@ -800,9 +835,13 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
*/
int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring)
{
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
int r;
+ /* skip vce ring1 ib test for now, since it's not reliable */
+ if (ring == &ring->adev->vce.ring[1])
+ return 0;
+
r = amdgpu_vce_get_create_msg(ring, 1, NULL);
if (r) {
DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r);
@@ -815,13 +854,13 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring)
goto error;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
} else {
DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
}
error:
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
return r;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 7ccdb5927da5..ba2da8ee5906 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -29,9 +29,9 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev);
int amdgpu_vce_suspend(struct amdgpu_device *adev);
int amdgpu_vce_resume(struct amdgpu_device *adev);
int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence);
+ struct fence **fence);
int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
- struct amdgpu_fence **fence);
+ struct fence **fence);
void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp);
int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx);
bool amdgpu_vce_ring_emit_semaphore(struct amdgpu_ring *ring,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9a4e3b63f1cb..f68b7cdc370a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -127,16 +127,16 @@ struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev,
/**
* amdgpu_vm_grab_id - allocate the next free VMID
*
- * @ring: ring we want to submit job to
* @vm: vm to allocate id for
+ * @ring: ring we want to submit job to
+ * @sync: sync object where we add dependencies
*
- * Allocate an id for the vm (cayman+).
- * Returns the fence we need to sync to (if any).
+ * Allocate an id for the vm, adding fences to the sync obj as necessary.
*
- * Global and local mutex must be locked!
+ * Global mutex must be locked!
*/
-struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
- struct amdgpu_vm *vm)
+int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
+ struct amdgpu_sync *sync)
{
struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {};
struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx];
@@ -148,7 +148,7 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
/* check if the id is still valid */
if (vm_id->id && vm_id->last_id_use &&
vm_id->last_id_use == adev->vm_manager.active[vm_id->id])
- return NULL;
+ return 0;
/* we definately need to flush */
vm_id->pd_gpu_addr = ~0ll;
@@ -161,7 +161,7 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
/* found a free one */
vm_id->id = i;
trace_amdgpu_vm_grab_id(i, ring->idx);
- return NULL;
+ return 0;
}
if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) {
@@ -172,15 +172,19 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
for (i = 0; i < 2; ++i) {
if (choices[i]) {
+ struct amdgpu_fence *fence;
+
+ fence = adev->vm_manager.active[choices[i]];
vm_id->id = choices[i];
+
trace_amdgpu_vm_grab_id(choices[i], ring->idx);
- return adev->vm_manager.active[choices[i]];
+ return amdgpu_sync_fence(ring->adev, sync, &fence->base);
}
}
/* should never happen */
BUG();
- return NULL;
+ return -EINVAL;
}
/**
@@ -196,17 +200,29 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring,
*/
void amdgpu_vm_flush(struct amdgpu_ring *ring,
struct amdgpu_vm *vm,
- struct amdgpu_fence *updates)
+ struct fence *updates)
{
uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx];
+ struct fence *flushed_updates = vm_id->flushed_updates;
+ bool is_earlier = false;
- if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates ||
- amdgpu_fence_is_earlier(vm_id->flushed_updates, updates)) {
+ if (flushed_updates && updates) {
+ BUG_ON(flushed_updates->context != updates->context);
+ is_earlier = (updates->seqno - flushed_updates->seqno <=
+ INT_MAX) ? true : false;
+ }
+
+ if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates ||
+ is_earlier) {
trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id);
- amdgpu_fence_unref(&vm_id->flushed_updates);
- vm_id->flushed_updates = amdgpu_fence_ref(updates);
+ if (is_earlier) {
+ vm_id->flushed_updates = fence_get(updates);
+ fence_put(flushed_updates);
+ }
+ if (!flushed_updates)
+ vm_id->flushed_updates = fence_get(updates);
vm_id->pd_gpu_addr = pd_addr;
amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr);
}
@@ -300,6 +316,15 @@ static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
}
}
+int amdgpu_vm_free_job(struct amdgpu_job *sched_job)
+{
+ int i;
+ for (i = 0; i < sched_job->num_ibs; i++)
+ amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]);
+ kfree(sched_job->ibs);
+ return 0;
+}
+
/**
* amdgpu_vm_clear_bo - initially clear the page dir/table
*
@@ -310,7 +335,8 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
struct amdgpu_bo *bo)
{
struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring;
- struct amdgpu_ib ib;
+ struct fence *fence = NULL;
+ struct amdgpu_ib *ib;
unsigned entries;
uint64_t addr;
int r;
@@ -330,24 +356,33 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
addr = amdgpu_bo_gpu_offset(bo);
entries = amdgpu_bo_size(bo) / 8;
- r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, &ib);
- if (r)
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
goto error_unreserve;
- ib.length_dw = 0;
-
- amdgpu_vm_update_pages(adev, &ib, addr, 0, entries, 0, 0, 0);
- amdgpu_vm_pad_ib(adev, &ib);
- WARN_ON(ib.length_dw > 64);
-
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM);
+ r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib);
if (r)
goto error_free;
- amdgpu_bo_fence(bo, ib.fence, true);
-
+ ib->length_dw = 0;
+
+ amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0);
+ amdgpu_vm_pad_ib(adev, ib);
+ WARN_ON(ib->length_dw > 64);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vm_free_job,
+ AMDGPU_FENCE_OWNER_VM,
+ &fence);
+ if (!r)
+ amdgpu_bo_fence(bo, fence, true);
+ fence_put(fence);
+ if (amdgpu_enable_scheduler) {
+ amdgpu_bo_unreserve(bo);
+ return 0;
+ }
error_free:
- amdgpu_ib_free(adev, &ib);
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
error_unreserve:
amdgpu_bo_unreserve(bo);
@@ -400,7 +435,9 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
uint64_t last_pde = ~0, last_pt = ~0;
unsigned count = 0, pt_idx, ndw;
- struct amdgpu_ib ib;
+ struct amdgpu_ib *ib;
+ struct fence *fence = NULL;
+
int r;
/* padding, etc. */
@@ -413,10 +450,14 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
if (ndw > 0xfffff)
return -ENOMEM;
- r = amdgpu_ib_get(ring, NULL, ndw * 4, &ib);
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
+ return -ENOMEM;
+
+ r = amdgpu_ib_get(ring, NULL, ndw * 4, ib);
if (r)
return r;
- ib.length_dw = 0;
+ ib->length_dw = 0;
/* walk over the address space and update the page directory */
for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
@@ -436,7 +477,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
((last_pt + incr * count) != pt)) {
if (count) {
- amdgpu_vm_update_pages(adev, &ib, last_pde,
+ amdgpu_vm_update_pages(adev, ib, last_pde,
last_pt, count, incr,
AMDGPU_PTE_VALID, 0);
}
@@ -450,23 +491,37 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
}
if (count)
- amdgpu_vm_update_pages(adev, &ib, last_pde, last_pt, count,
+ amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count,
incr, AMDGPU_PTE_VALID, 0);
- if (ib.length_dw != 0) {
- amdgpu_vm_pad_ib(adev, &ib);
- amdgpu_sync_resv(adev, &ib.sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM);
- WARN_ON(ib.length_dw > ndw);
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM);
- if (r) {
- amdgpu_ib_free(adev, &ib);
- return r;
- }
- amdgpu_bo_fence(pd, ib.fence, true);
+ if (ib->length_dw != 0) {
+ amdgpu_vm_pad_ib(adev, ib);
+ amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM);
+ WARN_ON(ib->length_dw > ndw);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vm_free_job,
+ AMDGPU_FENCE_OWNER_VM,
+ &fence);
+ if (r)
+ goto error_free;
+
+ amdgpu_bo_fence(pd, fence, true);
+ fence_put(vm->page_directory_fence);
+ vm->page_directory_fence = fence_get(fence);
+ fence_put(fence);
+ }
+
+ if (!amdgpu_enable_scheduler || ib->length_dw == 0) {
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
}
- amdgpu_ib_free(adev, &ib);
return 0;
+
+error_free:
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
+ return r;
}
/**
@@ -572,9 +627,14 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev,
{
uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
uint64_t last_pte = ~0, last_dst = ~0;
+ void *owner = AMDGPU_FENCE_OWNER_VM;
unsigned count = 0;
uint64_t addr;
+ /* sync to everything on unmapping */
+ if (!(flags & AMDGPU_PTE_VALID))
+ owner = AMDGPU_FENCE_OWNER_UNDEFINED;
+
/* walk over the address space and update the page tables */
for (addr = start; addr < end; ) {
uint64_t pt_idx = addr >> amdgpu_vm_block_size;
@@ -583,8 +643,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev,
uint64_t pte;
int r;
- amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv,
- AMDGPU_FENCE_OWNER_VM);
+ amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner);
r = reservation_object_reserve_shared(pt->tbo.resv);
if (r)
return r;
@@ -640,7 +699,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev,
*/
static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm,
uint64_t start, uint64_t end,
- struct amdgpu_fence *fence)
+ struct fence *fence)
{
unsigned i;
@@ -670,12 +729,13 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_bo_va_mapping *mapping,
uint64_t addr, uint32_t gtt_flags,
- struct amdgpu_fence **fence)
+ struct fence **fence)
{
struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring;
unsigned nptes, ncmds, ndw;
uint32_t flags = gtt_flags;
- struct amdgpu_ib ib;
+ struct amdgpu_ib *ib;
+ struct fence *f = NULL;
int r;
/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -722,46 +782,54 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
if (ndw > 0xfffff)
return -ENOMEM;
- r = amdgpu_ib_get(ring, NULL, ndw * 4, &ib);
- if (r)
- return r;
- ib.length_dw = 0;
-
- if (!(flags & AMDGPU_PTE_VALID)) {
- unsigned i;
+ ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL);
+ if (!ib)
+ return -ENOMEM;
- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- struct amdgpu_fence *f = vm->ids[i].last_id_use;
- amdgpu_sync_fence(&ib.sync, f);
- }
+ r = amdgpu_ib_get(ring, NULL, ndw * 4, ib);
+ if (r) {
+ kfree(ib);
+ return r;
}
- r = amdgpu_vm_update_ptes(adev, vm, &ib, mapping->it.start,
+ ib->length_dw = 0;
+
+ r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start,
mapping->it.last + 1, addr + mapping->offset,
flags, gtt_flags);
if (r) {
- amdgpu_ib_free(adev, &ib);
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
return r;
}
- amdgpu_vm_pad_ib(adev, &ib);
- WARN_ON(ib.length_dw > ndw);
+ amdgpu_vm_pad_ib(adev, ib);
+ WARN_ON(ib->length_dw > ndw);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,
+ &amdgpu_vm_free_job,
+ AMDGPU_FENCE_OWNER_VM,
+ &f);
+ if (r)
+ goto error_free;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM);
- if (r) {
- amdgpu_ib_free(adev, &ib);
- return r;
- }
amdgpu_vm_fence_pts(vm, mapping->it.start,
- mapping->it.last + 1, ib.fence);
+ mapping->it.last + 1, f);
if (fence) {
- amdgpu_fence_unref(fence);
- *fence = amdgpu_fence_ref(ib.fence);
+ fence_put(*fence);
+ *fence = fence_get(f);
+ }
+ fence_put(f);
+ if (!amdgpu_enable_scheduler) {
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
}
- amdgpu_ib_free(adev, &ib);
-
return 0;
+
+error_free:
+ amdgpu_ib_free(adev, ib);
+ kfree(ib);
+ return r;
}
/**
@@ -794,21 +862,25 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
addr = 0;
}
- if (addr == bo_va->addr)
- return 0;
-
flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
- list_for_each_entry(mapping, &bo_va->mappings, list) {
+ spin_lock(&vm->status_lock);
+ if (!list_empty(&bo_va->vm_status))
+ list_splice_init(&bo_va->valids, &bo_va->invalids);
+ spin_unlock(&vm->status_lock);
+
+ list_for_each_entry(mapping, &bo_va->invalids, list) {
r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr,
flags, &bo_va->last_pt_update);
if (r)
return r;
}
- bo_va->addr = addr;
spin_lock(&vm->status_lock);
+ list_splice_init(&bo_va->invalids, &bo_va->valids);
list_del_init(&bo_va->vm_status);
+ if (!mem)
+ list_add(&bo_va->vm_status, &vm->cleared);
spin_unlock(&vm->status_lock);
return 0;
@@ -861,7 +933,7 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
struct amdgpu_vm *vm, struct amdgpu_sync *sync)
{
struct amdgpu_bo_va *bo_va = NULL;
- int r;
+ int r = 0;
spin_lock(&vm->status_lock);
while (!list_empty(&vm->invalidated)) {
@@ -878,8 +950,9 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
spin_unlock(&vm->status_lock);
if (bo_va)
- amdgpu_sync_fence(sync, bo_va->last_pt_update);
- return 0;
+ r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
+
+ return r;
}
/**
@@ -907,10 +980,10 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
}
bo_va->vm = vm;
bo_va->bo = bo;
- bo_va->addr = 0;
bo_va->ref_count = 1;
INIT_LIST_HEAD(&bo_va->bo_list);
- INIT_LIST_HEAD(&bo_va->mappings);
+ INIT_LIST_HEAD(&bo_va->valids);
+ INIT_LIST_HEAD(&bo_va->invalids);
INIT_LIST_HEAD(&bo_va->vm_status);
mutex_lock(&vm->mutex);
@@ -999,12 +1072,10 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
mapping->offset = offset;
mapping->flags = flags;
- list_add(&mapping->list, &bo_va->mappings);
+ list_add(&mapping->list, &bo_va->invalids);
interval_tree_insert(&mapping->it, &vm->va);
trace_amdgpu_vm_bo_map(bo_va, mapping);
- bo_va->addr = 0;
-
/* Make sure the page tables are allocated */
saddr >>= amdgpu_vm_block_size;
eaddr >>= amdgpu_vm_block_size;
@@ -1028,7 +1099,9 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
AMDGPU_GPU_PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &pt);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
+ NULL, &pt);
if (r)
goto error_free;
@@ -1085,17 +1158,27 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
{
struct amdgpu_bo_va_mapping *mapping;
struct amdgpu_vm *vm = bo_va->vm;
+ bool valid = true;
saddr /= AMDGPU_GPU_PAGE_SIZE;
- list_for_each_entry(mapping, &bo_va->mappings, list) {
+ list_for_each_entry(mapping, &bo_va->valids, list) {
if (mapping->it.start == saddr)
break;
}
- if (&mapping->list == &bo_va->mappings) {
- amdgpu_bo_unreserve(bo_va->bo);
- return -ENOENT;
+ if (&mapping->list == &bo_va->valids) {
+ valid = false;
+
+ list_for_each_entry(mapping, &bo_va->invalids, list) {
+ if (mapping->it.start == saddr)
+ break;
+ }
+
+ if (&mapping->list == &bo_va->invalids) {
+ amdgpu_bo_unreserve(bo_va->bo);
+ return -ENOENT;
+ }
}
mutex_lock(&vm->mutex);
@@ -1103,12 +1186,10 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
interval_tree_remove(&mapping->it, &vm->va);
trace_amdgpu_vm_bo_unmap(bo_va, mapping);
- if (bo_va->addr) {
- /* clear the old address */
+ if (valid)
list_add(&mapping->list, &vm->freed);
- } else {
+ else
kfree(mapping);
- }
mutex_unlock(&vm->mutex);
amdgpu_bo_unreserve(bo_va->bo);
@@ -1139,16 +1220,19 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
list_del(&bo_va->vm_status);
spin_unlock(&vm->status_lock);
- list_for_each_entry_safe(mapping, next, &bo_va->mappings, list) {
+ list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
list_del(&mapping->list);
interval_tree_remove(&mapping->it, &vm->va);
trace_amdgpu_vm_bo_unmap(bo_va, mapping);
- if (bo_va->addr)
- list_add(&mapping->list, &vm->freed);
- else
- kfree(mapping);
+ list_add(&mapping->list, &vm->freed);
+ }
+ list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
+ list_del(&mapping->list);
+ interval_tree_remove(&mapping->it, &vm->va);
+ kfree(mapping);
}
- amdgpu_fence_unref(&bo_va->last_pt_update);
+
+ fence_put(bo_va->last_pt_update);
kfree(bo_va);
mutex_unlock(&vm->mutex);
@@ -1169,12 +1253,10 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va;
list_for_each_entry(bo_va, &bo->va, bo_list) {
- if (bo_va->addr) {
- spin_lock(&bo_va->vm->status_lock);
- list_del(&bo_va->vm_status);
+ spin_lock(&bo_va->vm->status_lock);
+ if (list_empty(&bo_va->vm_status))
list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
- spin_unlock(&bo_va->vm->status_lock);
- }
+ spin_unlock(&bo_va->vm->status_lock);
}
}
@@ -1202,6 +1284,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
vm->va = RB_ROOT;
spin_lock_init(&vm->status_lock);
INIT_LIST_HEAD(&vm->invalidated);
+ INIT_LIST_HEAD(&vm->cleared);
INIT_LIST_HEAD(&vm->freed);
pd_size = amdgpu_vm_directory_size(adev);
@@ -1215,8 +1298,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
return -ENOMEM;
}
+ vm->page_directory_fence = NULL;
+
r = amdgpu_bo_create(adev, pd_size, align, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0,
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
NULL, &vm->page_directory);
if (r)
return r;
@@ -1263,9 +1349,10 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
kfree(vm->page_tables);
amdgpu_bo_unref(&vm->page_directory);
+ fence_put(vm->page_directory_fence);
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
- amdgpu_fence_unref(&vm->ids[i].flushed_updates);
+ fence_put(vm->ids[i].flushed_updates);
amdgpu_fence_unref(&vm->ids[i].last_id_use);
}
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
index 9ba0a7d5bc8e..92b6acadfc52 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c
@@ -139,7 +139,8 @@ amdgpu_atombios_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *m
tx_buf[0] = msg->address & 0xff;
tx_buf[1] = msg->address >> 8;
- tx_buf[2] = msg->request << 4;
+ tx_buf[2] = (msg->request << 4) |
+ ((msg->address >> 16) & 0xf);
tx_buf[3] = msg->size ? (msg->size - 1) : 0;
switch (msg->request & ~DP_AUX_I2C_MOT) {
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index ae8caca61e04..cd6edc40c9cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -812,7 +812,7 @@ amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int a
else
args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG1_ENCODER;
- if ((adev->flags & AMDGPU_IS_APU) &&
+ if ((adev->flags & AMD_IS_APU) &&
(amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_UNIPHY)) {
if (is_dp ||
!amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 341c56681841..4b6ce74753cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -64,6 +64,8 @@
#include "oss/oss_2_0_d.h"
#include "oss/oss_2_0_sh_mask.h"
+#include "amdgpu_amdkfd.h"
+
/*
* Indirect registers accessor
*/
@@ -836,7 +838,7 @@ static u32 cik_get_xclk(struct amdgpu_device *adev)
{
u32 reference_clock = adev->clock.spll.reference_freq;
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
return reference_clock / 2;
} else {
@@ -1233,7 +1235,7 @@ static void cik_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask)
if (reset_mask & AMDGPU_RESET_VMC)
srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK;
- if (!(adev->flags & AMDGPU_IS_APU)) {
+ if (!(adev->flags & AMD_IS_APU)) {
if (reset_mask & AMDGPU_RESET_MC)
srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_MC_MASK;
}
@@ -1409,7 +1411,7 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
dev_warn(adev->dev, "Wait for MC idle timed out !\n");
}
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
kv_save_regs_for_reset(adev, &kv_save);
/* disable BM */
@@ -1427,7 +1429,7 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
}
/* does asic init need to be run first??? */
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
kv_restore_regs_for_reset(adev, &kv_save);
}
@@ -1568,7 +1570,7 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
if (amdgpu_pcie_gen2 == 0)
return;
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return;
ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
@@ -1728,7 +1730,7 @@ static void cik_program_aspm(struct amdgpu_device *adev)
return;
/* XXX double check APUs */
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return;
orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
@@ -2448,14 +2450,21 @@ static int cik_common_suspend(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ amdgpu_amdkfd_suspend(adev);
+
return cik_common_hw_fini(adev);
}
static int cik_common_resume(void *handle)
{
+ int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- return cik_common_hw_init(adev);
+ r = cik_common_hw_init(adev);
+ if (r)
+ return r;
+
+ return amdgpu_amdkfd_resume(adev);
}
static bool cik_common_is_idle(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 15df46c93f0a..9ea9de457da3 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -188,6 +188,19 @@ static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)
WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc);
}
+static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ int i;
+
+ for (i = 0; i < count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ amdgpu_ring_write(ring, ring->nop |
+ SDMA_NOP_COUNT(count - 1));
+ else
+ amdgpu_ring_write(ring, ring->nop);
+}
+
/**
* cik_sdma_ring_emit_ib - Schedule an IB on the DMA engine
*
@@ -213,8 +226,8 @@ static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,
amdgpu_ring_write(ring, next_rptr);
/* IB packet must end on a 8 DW boundary */
- while ((ring->wptr & 7) != 4)
- amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
+ cik_sdma_ring_insert_nop(ring, (12 - (ring->wptr & 7)) % 8);
+
amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
@@ -501,6 +514,8 @@ static int cik_sdma_load_microcode(struct amdgpu_device *adev)
fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma[i].feature_version >= 20)
+ adev->sdma[i].burst_nop = true;
fw_data = (const __le32 *)
(adev->sdma[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
@@ -614,6 +629,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_ib ib;
+ struct fence *f = NULL;
unsigned i;
unsigned index;
int r;
@@ -629,12 +645,11 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
gpu_addr = adev->wb.gpu_addr + (index * 4);
tmp = 0xCAFEDEAD;
adev->wb.wb[index] = cpu_to_le32(tmp);
-
+ memset(&ib, 0, sizeof(ib));
r = amdgpu_ib_get(ring, NULL, 256, &ib);
if (r) {
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
- return r;
+ goto err0;
}
ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
@@ -643,20 +658,16 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
ib.ptr[3] = 1;
ib.ptr[4] = 0xDEADBEEF;
ib.length_dw = 5;
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err1;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
+ r = fence_wait(f, false);
if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- return r;
- }
- r = amdgpu_fence_wait(ib.fence, false);
- if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
- return r;
+ goto err1;
}
for (i = 0; i < adev->usec_timeout; i++) {
tmp = le32_to_cpu(adev->wb.wb[index]);
@@ -666,12 +677,17 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
}
if (i < adev->usec_timeout) {
DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
- ib.fence->ring->idx, i);
+ ring->idx, i);
+ goto err1;
} else {
DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
r = -EINVAL;
}
+
+err1:
+ fence_put(f);
amdgpu_ib_free(adev, &ib);
+err0:
amdgpu_wb_free(adev, index);
return r;
}
@@ -814,8 +830,19 @@ static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib,
*/
static void cik_sdma_vm_pad_ib(struct amdgpu_ib *ib)
{
- while (ib->length_dw & 0x7)
- ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ u32 pad_count;
+ int i;
+
+ pad_count = (8 - (ib->length_dw & 0x7)) % 8;
+ for (i = 0; i < pad_count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ ib->ptr[ib->length_dw++] =
+ SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0) |
+ SDMA_NOP_COUNT(pad_count - 1);
+ else
+ ib->ptr[ib->length_dw++] =
+ SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
}
/**
@@ -1302,6 +1329,7 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
.test_ring = cik_sdma_ring_test_ring,
.test_ib = cik_sdma_ring_test_ib,
.is_lockup = cik_sdma_ring_is_lockup,
+ .insert_nop = cik_sdma_ring_insert_nop,
};
static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
@@ -1338,18 +1366,18 @@ static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)
* Used by the amdgpu ttm implementation to move pages if
* registered as the asic copy callback.
*/
-static void cik_sdma_emit_copy_buffer(struct amdgpu_ring *ring,
+static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib,
uint64_t src_offset,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0));
- amdgpu_ring_write(ring, byte_count);
- amdgpu_ring_write(ring, 0); /* src/dst endian swap */
- amdgpu_ring_write(ring, lower_32_bits(src_offset));
- amdgpu_ring_write(ring, upper_32_bits(src_offset));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
+ ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0);
+ ib->ptr[ib->length_dw++] = byte_count;
+ ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+ ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
}
/**
@@ -1362,16 +1390,16 @@ static void cik_sdma_emit_copy_buffer(struct amdgpu_ring *ring,
*
* Fill GPU buffers using the DMA engine (CIK).
*/
-static void cik_sdma_emit_fill_buffer(struct amdgpu_ring *ring,
+static void cik_sdma_emit_fill_buffer(struct amdgpu_ib *ib,
uint32_t src_data,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
- amdgpu_ring_write(ring, src_data);
- amdgpu_ring_write(ring, byte_count);
+ ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = src_data;
+ ib->ptr[ib->length_dw++] = byte_count;
}
static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {
@@ -1404,5 +1432,6 @@ static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/cikd.h b/drivers/gpu/drm/amd/amdgpu/cikd.h
index d19085a97064..7f6d457f250a 100644
--- a/drivers/gpu/drm/amd/amdgpu/cikd.h
+++ b/drivers/gpu/drm/amd/amdgpu/cikd.h
@@ -487,6 +487,7 @@
(((op) & 0xFF) << 0))
/* sDMA opcodes */
#define SDMA_OPCODE_NOP 0
+# define SDMA_NOP_COUNT(x) (((x) & 0x3FFF) << 16)
#define SDMA_OPCODE_COPY 1
# define SDMA_COPY_SUB_OPCODE_LINEAR 0
# define SDMA_COPY_SUB_OPCODE_TILED 1
@@ -552,6 +553,12 @@
#define VCE_CMD_IB_AUTO 0x00000005
#define VCE_CMD_SEMAPHORE 0x00000006
+/* if PTR32, these are the bases for scratch and lds */
+#define PRIVATE_BASE(x) ((x) << 0) /* scratch */
+#define SHARED_BASE(x) ((x) << 16) /* LDS */
+
+#define KFD_CIK_SDMA_QUEUE_OFFSET 0x200
+
/* valid for both DEFAULT_MTYPE and APE1_MTYPE */
enum {
MTYPE_CACHED = 0,
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index ace870afc7d4..44fa96ad4709 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -1596,9 +1596,9 @@ static int cz_dpm_update_low_memory_pstate(struct amdgpu_device *adev)
if (pi->sys_info.nb_dpm_enable) {
if (ps->force_high)
- cz_dpm_nbdpm_lm_pstate_enable(adev, true);
- else
cz_dpm_nbdpm_lm_pstate_enable(adev, false);
+ else
+ cz_dpm_nbdpm_lm_pstate_enable(adev, true);
}
return ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index e70a26f587a0..e4d101b1252a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -126,9 +126,31 @@ static const u32 tonga_mgcg_cgcg_init[] =
mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
};
+static const u32 golden_settings_fiji_a10[] =
+{
+ mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
+ mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
+ mmFBC_MISC, 0x1f311fff, 0x12300000,
+ mmHDMI_CONTROL, 0x31000111, 0x00000011,
+};
+
+static const u32 fiji_mgcg_cgcg_init[] =
+{
+ mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
+ mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
+};
+
static void dce_v10_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
+ case CHIP_FIJI:
+ amdgpu_program_register_sequence(adev,
+ fiji_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ golden_settings_fiji_a10,
+ (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+ break;
case CHIP_TONGA:
amdgpu_program_register_sequence(adev,
tonga_mgcg_cgcg_init,
@@ -803,11 +825,11 @@ static u32 dce_v10_0_line_buffer_adjust(struct amdgpu_device *adev,
buffer_alloc = 2;
} else if (mode->crtc_hdisplay < 4096) {
mem_cfg = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
} else {
DRM_DEBUG_KMS("Mode too big for LB!\n");
mem_cfg = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
}
} else {
mem_cfg = 1;
@@ -1331,7 +1353,7 @@ static void dce_v10_0_program_watermarks(struct amdgpu_device *adev,
tmp = REG_SET_FIELD(wm_mask, DPG_WATERMARK_MASK_CONTROL, URGENCY_WATERMARK_MASK, 2);
WREG32(mmDPG_WATERMARK_MASK_CONTROL + amdgpu_crtc->crtc_offset, tmp);
tmp = RREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset);
- tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_a);
+ tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_b);
tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, line_time);
WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset, tmp);
/* restore original selection */
@@ -2888,6 +2910,7 @@ static int dce_v10_0_early_init(void *handle)
dce_v10_0_set_irq_funcs(adev);
switch (adev->asic_type) {
+ case CHIP_FIJI:
case CHIP_TONGA:
adev->mode_info.num_crtc = 6; /* XXX 7??? */
adev->mode_info.num_hpd = 6;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index dcb402ee048a..6411e8244671 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -801,11 +801,11 @@ static u32 dce_v11_0_line_buffer_adjust(struct amdgpu_device *adev,
buffer_alloc = 2;
} else if (mode->crtc_hdisplay < 4096) {
mem_cfg = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
} else {
DRM_DEBUG_KMS("Mode too big for LB!\n");
mem_cfg = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
}
} else {
mem_cfg = 1;
@@ -1329,7 +1329,7 @@ static void dce_v11_0_program_watermarks(struct amdgpu_device *adev,
tmp = REG_SET_FIELD(wm_mask, DPG_WATERMARK_MASK_CONTROL, URGENCY_WATERMARK_MASK, 2);
WREG32(mmDPG_WATERMARK_MASK_CONTROL + amdgpu_crtc->crtc_offset, tmp);
tmp = RREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset);
- tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_a);
+ tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_b);
tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, line_time);
WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset, tmp);
/* restore original selection */
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index cc050a329c49..c86911c2ea2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -770,11 +770,11 @@ static u32 dce_v8_0_line_buffer_adjust(struct amdgpu_device *adev,
buffer_alloc = 2;
} else if (mode->crtc_hdisplay < 4096) {
tmp = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
} else {
DRM_DEBUG_KMS("Mode too big for LB!\n");
tmp = 0;
- buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4;
+ buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4;
}
} else {
tmp = 1;
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
new file mode 100644
index 000000000000..8f9845d9a986
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "amdgpu.h"
+#include "fiji_smumgr.h"
+
+MODULE_FIRMWARE("amdgpu/fiji_smc.bin");
+
+static void fiji_dpm_set_funcs(struct amdgpu_device *adev);
+
+static int fiji_dpm_early_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ fiji_dpm_set_funcs(adev);
+
+ return 0;
+}
+
+static int fiji_dpm_init_microcode(struct amdgpu_device *adev)
+{
+ char fw_name[30] = "amdgpu/fiji_smc.bin";
+ int err;
+
+ err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->pm.fw);
+
+out:
+ if (err) {
+ DRM_ERROR("Failed to load firmware \"%s\"", fw_name);
+ release_firmware(adev->pm.fw);
+ adev->pm.fw = NULL;
+ }
+ return err;
+}
+
+static int fiji_dpm_sw_init(void *handle)
+{
+ int ret;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ ret = fiji_dpm_init_microcode(adev);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static int fiji_dpm_sw_fini(void *handle)
+{
+ return 0;
+}
+
+static int fiji_dpm_hw_init(void *handle)
+{
+ int ret;
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ mutex_lock(&adev->pm.mutex);
+
+ ret = fiji_smu_init(adev);
+ if (ret) {
+ DRM_ERROR("SMU initialization failed\n");
+ goto fail;
+ }
+
+ ret = fiji_smu_start(adev);
+ if (ret) {
+ DRM_ERROR("SMU start failed\n");
+ goto fail;
+ }
+
+ mutex_unlock(&adev->pm.mutex);
+ return 0;
+
+fail:
+ adev->firmware.smu_load = false;
+ mutex_unlock(&adev->pm.mutex);
+ return -EINVAL;
+}
+
+static int fiji_dpm_hw_fini(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ mutex_lock(&adev->pm.mutex);
+ fiji_smu_fini(adev);
+ mutex_unlock(&adev->pm.mutex);
+ return 0;
+}
+
+static int fiji_dpm_suspend(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ fiji_dpm_hw_fini(adev);
+
+ return 0;
+}
+
+static int fiji_dpm_resume(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ fiji_dpm_hw_init(adev);
+
+ return 0;
+}
+
+static int fiji_dpm_set_clockgating_state(void *handle,
+ enum amd_clockgating_state state)
+{
+ return 0;
+}
+
+static int fiji_dpm_set_powergating_state(void *handle,
+ enum amd_powergating_state state)
+{
+ return 0;
+}
+
+const struct amd_ip_funcs fiji_dpm_ip_funcs = {
+ .early_init = fiji_dpm_early_init,
+ .late_init = NULL,
+ .sw_init = fiji_dpm_sw_init,
+ .sw_fini = fiji_dpm_sw_fini,
+ .hw_init = fiji_dpm_hw_init,
+ .hw_fini = fiji_dpm_hw_fini,
+ .suspend = fiji_dpm_suspend,
+ .resume = fiji_dpm_resume,
+ .is_idle = NULL,
+ .wait_for_idle = NULL,
+ .soft_reset = NULL,
+ .print_status = NULL,
+ .set_clockgating_state = fiji_dpm_set_clockgating_state,
+ .set_powergating_state = fiji_dpm_set_powergating_state,
+};
+
+static const struct amdgpu_dpm_funcs fiji_dpm_funcs = {
+ .get_temperature = NULL,
+ .pre_set_power_state = NULL,
+ .set_power_state = NULL,
+ .post_set_power_state = NULL,
+ .display_configuration_changed = NULL,
+ .get_sclk = NULL,
+ .get_mclk = NULL,
+ .print_power_state = NULL,
+ .debugfs_print_current_performance_level = NULL,
+ .force_performance_level = NULL,
+ .vblank_too_short = NULL,
+ .powergate_uvd = NULL,
+};
+
+static void fiji_dpm_set_funcs(struct amdgpu_device *adev)
+{
+ if (NULL == adev->pm.funcs)
+ adev->pm.funcs = &fiji_dpm_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h b/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h
new file mode 100644
index 000000000000..3c4824082990
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef FIJI_PP_SMC_H
+#define FIJI_PP_SMC_H
+
+#pragma pack(push, 1)
+
+#define PPSMC_SWSTATE_FLAG_DC 0x01
+#define PPSMC_SWSTATE_FLAG_UVD 0x02
+#define PPSMC_SWSTATE_FLAG_VCE 0x04
+
+#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL 0x00
+#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL 0x01
+#define PPSMC_THERMAL_PROTECT_TYPE_NONE 0xff
+
+#define PPSMC_SYSTEMFLAG_GPIO_DC 0x01
+#define PPSMC_SYSTEMFLAG_STEPVDDC 0x02
+#define PPSMC_SYSTEMFLAG_GDDR5 0x04
+
+#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP 0x08
+
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT 0x10
+#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG 0x20
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK 0x07
+#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK 0x08
+
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE 0x00
+#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE 0x01
+
+#define PPSMC_DPM2FLAGS_TDPCLMP 0x01
+#define PPSMC_DPM2FLAGS_PWRSHFT 0x02
+#define PPSMC_DPM2FLAGS_OCP 0x04
+
+#define PPSMC_DISPLAY_WATERMARK_LOW 0
+#define PPSMC_DISPLAY_WATERMARK_HIGH 1
+
+#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP 0x01
+#define PPSMC_STATEFLAG_POWERBOOST 0x02
+#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 0x04
+#define PPSMC_STATEFLAG_POWERSHIFT 0x08
+#define PPSMC_STATEFLAG_SLOW_READ_MARGIN 0x10
+#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20
+#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS 0x40
+
+#define FDO_MODE_HARDWARE 0
+#define FDO_MODE_PIECE_WISE_LINEAR 1
+
+enum FAN_CONTROL {
+ FAN_CONTROL_FUZZY,
+ FAN_CONTROL_TABLE
+};
+
+//Gemini Modes
+#define PPSMC_GeminiModeNone 0 //Single GPU board
+#define PPSMC_GeminiModeMaster 1 //Master GPU on a Gemini board
+#define PPSMC_GeminiModeSlave 2 //Slave GPU on a Gemini board
+
+#define PPSMC_Result_OK ((uint16_t)0x01)
+#define PPSMC_Result_NoMore ((uint16_t)0x02)
+#define PPSMC_Result_NotNow ((uint16_t)0x03)
+#define PPSMC_Result_Failed ((uint16_t)0xFF)
+#define PPSMC_Result_UnknownCmd ((uint16_t)0xFE)
+#define PPSMC_Result_UnknownVT ((uint16_t)0xFD)
+
+typedef uint16_t PPSMC_Result;
+
+#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x))
+
+#define PPSMC_MSG_Halt ((uint16_t)0x10)
+#define PPSMC_MSG_Resume ((uint16_t)0x11)
+#define PPSMC_MSG_EnableDPMLevel ((uint16_t)0x12)
+#define PPSMC_MSG_ZeroLevelsDisabled ((uint16_t)0x13)
+#define PPSMC_MSG_OneLevelsDisabled ((uint16_t)0x14)
+#define PPSMC_MSG_TwoLevelsDisabled ((uint16_t)0x15)
+#define PPSMC_MSG_EnableThermalInterrupt ((uint16_t)0x16)
+#define PPSMC_MSG_RunningOnAC ((uint16_t)0x17)
+#define PPSMC_MSG_LevelUp ((uint16_t)0x18)
+#define PPSMC_MSG_LevelDown ((uint16_t)0x19)
+#define PPSMC_MSG_ResetDPMCounters ((uint16_t)0x1a)
+#define PPSMC_MSG_SwitchToSwState ((uint16_t)0x20)
+#define PPSMC_MSG_SwitchToSwStateLast ((uint16_t)0x3f)
+#define PPSMC_MSG_SwitchToInitialState ((uint16_t)0x40)
+#define PPSMC_MSG_NoForcedLevel ((uint16_t)0x41)
+#define PPSMC_MSG_ForceHigh ((uint16_t)0x42)
+#define PPSMC_MSG_ForceMediumOrHigh ((uint16_t)0x43)
+#define PPSMC_MSG_SwitchToMinimumPower ((uint16_t)0x51)
+#define PPSMC_MSG_ResumeFromMinimumPower ((uint16_t)0x52)
+#define PPSMC_MSG_EnableCac ((uint16_t)0x53)
+#define PPSMC_MSG_DisableCac ((uint16_t)0x54)
+#define PPSMC_DPMStateHistoryStart ((uint16_t)0x55)
+#define PPSMC_DPMStateHistoryStop ((uint16_t)0x56)
+#define PPSMC_CACHistoryStart ((uint16_t)0x57)
+#define PPSMC_CACHistoryStop ((uint16_t)0x58)
+#define PPSMC_TDPClampingActive ((uint16_t)0x59)
+#define PPSMC_TDPClampingInactive ((uint16_t)0x5A)
+#define PPSMC_StartFanControl ((uint16_t)0x5B)
+#define PPSMC_StopFanControl ((uint16_t)0x5C)
+#define PPSMC_NoDisplay ((uint16_t)0x5D)
+#define PPSMC_HasDisplay ((uint16_t)0x5E)
+#define PPSMC_MSG_UVDPowerOFF ((uint16_t)0x60)
+#define PPSMC_MSG_UVDPowerON ((uint16_t)0x61)
+#define PPSMC_MSG_EnableULV ((uint16_t)0x62)
+#define PPSMC_MSG_DisableULV ((uint16_t)0x63)
+#define PPSMC_MSG_EnterULV ((uint16_t)0x64)
+#define PPSMC_MSG_ExitULV ((uint16_t)0x65)
+#define PPSMC_PowerShiftActive ((uint16_t)0x6A)
+#define PPSMC_PowerShiftInactive ((uint16_t)0x6B)
+#define PPSMC_OCPActive ((uint16_t)0x6C)
+#define PPSMC_OCPInactive ((uint16_t)0x6D)
+#define PPSMC_CACLongTermAvgEnable ((uint16_t)0x6E)
+#define PPSMC_CACLongTermAvgDisable ((uint16_t)0x6F)
+#define PPSMC_MSG_InferredStateSweep_Start ((uint16_t)0x70)
+#define PPSMC_MSG_InferredStateSweep_Stop ((uint16_t)0x71)
+#define PPSMC_MSG_SwitchToLowestInfState ((uint16_t)0x72)
+#define PPSMC_MSG_SwitchToNonInfState ((uint16_t)0x73)
+#define PPSMC_MSG_AllStateSweep_Start ((uint16_t)0x74)
+#define PPSMC_MSG_AllStateSweep_Stop ((uint16_t)0x75)
+#define PPSMC_MSG_SwitchNextLowerInfState ((uint16_t)0x76)
+#define PPSMC_MSG_SwitchNextHigherInfState ((uint16_t)0x77)
+#define PPSMC_MSG_MclkRetrainingTest ((uint16_t)0x78)
+#define PPSMC_MSG_ForceTDPClamping ((uint16_t)0x79)
+#define PPSMC_MSG_CollectCAC_PowerCorreln ((uint16_t)0x7A)
+#define PPSMC_MSG_CollectCAC_WeightCalib ((uint16_t)0x7B)
+#define PPSMC_MSG_CollectCAC_SQonly ((uint16_t)0x7C)
+#define PPSMC_MSG_CollectCAC_TemperaturePwr ((uint16_t)0x7D)
+#define PPSMC_MSG_ExtremitiesTest_Start ((uint16_t)0x7E)
+#define PPSMC_MSG_ExtremitiesTest_Stop ((uint16_t)0x7F)
+#define PPSMC_FlushDataCache ((uint16_t)0x80)
+#define PPSMC_FlushInstrCache ((uint16_t)0x81)
+#define PPSMC_MSG_SetEnabledLevels ((uint16_t)0x82)
+#define PPSMC_MSG_SetForcedLevels ((uint16_t)0x83)
+#define PPSMC_MSG_ResetToDefaults ((uint16_t)0x84)
+#define PPSMC_MSG_SetForcedLevelsAndJump ((uint16_t)0x85)
+#define PPSMC_MSG_SetCACHistoryMode ((uint16_t)0x86)
+#define PPSMC_MSG_EnableDTE ((uint16_t)0x87)
+#define PPSMC_MSG_DisableDTE ((uint16_t)0x88)
+#define PPSMC_MSG_SmcSpaceSetAddress ((uint16_t)0x89)
+#define PPSMC_MSG_SmcSpaceWriteDWordInc ((uint16_t)0x8A)
+#define PPSMC_MSG_SmcSpaceWriteWordInc ((uint16_t)0x8B)
+#define PPSMC_MSG_SmcSpaceWriteByteInc ((uint16_t)0x8C)
+
+#define PPSMC_MSG_BREAK ((uint16_t)0xF8)
+
+#define PPSMC_MSG_Test ((uint16_t)0x100)
+#define PPSMC_MSG_DRV_DRAM_ADDR_HI ((uint16_t)0x250)
+#define PPSMC_MSG_DRV_DRAM_ADDR_LO ((uint16_t)0x251)
+#define PPSMC_MSG_SMU_DRAM_ADDR_HI ((uint16_t)0x252)
+#define PPSMC_MSG_SMU_DRAM_ADDR_LO ((uint16_t)0x253)
+#define PPSMC_MSG_LoadUcodes ((uint16_t)0x254)
+
+typedef uint16_t PPSMC_Msg;
+
+#define PPSMC_EVENT_STATUS_THERMAL 0x00000001
+#define PPSMC_EVENT_STATUS_REGULATORHOT 0x00000002
+#define PPSMC_EVENT_STATUS_DC 0x00000004
+#define PPSMC_EVENT_STATUS_GPIO17 0x00000008
+
+#pragma pack(pop)
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
new file mode 100644
index 000000000000..322edea65857
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
@@ -0,0 +1,857 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/firmware.h>
+#include "drmP.h"
+#include "amdgpu.h"
+#include "fiji_ppsmc.h"
+#include "fiji_smumgr.h"
+#include "smu_ucode_xfer_vi.h"
+#include "amdgpu_ucode.h"
+
+#include "smu/smu_7_1_3_d.h"
+#include "smu/smu_7_1_3_sh_mask.h"
+
+#define FIJI_SMC_SIZE 0x20000
+
+static int fiji_set_smc_sram_address(struct amdgpu_device *adev, uint32_t smc_address, uint32_t limit)
+{
+ uint32_t val;
+
+ if (smc_address & 3)
+ return -EINVAL;
+
+ if ((smc_address + 3) > limit)
+ return -EINVAL;
+
+ WREG32(mmSMC_IND_INDEX_0, smc_address);
+
+ val = RREG32(mmSMC_IND_ACCESS_CNTL);
+ val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+ WREG32(mmSMC_IND_ACCESS_CNTL, val);
+
+ return 0;
+}
+
+static int fiji_copy_bytes_to_smc(struct amdgpu_device *adev, uint32_t smc_start_address, const uint8_t *src, uint32_t byte_count, uint32_t limit)
+{
+ uint32_t addr;
+ uint32_t data, orig_data;
+ int result = 0;
+ uint32_t extra_shift;
+ unsigned long flags;
+
+ if (smc_start_address & 3)
+ return -EINVAL;
+
+ if ((smc_start_address + byte_count) > limit)
+ return -EINVAL;
+
+ addr = smc_start_address;
+
+ spin_lock_irqsave(&adev->smc_idx_lock, flags);
+ while (byte_count >= 4) {
+ /* Bytes are written into the SMC addres space with the MSB first */
+ data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3];
+
+ result = fiji_set_smc_sram_address(adev, addr, limit);
+
+ if (result)
+ goto out;
+
+ WREG32(mmSMC_IND_DATA_0, data);
+
+ src += 4;
+ byte_count -= 4;
+ addr += 4;
+ }
+
+ if (0 != byte_count) {
+ /* Now write odd bytes left, do a read modify write cycle */
+ data = 0;
+
+ result = fiji_set_smc_sram_address(adev, addr, limit);
+ if (result)
+ goto out;
+
+ orig_data = RREG32(mmSMC_IND_DATA_0);
+ extra_shift = 8 * (4 - byte_count);
+
+ while (byte_count > 0) {
+ data = (data << 8) + *src++;
+ byte_count--;
+ }
+
+ data <<= extra_shift;
+ data |= (orig_data & ~((~0UL) << extra_shift));
+
+ result = fiji_set_smc_sram_address(adev, addr, limit);
+ if (result)
+ goto out;
+
+ WREG32(mmSMC_IND_DATA_0, data);
+ }
+
+out:
+ spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+ return result;
+}
+
+static int fiji_program_jump_on_start(struct amdgpu_device *adev)
+{
+ static unsigned char data[] = {0xE0, 0x00, 0x80, 0x40};
+ fiji_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1);
+
+ return 0;
+}
+
+static bool fiji_is_smc_ram_running(struct amdgpu_device *adev)
+{
+ uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
+ val = REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable);
+
+ return ((0 == val) && (0x20100 <= RREG32_SMC(ixSMC_PC_C)));
+}
+
+static int wait_smu_response(struct amdgpu_device *adev)
+{
+ int i;
+ uint32_t val;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32(mmSMC_RESP_0);
+ if (REG_GET_FIELD(val, SMC_RESP_0, SMC_RESP))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int fiji_send_msg_to_smc_offset(struct amdgpu_device *adev)
+{
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send previous message\n");
+ return -EINVAL;
+ }
+
+ WREG32(mmSMC_MSG_ARG_0, 0x20000);
+ WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_Test);
+
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send message\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int fiji_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
+{
+ if (!fiji_is_smc_ram_running(adev))
+ {
+ return -EINVAL;;
+ }
+
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send previous message\n");
+ return -EINVAL;
+ }
+
+ WREG32(mmSMC_MESSAGE_0, msg);
+
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send message\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int fiji_send_msg_to_smc_without_waiting(struct amdgpu_device *adev,
+ PPSMC_Msg msg)
+{
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send previous message\n");
+ return -EINVAL;
+ }
+
+ WREG32(mmSMC_MESSAGE_0, msg);
+
+ return 0;
+}
+
+static int fiji_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
+ PPSMC_Msg msg,
+ uint32_t parameter)
+{
+ if (!fiji_is_smc_ram_running(adev))
+ return -EINVAL;
+
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send previous message\n");
+ return -EINVAL;
+ }
+
+ WREG32(mmSMC_MSG_ARG_0, parameter);
+
+ return fiji_send_msg_to_smc(adev, msg);
+}
+
+static int fiji_send_msg_to_smc_with_parameter_without_waiting(
+ struct amdgpu_device *adev,
+ PPSMC_Msg msg, uint32_t parameter)
+{
+ if (wait_smu_response(adev)) {
+ DRM_ERROR("Failed to send previous message\n");
+ return -EINVAL;
+ }
+
+ WREG32(mmSMC_MSG_ARG_0, parameter);
+
+ return fiji_send_msg_to_smc_without_waiting(adev, msg);
+}
+
+#if 0 /* not used yet */
+static int fiji_wait_for_smc_inactive(struct amdgpu_device *adev)
+{
+ int i;
+ uint32_t val;
+
+ if (!fiji_is_smc_ram_running(adev))
+ return -EINVAL;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
+ if (REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, cken) == 0)
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout)
+ return -EINVAL;
+
+ return 0;
+}
+#endif
+
+static int fiji_smu_upload_firmware_image(struct amdgpu_device *adev)
+{
+ const struct smc_firmware_header_v1_0 *hdr;
+ uint32_t ucode_size;
+ uint32_t ucode_start_address;
+ const uint8_t *src;
+ uint32_t val;
+ uint32_t byte_count;
+ uint32_t *data;
+ unsigned long flags;
+
+ if (!adev->pm.fw)
+ return -EINVAL;
+
+ hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
+ amdgpu_ucode_print_smc_hdr(&hdr->header);
+
+ adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
+ ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
+ ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
+ src = (const uint8_t *)
+ (adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+
+ if (ucode_size & 3) {
+ DRM_ERROR("SMC ucode is not 4 bytes aligned\n");
+ return -EINVAL;
+ }
+
+ if (ucode_size > FIJI_SMC_SIZE) {
+ DRM_ERROR("SMC address is beyond the SMC RAM area\n");
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&adev->smc_idx_lock, flags);
+ WREG32(mmSMC_IND_INDEX_0, ucode_start_address);
+
+ val = RREG32(mmSMC_IND_ACCESS_CNTL);
+ val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1);
+ WREG32(mmSMC_IND_ACCESS_CNTL, val);
+
+ byte_count = ucode_size;
+ data = (uint32_t *)src;
+ for (; byte_count >= 4; data++, byte_count -= 4)
+ WREG32(mmSMC_IND_DATA_0, data[0]);
+
+ val = RREG32(mmSMC_IND_ACCESS_CNTL);
+ val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0);
+ WREG32(mmSMC_IND_ACCESS_CNTL, val);
+ spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+
+ return 0;
+}
+
+#if 0 /* not used yet */
+static int fiji_read_smc_sram_dword(struct amdgpu_device *adev,
+ uint32_t smc_address,
+ uint32_t *value,
+ uint32_t limit)
+{
+ int result;
+ unsigned long flags;
+
+ spin_lock_irqsave(&adev->smc_idx_lock, flags);
+ result = fiji_set_smc_sram_address(adev, smc_address, limit);
+ if (result == 0)
+ *value = RREG32(mmSMC_IND_DATA_0);
+ spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+ return result;
+}
+
+static int fiji_write_smc_sram_dword(struct amdgpu_device *adev,
+ uint32_t smc_address,
+ uint32_t value,
+ uint32_t limit)
+{
+ int result;
+ unsigned long flags;
+
+ spin_lock_irqsave(&adev->smc_idx_lock, flags);
+ result = fiji_set_smc_sram_address(adev, smc_address, limit);
+ if (result == 0)
+ WREG32(mmSMC_IND_DATA_0, value);
+ spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
+ return result;
+}
+
+static int fiji_smu_stop_smc(struct amdgpu_device *adev)
+{
+ uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
+ val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+ WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
+
+ val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
+ val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 1);
+ WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
+
+ return 0;
+}
+#endif
+
+static enum AMDGPU_UCODE_ID fiji_convert_fw_type(uint32_t fw_type)
+{
+ switch (fw_type) {
+ case UCODE_ID_SDMA0:
+ return AMDGPU_UCODE_ID_SDMA0;
+ case UCODE_ID_SDMA1:
+ return AMDGPU_UCODE_ID_SDMA1;
+ case UCODE_ID_CP_CE:
+ return AMDGPU_UCODE_ID_CP_CE;
+ case UCODE_ID_CP_PFP:
+ return AMDGPU_UCODE_ID_CP_PFP;
+ case UCODE_ID_CP_ME:
+ return AMDGPU_UCODE_ID_CP_ME;
+ case UCODE_ID_CP_MEC:
+ case UCODE_ID_CP_MEC_JT1:
+ case UCODE_ID_CP_MEC_JT2:
+ return AMDGPU_UCODE_ID_CP_MEC1;
+ case UCODE_ID_RLC_G:
+ return AMDGPU_UCODE_ID_RLC_G;
+ default:
+ DRM_ERROR("ucode type is out of range!\n");
+ return AMDGPU_UCODE_ID_MAXIMUM;
+ }
+}
+
+static int fiji_smu_populate_single_firmware_entry(struct amdgpu_device *adev,
+ uint32_t fw_type,
+ struct SMU_Entry *entry)
+{
+ enum AMDGPU_UCODE_ID id = fiji_convert_fw_type(fw_type);
+ struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id];
+ const struct gfx_firmware_header_v1_0 *header = NULL;
+ uint64_t gpu_addr;
+ uint32_t data_size;
+
+ if (ucode->fw == NULL)
+ return -EINVAL;
+ gpu_addr = ucode->mc_addr;
+ header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
+ data_size = le32_to_cpu(header->header.ucode_size_bytes);
+
+ if ((fw_type == UCODE_ID_CP_MEC_JT1) ||
+ (fw_type == UCODE_ID_CP_MEC_JT2)) {
+ gpu_addr += le32_to_cpu(header->jt_offset) << 2;
+ data_size = le32_to_cpu(header->jt_size) << 2;
+ }
+
+ entry->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
+ entry->id = (uint16_t)fw_type;
+ entry->image_addr_high = upper_32_bits(gpu_addr);
+ entry->image_addr_low = lower_32_bits(gpu_addr);
+ entry->meta_data_addr_high = 0;
+ entry->meta_data_addr_low = 0;
+ entry->data_size_byte = data_size;
+ entry->num_register_entries = 0;
+
+ if (fw_type == UCODE_ID_RLC_G)
+ entry->flags = 1;
+ else
+ entry->flags = 0;
+
+ return 0;
+}
+
+static int fiji_smu_request_load_fw(struct amdgpu_device *adev)
+{
+ struct fiji_smu_private_data *private = (struct fiji_smu_private_data *)adev->smu.priv;
+ struct SMU_DRAMData_TOC *toc;
+ uint32_t fw_to_load;
+
+ WREG32_SMC(ixSOFT_REGISTERS_TABLE_28, 0);
+
+ fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_HI, private->smu_buffer_addr_high);
+ fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_LO, private->smu_buffer_addr_low);
+
+ toc = (struct SMU_DRAMData_TOC *)private->header;
+ toc->num_entries = 0;
+ toc->structure_version = 1;
+
+ if (!adev->firmware.smu_load)
+ return 0;
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_RLC_G,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for RLC\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_CE,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for CE\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_PFP,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for PFP\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_ME,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for ME\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for MEC\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT1,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for MEC_JT1\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT2,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for MEC_JT2\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA0,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for SDMA0\n");
+ return -EINVAL;
+ }
+
+ if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA1,
+ &toc->entry[toc->num_entries++])) {
+ DRM_ERROR("Failed to get firmware entry for SDMA1\n");
+ return -EINVAL;
+ }
+
+ fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_HI, private->header_addr_high);
+ fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_LO, private->header_addr_low);
+
+ fw_to_load = UCODE_ID_RLC_G_MASK |
+ UCODE_ID_SDMA0_MASK |
+ UCODE_ID_SDMA1_MASK |
+ UCODE_ID_CP_CE_MASK |
+ UCODE_ID_CP_ME_MASK |
+ UCODE_ID_CP_PFP_MASK |
+ UCODE_ID_CP_MEC_MASK;
+
+ if (fiji_send_msg_to_smc_with_parameter_without_waiting(adev, PPSMC_MSG_LoadUcodes, fw_to_load)) {
+ DRM_ERROR("Fail to request SMU load ucode\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static uint32_t fiji_smu_get_mask_for_fw_type(uint32_t fw_type)
+{
+ switch (fw_type) {
+ case AMDGPU_UCODE_ID_SDMA0:
+ return UCODE_ID_SDMA0_MASK;
+ case AMDGPU_UCODE_ID_SDMA1:
+ return UCODE_ID_SDMA1_MASK;
+ case AMDGPU_UCODE_ID_CP_CE:
+ return UCODE_ID_CP_CE_MASK;
+ case AMDGPU_UCODE_ID_CP_PFP:
+ return UCODE_ID_CP_PFP_MASK;
+ case AMDGPU_UCODE_ID_CP_ME:
+ return UCODE_ID_CP_ME_MASK;
+ case AMDGPU_UCODE_ID_CP_MEC1:
+ return UCODE_ID_CP_MEC_MASK;
+ case AMDGPU_UCODE_ID_CP_MEC2:
+ return UCODE_ID_CP_MEC_MASK;
+ case AMDGPU_UCODE_ID_RLC_G:
+ return UCODE_ID_RLC_G_MASK;
+ default:
+ DRM_ERROR("ucode type is out of range!\n");
+ return 0;
+ }
+}
+
+static int fiji_smu_check_fw_load_finish(struct amdgpu_device *adev,
+ uint32_t fw_type)
+{
+ uint32_t fw_mask = fiji_smu_get_mask_for_fw_type(fw_type);
+ int i;
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ if (fw_mask == (RREG32_SMC(ixSOFT_REGISTERS_TABLE_28) & fw_mask))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("check firmware loading failed\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int fiji_smu_start_in_protection_mode(struct amdgpu_device *adev)
+{
+ int result;
+ uint32_t val;
+ int i;
+
+ /* Assert reset */
+ val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
+ val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+ WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
+
+ result = fiji_smu_upload_firmware_image(adev);
+ if (result)
+ return result;
+
+ /* Clear status */
+ WREG32_SMC(ixSMU_STATUS, 0);
+
+ /* Enable clock */
+ val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
+ val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+ WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
+
+ /* De-assert reset */
+ val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
+ val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+ WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
+
+ /* Set SMU Auto Start */
+ val = RREG32_SMC(ixSMU_INPUT_DATA);
+ val = REG_SET_FIELD(val, SMU_INPUT_DATA, AUTO_START, 1);
+ WREG32_SMC(ixSMU_INPUT_DATA, val);
+
+ /* Clear firmware interrupt enable flag */
+ WREG32_SMC(ixFIRMWARE_FLAGS, 0);
+
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixRCU_UC_EVENTS);
+ if (REG_GET_FIELD(val, RCU_UC_EVENTS, INTERRUPTS_ENABLED))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("Interrupt is not enabled by firmware\n");
+ return -EINVAL;
+ }
+
+ /* Call Test SMU message with 0x20000 offset
+ * to trigger SMU start
+ */
+ fiji_send_msg_to_smc_offset(adev);
+ DRM_INFO("[FM]try triger smu start\n");
+ /* Wait for done bit to be set */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixSMU_STATUS);
+ if (REG_GET_FIELD(val, SMU_STATUS, SMU_DONE))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("Timeout for SMU start\n");
+ return -EINVAL;
+ }
+
+ /* Check pass/failed indicator */
+ val = RREG32_SMC(ixSMU_STATUS);
+ if (!REG_GET_FIELD(val, SMU_STATUS, SMU_PASS)) {
+ DRM_ERROR("SMU Firmware start failed\n");
+ return -EINVAL;
+ }
+ DRM_INFO("[FM]smu started\n");
+ /* Wait for firmware to initialize */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixFIRMWARE_FLAGS);
+ if(REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("SMU firmware initialization failed\n");
+ return -EINVAL;
+ }
+ DRM_INFO("[FM]smu initialized\n");
+
+ return 0;
+}
+
+static int fiji_smu_start_in_non_protection_mode(struct amdgpu_device *adev)
+{
+ int i, result;
+ uint32_t val;
+
+ /* wait for smc boot up */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixRCU_UC_EVENTS);
+ val = REG_GET_FIELD(val, RCU_UC_EVENTS, boot_seq_done);
+ if (val)
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("SMC boot sequence is not completed\n");
+ return -EINVAL;
+ }
+
+ /* Clear firmware interrupt enable flag */
+ WREG32_SMC(ixFIRMWARE_FLAGS, 0);
+
+ /* Assert reset */
+ val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
+ val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1);
+ WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
+
+ result = fiji_smu_upload_firmware_image(adev);
+ if (result)
+ return result;
+
+ /* Set smc instruct start point at 0x0 */
+ fiji_program_jump_on_start(adev);
+
+ /* Enable clock */
+ val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
+ val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0);
+ WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val);
+
+ /* De-assert reset */
+ val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL);
+ val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0);
+ WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val);
+
+ /* Wait for firmware to initialize */
+ for (i = 0; i < adev->usec_timeout; i++) {
+ val = RREG32_SMC(ixFIRMWARE_FLAGS);
+ if (REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED))
+ break;
+ udelay(1);
+ }
+
+ if (i == adev->usec_timeout) {
+ DRM_ERROR("Timeout for SMC firmware initialization\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int fiji_smu_start(struct amdgpu_device *adev)
+{
+ int result;
+ uint32_t val;
+
+ if (!fiji_is_smc_ram_running(adev)) {
+ val = RREG32_SMC(ixSMU_FIRMWARE);
+ if (!REG_GET_FIELD(val, SMU_FIRMWARE, SMU_MODE)) {
+ DRM_INFO("[FM]start smu in nonprotection mode\n");
+ result = fiji_smu_start_in_non_protection_mode(adev);
+ if (result)
+ return result;
+ } else {
+ DRM_INFO("[FM]start smu in protection mode\n");
+ result = fiji_smu_start_in_protection_mode(adev);
+ if (result)
+ return result;
+ }
+ }
+
+ return fiji_smu_request_load_fw(adev);
+}
+
+static const struct amdgpu_smumgr_funcs fiji_smumgr_funcs = {
+ .check_fw_load_finish = fiji_smu_check_fw_load_finish,
+ .request_smu_load_fw = NULL,
+ .request_smu_specific_fw = NULL,
+};
+
+int fiji_smu_init(struct amdgpu_device *adev)
+{
+ struct fiji_smu_private_data *private;
+ uint32_t image_size = ((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
+ uint32_t smu_internal_buffer_size = 200*4096;
+ struct amdgpu_bo **toc_buf = &adev->smu.toc_buf;
+ struct amdgpu_bo **smu_buf = &adev->smu.smu_buf;
+ uint64_t mc_addr;
+ void *toc_buf_ptr;
+ void *smu_buf_ptr;
+ int ret;
+
+ private = kzalloc(sizeof(struct fiji_smu_private_data), GFP_KERNEL);
+ if (NULL == private)
+ return -ENOMEM;
+
+ /* allocate firmware buffers */
+ if (adev->firmware.smu_load)
+ amdgpu_ucode_init_bo(adev);
+
+ adev->smu.priv = private;
+ adev->smu.fw_flags = 0;
+
+ /* Allocate FW image data structure and header buffer */
+ ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
+ true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, toc_buf);
+ if (ret) {
+ DRM_ERROR("Failed to allocate memory for TOC buffer\n");
+ return -ENOMEM;
+ }
+
+ /* Allocate buffer for SMU internal buffer */
+ ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
+ true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, smu_buf);
+ if (ret) {
+ DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
+ return -ENOMEM;
+ }
+
+ /* Retrieve GPU address for header buffer and internal buffer */
+ ret = amdgpu_bo_reserve(adev->smu.toc_buf, false);
+ if (ret) {
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to reserve the TOC buffer\n");
+ return -EINVAL;
+ }
+
+ ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
+ if (ret) {
+ amdgpu_bo_unreserve(adev->smu.toc_buf);
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to pin the TOC buffer\n");
+ return -EINVAL;
+ }
+
+ ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr);
+ if (ret) {
+ amdgpu_bo_unreserve(adev->smu.toc_buf);
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to map the TOC buffer\n");
+ return -EINVAL;
+ }
+
+ amdgpu_bo_unreserve(adev->smu.toc_buf);
+ private->header_addr_low = lower_32_bits(mc_addr);
+ private->header_addr_high = upper_32_bits(mc_addr);
+ private->header = toc_buf_ptr;
+
+ ret = amdgpu_bo_reserve(adev->smu.smu_buf, false);
+ if (ret) {
+ amdgpu_bo_unref(&adev->smu.smu_buf);
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to reserve the SMU internal buffer\n");
+ return -EINVAL;
+ }
+
+ ret = amdgpu_bo_pin(adev->smu.smu_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr);
+ if (ret) {
+ amdgpu_bo_unreserve(adev->smu.smu_buf);
+ amdgpu_bo_unref(&adev->smu.smu_buf);
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to pin the SMU internal buffer\n");
+ return -EINVAL;
+ }
+
+ ret = amdgpu_bo_kmap(*smu_buf, &smu_buf_ptr);
+ if (ret) {
+ amdgpu_bo_unreserve(adev->smu.smu_buf);
+ amdgpu_bo_unref(&adev->smu.smu_buf);
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ DRM_ERROR("Failed to map the SMU internal buffer\n");
+ return -EINVAL;
+ }
+
+ amdgpu_bo_unreserve(adev->smu.smu_buf);
+ private->smu_buffer_addr_low = lower_32_bits(mc_addr);
+ private->smu_buffer_addr_high = upper_32_bits(mc_addr);
+
+ adev->smu.smumgr_funcs = &fiji_smumgr_funcs;
+
+ return 0;
+}
+
+int fiji_smu_fini(struct amdgpu_device *adev)
+{
+ amdgpu_bo_unref(&adev->smu.toc_buf);
+ amdgpu_bo_unref(&adev->smu.smu_buf);
+ kfree(adev->smu.priv);
+ adev->smu.priv = NULL;
+ if (adev->firmware.fw_buf)
+ amdgpu_ucode_fini_bo(adev);
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_family.h b/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h
index 0698764354a2..1cef03deeac3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_family.h
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h
@@ -1,7 +1,5 @@
/*
- * Copyright 2008 Advanced Micro Devices, Inc.
- * Copyright 2008 Red Hat Inc.
- * Copyright 2009 Jerome Glisse.
+ * Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -21,42 +19,24 @@
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
- * Authors: Dave Airlie
- * Alex Deucher
- * Jerome Glisse
*/
-/* this file defines the CHIP_ and family flags used in the pciids,
- * its is common between kms and non-kms because duplicating it and
- * changing one place is fail.
- */
-#ifndef AMDGPU_FAMILY_H
-#define AMDGPU_FAMILY_H
-/*
- * Supported ASIC types
- */
-enum amdgpu_asic_type {
- CHIP_BONAIRE = 0,
- CHIP_KAVERI,
- CHIP_KABINI,
- CHIP_HAWAII,
- CHIP_MULLINS,
- CHIP_TOPAZ,
- CHIP_TONGA,
- CHIP_CARRIZO,
- CHIP_LAST,
-};
+#ifndef FIJI_SMUMGR_H
+#define FIJI_SMUMGR_H
-/*
- * Chip flags
- */
-enum amdgpu_chip_flags {
- AMDGPU_ASIC_MASK = 0x0000ffffUL,
- AMDGPU_FLAGS_MASK = 0xffff0000UL,
- AMDGPU_IS_MOBILITY = 0x00010000UL,
- AMDGPU_IS_APU = 0x00020000UL,
- AMDGPU_IS_PX = 0x00040000UL,
- AMDGPU_EXP_HW_SUPPORT = 0x00080000UL,
+#include "fiji_ppsmc.h"
+
+int fiji_smu_init(struct amdgpu_device *adev);
+int fiji_smu_fini(struct amdgpu_device *adev);
+int fiji_smu_start(struct amdgpu_device *adev);
+
+struct fiji_smu_private_data
+{
+ uint8_t *header;
+ uint32_t smu_buffer_addr_high;
+ uint32_t smu_buffer_addr_low;
+ uint32_t header_addr_high;
+ uint32_t header_addr_low;
};
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 0d8bf2cb1956..4bd1e5cf65ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2173,7 +2173,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
adev->gfx.config.mem_max_burst_length_bytes = 256;
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
/* Get memory bank mapping mode. */
tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
@@ -2648,6 +2648,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_ib ib;
+ struct fence *f = NULL;
uint32_t scratch;
uint32_t tmp = 0;
unsigned i;
@@ -2659,29 +2660,27 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
return r;
}
WREG32(scratch, 0xCAFEDEAD);
+ memset(&ib, 0, sizeof(ib));
r = amdgpu_ib_get(ring, NULL, 256, &ib);
if (r) {
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
- amdgpu_gfx_scratch_free(adev, scratch);
- return r;
+ goto err1;
}
ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
ib.ptr[2] = 0xDEADBEEF;
ib.length_dw = 3;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- amdgpu_gfx_scratch_free(adev, scratch);
- amdgpu_ib_free(adev, &ib);
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- return r;
- }
- r = amdgpu_fence_wait(ib.fence, false);
+
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err2;
+
+ r = fence_wait(f, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
- amdgpu_gfx_scratch_free(adev, scratch);
- amdgpu_ib_free(adev, &ib);
- return r;
+ goto err2;
}
for (i = 0; i < adev->usec_timeout; i++) {
tmp = RREG32(scratch);
@@ -2691,14 +2690,19 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
}
if (i < adev->usec_timeout) {
DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
- ib.fence->ring->idx, i);
+ ring->idx, i);
+ goto err2;
} else {
DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
scratch, tmp);
r = -EINVAL;
}
- amdgpu_gfx_scratch_free(adev, scratch);
+
+err2:
+ fence_put(f);
amdgpu_ib_free(adev, &ib);
+err1:
+ amdgpu_gfx_scratch_free(adev, scratch);
return r;
}
@@ -3758,7 +3762,7 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
int r;
/* allocate rlc buffers */
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
if (adev->asic_type == CHIP_KAVERI) {
adev->gfx.rlc.reg_list = spectre_rlc_save_restore_register_list;
adev->gfx.rlc.reg_list_size =
@@ -3782,7 +3786,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
/* save restore block */
if (adev->gfx.rlc.save_restore_obj == NULL) {
r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.save_restore_obj);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &adev->gfx.rlc.save_restore_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r);
return r;
@@ -3823,7 +3829,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
if (adev->gfx.rlc.clear_state_obj == NULL) {
r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.clear_state_obj);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &adev->gfx.rlc.clear_state_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
gfx_v7_0_rlc_fini(adev);
@@ -3860,7 +3868,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
if (adev->gfx.rlc.cp_table_size) {
if (adev->gfx.rlc.cp_table_obj == NULL) {
r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
- AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.cp_table_obj);
+ AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, &adev->gfx.rlc.cp_table_obj);
if (r) {
dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
gfx_v7_0_rlc_fini(adev);
@@ -5594,6 +5604,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
.test_ring = gfx_v7_0_ring_test_ring,
.test_ib = gfx_v7_0_ring_test_ib,
.is_lockup = gfx_v7_0_ring_is_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
@@ -5610,6 +5621,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
.test_ring = gfx_v7_0_ring_test_ring,
.test_ib = gfx_v7_0_ring_test_ib,
.is_lockup = gfx_v7_0_ring_is_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 20e2cfd521d5..53f07439a512 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -87,6 +87,13 @@ MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
MODULE_FIRMWARE("amdgpu/topaz_mec2.bin");
MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
+MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
+MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
+MODULE_FIRMWARE("amdgpu/fiji_me.bin");
+MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
+MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
+MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
+
static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
{
{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
@@ -217,6 +224,71 @@ static const u32 tonga_mgcg_cgcg_init[] =
mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
};
+static const u32 fiji_golden_common_all[] =
+{
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
+ mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
+ mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
+ mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
+ mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
+ mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
+ mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
+};
+
+static const u32 golden_settings_fiji_a10[] =
+{
+ mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
+ mmDB_DEBUG2, 0xf00fffff, 0x00000400,
+ mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
+ mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x00000100,
+ mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+ mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
+ mmTCC_CTRL, 0x00100000, 0xf30fff7f,
+ mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
+ mmTCP_CHAN_STEER_HI, 0xffffffff, 0x7d6cf5e4,
+ mmTCP_CHAN_STEER_LO, 0xffffffff, 0x3928b1a0,
+};
+
+static const u32 fiji_mgcg_cgcg_init[] =
+{
+ mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffc0,
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
+ mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
+ mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
+ mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
+ mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
+ mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
+ mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
+ mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
+ mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
+ mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
+ mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
+ mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
+ mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+ mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
+ mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
+ mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
+ mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
+ mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
+ mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
+ mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
+};
+
static const u32 golden_settings_iceland_a11[] =
{
mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
@@ -439,6 +511,18 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
iceland_golden_common_all,
(const u32)ARRAY_SIZE(iceland_golden_common_all));
break;
+ case CHIP_FIJI:
+ amdgpu_program_register_sequence(adev,
+ fiji_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ golden_settings_fiji_a10,
+ (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+ amdgpu_program_register_sequence(adev,
+ fiji_golden_common_all,
+ (const u32)ARRAY_SIZE(fiji_golden_common_all));
+ break;
+
case CHIP_TONGA:
amdgpu_program_register_sequence(adev,
tonga_mgcg_cgcg_init,
@@ -526,6 +610,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_ib ib;
+ struct fence *f = NULL;
uint32_t scratch;
uint32_t tmp = 0;
unsigned i;
@@ -537,29 +622,27 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
return r;
}
WREG32(scratch, 0xCAFEDEAD);
+ memset(&ib, 0, sizeof(ib));
r = amdgpu_ib_get(ring, NULL, 256, &ib);
if (r) {
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
- amdgpu_gfx_scratch_free(adev, scratch);
- return r;
+ goto err1;
}
ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
ib.ptr[2] = 0xDEADBEEF;
ib.length_dw = 3;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- amdgpu_gfx_scratch_free(adev, scratch);
- amdgpu_ib_free(adev, &ib);
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- return r;
- }
- r = amdgpu_fence_wait(ib.fence, false);
+
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err2;
+
+ r = fence_wait(f, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
- amdgpu_gfx_scratch_free(adev, scratch);
- amdgpu_ib_free(adev, &ib);
- return r;
+ goto err2;
}
for (i = 0; i < adev->usec_timeout; i++) {
tmp = RREG32(scratch);
@@ -569,14 +652,18 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
}
if (i < adev->usec_timeout) {
DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
- ib.fence->ring->idx, i);
+ ring->idx, i);
+ goto err2;
} else {
DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
scratch, tmp);
r = -EINVAL;
}
- amdgpu_gfx_scratch_free(adev, scratch);
+err2:
+ fence_put(f);
amdgpu_ib_free(adev, &ib);
+err1:
+ amdgpu_gfx_scratch_free(adev, scratch);
return r;
}
@@ -601,6 +688,9 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
case CHIP_CARRIZO:
chip_name = "carrizo";
break;
+ case CHIP_FIJI:
+ chip_name = "fiji";
+ break;
default:
BUG();
}
@@ -1236,6 +1326,7 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden;
WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden);
}
+ case CHIP_FIJI:
case CHIP_TONGA:
for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
switch (reg_offset) {
@@ -1914,7 +2005,7 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev,
}
/**
- * gmc_v8_0_init_compute_vmid - gart enable
+ * gfx_v8_0_init_compute_vmid - gart enable
*
* @rdev: amdgpu_device pointer
*
@@ -1924,7 +2015,7 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev,
#define DEFAULT_SH_MEM_BASES (0x6000)
#define FIRST_COMPUTE_VMID (8)
#define LAST_COMPUTE_VMID (16)
-static void gmc_v8_0_init_compute_vmid(struct amdgpu_device *adev)
+static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
{
int i;
uint32_t sh_mem_config;
@@ -1984,6 +2075,23 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
break;
+ case CHIP_FIJI:
+ adev->gfx.config.max_shader_engines = 4;
+ adev->gfx.config.max_tile_pipes = 16;
+ adev->gfx.config.max_cu_per_sh = 16;
+ adev->gfx.config.max_sh_per_se = 1;
+ adev->gfx.config.max_backends_per_se = 4;
+ adev->gfx.config.max_texture_channel_caches = 8;
+ adev->gfx.config.max_gprs = 256;
+ adev->gfx.config.max_gs_threads = 32;
+ adev->gfx.config.max_hw_contexts = 8;
+
+ adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
+ adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
+ adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
+ adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
+ gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
+ break;
case CHIP_TONGA:
adev->gfx.config.max_shader_engines = 4;
adev->gfx.config.max_tile_pipes = 8;
@@ -2078,7 +2186,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
adev->gfx.config.mem_max_burst_length_bytes = 256;
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
/* Get memory bank mapping mode. */
tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
@@ -2174,7 +2282,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
vi_srbm_select(adev, 0, 0, 0, 0);
mutex_unlock(&adev->srbm_mutex);
- gmc_v8_0_init_compute_vmid(adev);
+ gfx_v8_0_init_compute_vmid(adev);
mutex_lock(&adev->grbm_idx_mutex);
/*
@@ -2490,6 +2598,7 @@ static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
switch (adev->asic_type) {
case CHIP_TONGA:
+ case CHIP_FIJI:
amdgpu_ring_write(ring, 0x16000012);
amdgpu_ring_write(ring, 0x0000002A);
break;
@@ -3131,7 +3240,8 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
/* enable the doorbell if requested */
if (use_doorbell) {
- if (adev->asic_type == CHIP_CARRIZO) {
+ if ((adev->asic_type == CHIP_CARRIZO) ||
+ (adev->asic_type == CHIP_FIJI)) {
WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER,
AMDGPU_DOORBELL_KIQ << 2);
WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER,
@@ -3875,7 +3985,8 @@ static bool gfx_v8_0_ring_emit_semaphore(struct amdgpu_ring *ring,
unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
if (ring->adev->asic_type == CHIP_TOPAZ ||
- ring->adev->asic_type == CHIP_TONGA)
+ ring->adev->asic_type == CHIP_TONGA ||
+ ring->adev->asic_type == CHIP_FIJI)
/* we got a hw semaphore bug in VI TONGA, return false to switch back to sw fence wait */
return false;
else {
@@ -4268,6 +4379,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
.test_ring = gfx_v8_0_ring_test_ring,
.test_ib = gfx_v8_0_ring_test_ib,
.is_lockup = gfx_v8_0_ring_is_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
@@ -4284,6 +4396,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
.test_ring = gfx_v8_0_ring_test_ring,
.test_ib = gfx_v8_0_ring_test_ib,
.is_lockup = gfx_v8_0_ring_is_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index ae37fce36520..774528ab8704 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -523,17 +523,11 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
tmp = RREG32(mmVM_CONTEXT1_CNTL);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
amdgpu_vm_block_size - 9);
@@ -636,7 +630,7 @@ static int gmc_v7_0_vm_init(struct amdgpu_device *adev)
adev->vm_manager.nvm = AMDGPU_NUM_OF_VMIDS;
/* base offset of vram pages */
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
tmp <<= 22;
adev->vm_manager.vram_base_offset = tmp;
@@ -841,7 +835,7 @@ static int gmc_v7_0_early_init(void *handle)
gmc_v7_0_set_gart_funcs(adev);
gmc_v7_0_set_irq_funcs(adev);
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
} else {
u32 tmp = RREG32(mmMC_SEQ_MISC0);
@@ -852,6 +846,13 @@ static int gmc_v7_0_early_init(void *handle)
return 0;
}
+static int gmc_v7_0_late_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
+}
+
static int gmc_v7_0_sw_init(void *handle)
{
int r;
@@ -957,7 +958,7 @@ static int gmc_v7_0_hw_init(void *handle)
gmc_v7_0_mc_program(adev);
- if (!(adev->flags & AMDGPU_IS_APU)) {
+ if (!(adev->flags & AMD_IS_APU)) {
r = gmc_v7_0_mc_load_microcode(adev);
if (r) {
DRM_ERROR("Failed to load MC firmware!\n");
@@ -976,6 +977,7 @@ static int gmc_v7_0_hw_fini(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
gmc_v7_0_gart_disable(adev);
return 0;
@@ -1172,7 +1174,7 @@ static int gmc_v7_0_soft_reset(void *handle)
if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
- if (!(adev->flags & AMDGPU_IS_APU))
+ if (!(adev->flags & AMD_IS_APU))
srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
}
@@ -1282,7 +1284,7 @@ static int gmc_v7_0_set_clockgating_state(void *handle,
if (state == AMD_CG_STATE_GATE)
gate = true;
- if (!(adev->flags & AMDGPU_IS_APU)) {
+ if (!(adev->flags & AMD_IS_APU)) {
gmc_v7_0_enable_mc_mgcg(adev, gate);
gmc_v7_0_enable_mc_ls(adev, gate);
}
@@ -1301,7 +1303,7 @@ static int gmc_v7_0_set_powergating_state(void *handle,
const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
.early_init = gmc_v7_0_early_init,
- .late_init = NULL,
+ .late_init = gmc_v7_0_late_init,
.sw_init = gmc_v7_0_sw_init,
.sw_fini = gmc_v7_0_sw_fini,
.hw_init = gmc_v7_0_hw_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 8135963a66be..9a07742620d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -44,6 +44,7 @@ static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
+MODULE_FIRMWARE("amdgpu/fiji_mc.bin");
static const u32 golden_settings_tonga_a11[] =
{
@@ -61,6 +62,19 @@ static const u32 tonga_mgcg_cgcg_init[] =
mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
};
+static const u32 golden_settings_fiji_a10[] =
+{
+ mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+ mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+ mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+ mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
+};
+
+static const u32 fiji_mgcg_cgcg_init[] =
+{
+ mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
+};
+
static const u32 golden_settings_iceland_a11[] =
{
mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
@@ -90,6 +104,14 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
golden_settings_iceland_a11,
(const u32)ARRAY_SIZE(golden_settings_iceland_a11));
break;
+ case CHIP_FIJI:
+ amdgpu_program_register_sequence(adev,
+ fiji_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ golden_settings_fiji_a10,
+ (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+ break;
case CHIP_TONGA:
amdgpu_program_register_sequence(adev,
tonga_mgcg_cgcg_init,
@@ -202,6 +224,9 @@ static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
case CHIP_TONGA:
chip_name = "tonga";
break;
+ case CHIP_FIJI:
+ chip_name = "fiji";
+ break;
case CHIP_CARRIZO:
return 0;
default: BUG();
@@ -628,19 +653,12 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
tmp = RREG32(mmVM_CONTEXT1_CNTL);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
- tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
amdgpu_vm_block_size - 9);
@@ -737,7 +755,7 @@ static int gmc_v8_0_vm_init(struct amdgpu_device *adev)
adev->vm_manager.nvm = AMDGPU_NUM_OF_VMIDS;
/* base offset of vram pages */
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
tmp <<= 22;
adev->vm_manager.vram_base_offset = tmp;
@@ -816,7 +834,7 @@ static int gmc_v8_0_early_init(void *handle)
gmc_v8_0_set_gart_funcs(adev);
gmc_v8_0_set_irq_funcs(adev);
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
} else {
u32 tmp = RREG32(mmMC_SEQ_MISC0);
@@ -827,6 +845,13 @@ static int gmc_v8_0_early_init(void *handle)
return 0;
}
+static int gmc_v8_0_late_init(void *handle)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+ return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
+}
+
static int gmc_v8_0_sw_init(void *handle)
{
int r;
@@ -934,7 +959,7 @@ static int gmc_v8_0_hw_init(void *handle)
gmc_v8_0_mc_program(adev);
- if (!(adev->flags & AMDGPU_IS_APU)) {
+ if (!(adev->flags & AMD_IS_APU)) {
r = gmc_v8_0_mc_load_microcode(adev);
if (r) {
DRM_ERROR("Failed to load MC firmware!\n");
@@ -953,6 +978,7 @@ static int gmc_v8_0_hw_fini(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
gmc_v8_0_gart_disable(adev);
return 0;
@@ -1147,7 +1173,7 @@ static int gmc_v8_0_soft_reset(void *handle)
if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
- if (!(adev->flags & AMDGPU_IS_APU))
+ if (!(adev->flags & AMD_IS_APU))
srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
}
@@ -1263,7 +1289,7 @@ static int gmc_v8_0_set_powergating_state(void *handle,
const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
.early_init = gmc_v8_0_early_init,
- .late_init = NULL,
+ .late_init = gmc_v8_0_late_init,
.sw_init = gmc_v8_0_sw_init,
.sw_fini = gmc_v8_0_sw_fini,
.hw_init = gmc_v8_0_hw_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h
index c723602c7b0c..ee6a041cb288 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h
@@ -2163,5 +2163,10 @@
#define SDMA_PKT_NOP_HEADER_sub_op_shift 8
#define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift)
+/*define for count field*/
+#define SDMA_PKT_NOP_HEADER_count_offset 0
+#define SDMA_PKT_NOP_HEADER_count_mask 0x00003FFF
+#define SDMA_PKT_NOP_HEADER_count_shift 16
+#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift)
#endif /* __ICELAND_SDMA_PKT_OPEN_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
index c6f1e2f12b5f..c900aa942ade 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
@@ -623,7 +623,9 @@ int iceland_smu_init(struct amdgpu_device *adev)
/* Allocate FW image data structure and header buffer */
ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
- true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, toc_buf);
+ true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, toc_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for TOC buffer\n");
return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index a988dfb1d394..14e87234171a 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -146,6 +146,8 @@ static int sdma_v2_4_init_microcode(struct amdgpu_device *adev)
hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma[i].feature_version >= 20)
+ adev->sdma[i].burst_nop = true;
if (adev->firmware.smu_load) {
info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
@@ -218,6 +220,19 @@ static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring)
WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], ring->wptr << 2);
}
+static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ int i;
+
+ for (i = 0; i < count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ amdgpu_ring_write(ring, ring->nop |
+ SDMA_PKT_NOP_HEADER_COUNT(count - 1));
+ else
+ amdgpu_ring_write(ring, ring->nop);
+}
+
/**
* sdma_v2_4_ring_emit_ib - Schedule an IB on the DMA engine
*
@@ -245,8 +260,8 @@ static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring,
amdgpu_ring_write(ring, next_rptr);
/* IB packet must end on a 8 DW boundary */
- while ((ring->wptr & 7) != 2)
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_NOP));
+ sdma_v2_4_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8);
+
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
/* base must be 32 byte aligned */
@@ -673,6 +688,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_ib ib;
+ struct fence *f = NULL;
unsigned i;
unsigned index;
int r;
@@ -688,12 +704,11 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
gpu_addr = adev->wb.gpu_addr + (index * 4);
tmp = 0xCAFEDEAD;
adev->wb.wb[index] = cpu_to_le32(tmp);
-
+ memset(&ib, 0, sizeof(ib));
r = amdgpu_ib_get(ring, NULL, 256, &ib);
if (r) {
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
- return r;
+ goto err0;
}
ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
@@ -707,19 +722,16 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
ib.length_dw = 8;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- return r;
- }
- r = amdgpu_fence_wait(ib.fence, false);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err1;
+
+ r = fence_wait(f, false);
if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
- return r;
+ goto err1;
}
for (i = 0; i < adev->usec_timeout; i++) {
tmp = le32_to_cpu(adev->wb.wb[index]);
@@ -729,12 +741,17 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
}
if (i < adev->usec_timeout) {
DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
- ib.fence->ring->idx, i);
+ ring->idx, i);
+ goto err1;
} else {
DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
r = -EINVAL;
}
+
+err1:
+ fence_put(f);
amdgpu_ib_free(adev, &ib);
+err0:
amdgpu_wb_free(adev, index);
return r;
}
@@ -877,8 +894,19 @@ static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib,
*/
static void sdma_v2_4_vm_pad_ib(struct amdgpu_ib *ib)
{
- while (ib->length_dw & 0x7)
- ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ u32 pad_count;
+ int i;
+
+ pad_count = (8 - (ib->length_dw & 0x7)) % 8;
+ for (i = 0; i < pad_count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
+ SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
+ else
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
}
/**
@@ -1312,6 +1340,7 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
.test_ring = sdma_v2_4_ring_test_ring,
.test_ib = sdma_v2_4_ring_test_ib,
.is_lockup = sdma_v2_4_ring_is_lockup,
+ .insert_nop = sdma_v2_4_ring_insert_nop,
};
static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)
@@ -1348,19 +1377,19 @@ static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev)
* Used by the amdgpu ttm implementation to move pages if
* registered as the asic copy callback.
*/
-static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ring *ring,
+static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ib *ib,
uint64_t src_offset,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
- SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR));
- amdgpu_ring_write(ring, byte_count);
- amdgpu_ring_write(ring, 0); /* src/dst endian swap */
- amdgpu_ring_write(ring, lower_32_bits(src_offset));
- amdgpu_ring_write(ring, upper_32_bits(src_offset));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+ ib->ptr[ib->length_dw++] = byte_count;
+ ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+ ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
}
/**
@@ -1373,16 +1402,16 @@ static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ring *ring,
*
* Fill GPU buffers using the DMA engine (VI).
*/
-static void sdma_v2_4_emit_fill_buffer(struct amdgpu_ring *ring,
+static void sdma_v2_4_emit_fill_buffer(struct amdgpu_ib *ib,
uint32_t src_data,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
- amdgpu_ring_write(ring, src_data);
- amdgpu_ring_write(ring, byte_count);
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = src_data;
+ ib->ptr[ib->length_dw++] = byte_count;
}
static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = {
@@ -1415,5 +1444,6 @@ static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev)
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &sdma_v2_4_vm_pte_funcs;
adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 2b86569b18d3..9bfe92df15f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -53,6 +53,8 @@ MODULE_FIRMWARE("amdgpu/tonga_sdma.bin");
MODULE_FIRMWARE("amdgpu/tonga_sdma1.bin");
MODULE_FIRMWARE("amdgpu/carrizo_sdma.bin");
MODULE_FIRMWARE("amdgpu/carrizo_sdma1.bin");
+MODULE_FIRMWARE("amdgpu/fiji_sdma.bin");
+MODULE_FIRMWARE("amdgpu/fiji_sdma1.bin");
static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
{
@@ -80,6 +82,24 @@ static const u32 tonga_mgcg_cgcg_init[] =
mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100
};
+static const u32 golden_settings_fiji_a10[] =
+{
+ mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
+ mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+ mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+ mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+ mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007,
+ mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100,
+ mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100,
+ mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100,
+};
+
+static const u32 fiji_mgcg_cgcg_init[] =
+{
+ mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100,
+ mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100
+};
+
static const u32 cz_golden_settings_a11[] =
{
mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007,
@@ -122,6 +142,14 @@ static const u32 cz_mgcg_cgcg_init[] =
static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev)
{
switch (adev->asic_type) {
+ case CHIP_FIJI:
+ amdgpu_program_register_sequence(adev,
+ fiji_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+ amdgpu_program_register_sequence(adev,
+ golden_settings_fiji_a10,
+ (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+ break;
case CHIP_TONGA:
amdgpu_program_register_sequence(adev,
tonga_mgcg_cgcg_init,
@@ -167,6 +195,9 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev)
case CHIP_TONGA:
chip_name = "tonga";
break;
+ case CHIP_FIJI:
+ chip_name = "fiji";
+ break;
case CHIP_CARRIZO:
chip_name = "carrizo";
break;
@@ -187,6 +218,8 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev)
hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data;
adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
+ if (adev->sdma[i].feature_version >= 20)
+ adev->sdma[i].burst_nop = true;
if (adev->firmware.smu_load) {
info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
@@ -273,6 +306,19 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
}
}
+static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring);
+ int i;
+
+ for (i = 0; i < count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ amdgpu_ring_write(ring, ring->nop |
+ SDMA_PKT_NOP_HEADER_COUNT(count - 1));
+ else
+ amdgpu_ring_write(ring, ring->nop);
+}
+
/**
* sdma_v3_0_ring_emit_ib - Schedule an IB on the DMA engine
*
@@ -299,8 +345,7 @@ static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
amdgpu_ring_write(ring, next_rptr);
/* IB packet must end on a 8 DW boundary */
- while ((ring->wptr & 7) != 2)
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_NOP));
+ sdma_v3_0_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8);
amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
@@ -763,6 +808,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_ib ib;
+ struct fence *f = NULL;
unsigned i;
unsigned index;
int r;
@@ -778,12 +824,11 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
gpu_addr = adev->wb.gpu_addr + (index * 4);
tmp = 0xCAFEDEAD;
adev->wb.wb[index] = cpu_to_le32(tmp);
-
+ memset(&ib, 0, sizeof(ib));
r = amdgpu_ib_get(ring, NULL, 256, &ib);
if (r) {
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
- return r;
+ goto err0;
}
ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
@@ -797,19 +842,16 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
ib.length_dw = 8;
- r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED);
- if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
- DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r);
- return r;
- }
- r = amdgpu_fence_wait(ib.fence, false);
+ r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,
+ AMDGPU_FENCE_OWNER_UNDEFINED,
+ &f);
+ if (r)
+ goto err1;
+
+ r = fence_wait(f, false);
if (r) {
- amdgpu_ib_free(adev, &ib);
- amdgpu_wb_free(adev, index);
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
- return r;
+ goto err1;
}
for (i = 0; i < adev->usec_timeout; i++) {
tmp = le32_to_cpu(adev->wb.wb[index]);
@@ -819,12 +861,16 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
}
if (i < adev->usec_timeout) {
DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
- ib.fence->ring->idx, i);
+ ring->idx, i);
+ goto err1;
} else {
DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
r = -EINVAL;
}
+err1:
+ fence_put(f);
amdgpu_ib_free(adev, &ib);
+err0:
amdgpu_wb_free(adev, index);
return r;
}
@@ -967,8 +1013,19 @@ static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib,
*/
static void sdma_v3_0_vm_pad_ib(struct amdgpu_ib *ib)
{
- while (ib->length_dw & 0x7)
- ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+ struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring);
+ u32 pad_count;
+ int i;
+
+ pad_count = (8 - (ib->length_dw & 0x7)) % 8;
+ for (i = 0; i < pad_count; i++)
+ if (sdma && sdma->burst_nop && (i == 0))
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
+ SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
+ else
+ ib->ptr[ib->length_dw++] =
+ SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
}
/**
@@ -1406,6 +1463,7 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
.test_ring = sdma_v3_0_ring_test_ring,
.test_ib = sdma_v3_0_ring_test_ib,
.is_lockup = sdma_v3_0_ring_is_lockup,
+ .insert_nop = sdma_v3_0_ring_insert_nop,
};
static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -1442,19 +1500,19 @@ static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev)
* Used by the amdgpu ttm implementation to move pages if
* registered as the asic copy callback.
*/
-static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ring *ring,
+static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib,
uint64_t src_offset,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
- SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR));
- amdgpu_ring_write(ring, byte_count);
- amdgpu_ring_write(ring, 0); /* src/dst endian swap */
- amdgpu_ring_write(ring, lower_32_bits(src_offset));
- amdgpu_ring_write(ring, upper_32_bits(src_offset));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
+ SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+ ib->ptr[ib->length_dw++] = byte_count;
+ ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
+ ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
}
/**
@@ -1467,16 +1525,16 @@ static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ring *ring,
*
* Fill GPU buffers using the DMA engine (VI).
*/
-static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ring *ring,
+static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib,
uint32_t src_data,
uint64_t dst_offset,
uint32_t byte_count)
{
- amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL));
- amdgpu_ring_write(ring, lower_32_bits(dst_offset));
- amdgpu_ring_write(ring, upper_32_bits(dst_offset));
- amdgpu_ring_write(ring, src_data);
- amdgpu_ring_write(ring, byte_count);
+ ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
+ ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
+ ib->ptr[ib->length_dw++] = src_data;
+ ib->ptr[ib->length_dw++] = byte_count;
}
static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
@@ -1509,5 +1567,6 @@ static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev)
if (adev->vm_manager.vm_pte_funcs == NULL) {
adev->vm_manager.vm_pte_funcs = &sdma_v3_0_vm_pte_funcs;
adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring;
+ adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;
}
}
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h
index 099b7b56113c..e5ebd084288d 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h
@@ -2236,5 +2236,10 @@
#define SDMA_PKT_NOP_HEADER_sub_op_shift 8
#define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift)
+/*define for count field*/
+#define SDMA_PKT_NOP_HEADER_count_offset 0
+#define SDMA_PKT_NOP_HEADER_count_mask 0x00003FFF
+#define SDMA_PKT_NOP_HEADER_count_shift 16
+#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift)
#endif /* __TONGA_SDMA_PKT_OPEN_H_ */
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
index 5fc53a40c7ac..1f5ac941a610 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
@@ -761,7 +761,9 @@ int tonga_smu_init(struct amdgpu_device *adev)
/* Allocate FW image data structure and header buffer */
ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE,
- true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, toc_buf);
+ true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, toc_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for TOC buffer\n");
return -ENOMEM;
@@ -769,7 +771,9 @@ int tonga_smu_init(struct amdgpu_device *adev)
/* Allocate buffer for SMU internal buffer */
ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE,
- true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, smu_buf);
+ true, AMDGPU_GEM_DOMAIN_VRAM,
+ AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+ NULL, smu_buf);
if (ret) {
DRM_ERROR("Failed to allocate memory for SMU internal buffer\n");
return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 4efd671d7a9b..5fac5da694f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -534,7 +534,7 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
int r;
r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
@@ -555,14 +555,14 @@ static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring)
goto error;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
goto error;
}
DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
error:
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
amdgpu_asic_set_uvd_clocks(adev, 0, 0);
return r;
}
@@ -886,6 +886,7 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
.test_ring = uvd_v4_2_ring_test_ring,
.test_ib = uvd_v4_2_ring_test_ib,
.is_lockup = amdgpu_ring_test_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index b756bd99c0fd..2d5c59c318af 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -580,7 +580,7 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring)
{
struct amdgpu_device *adev = ring->adev;
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
int r;
r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
@@ -601,14 +601,14 @@ static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring)
goto error;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
goto error;
}
DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
error:
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
amdgpu_asic_set_uvd_clocks(adev, 0, 0);
return r;
}
@@ -825,6 +825,7 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
.test_ring = uvd_v5_0_ring_test_ring,
.test_ib = uvd_v5_0_ring_test_ib,
.is_lockup = amdgpu_ring_test_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 49aa931b2cb4..d9f553fce531 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -575,7 +575,7 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
*/
static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring)
{
- struct amdgpu_fence *fence = NULL;
+ struct fence *fence = NULL;
int r;
r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
@@ -590,14 +590,14 @@ static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring)
goto error;
}
- r = amdgpu_fence_wait(fence, false);
+ r = fence_wait(fence, false);
if (r) {
DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
goto error;
}
DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
error:
- amdgpu_fence_unref(&fence);
+ fence_put(fence);
return r;
}
@@ -805,6 +805,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_funcs = {
.test_ring = uvd_v6_0_ring_test_ring,
.test_ib = uvd_v6_0_ring_test_ib,
.is_lockup = amdgpu_ring_test_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 303d961d57bd..cd16df543f64 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -643,6 +643,7 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
.test_ring = amdgpu_vce_ring_test_ring,
.test_ib = amdgpu_vce_ring_test_ib,
.is_lockup = amdgpu_ring_test_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index d1064ca3670e..f0656dfb53f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -32,8 +32,8 @@
#include "vid.h"
#include "vce/vce_3_0_d.h"
#include "vce/vce_3_0_sh_mask.h"
-#include "oss/oss_2_0_d.h"
-#include "oss/oss_2_0_sh_mask.h"
+#include "oss/oss_3_0_d.h"
+#include "oss/oss_3_0_sh_mask.h"
#include "gca/gfx_8_0_d.h"
#include "smu/smu_7_1_2_d.h"
#include "smu/smu_7_1_2_sh_mask.h"
@@ -205,7 +205,14 @@ static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev)
u32 tmp;
unsigned ret;
- if (adev->flags & AMDGPU_IS_APU)
+ /* Fiji is single pipe */
+ if (adev->asic_type == CHIP_FIJI) {
+ ret = AMDGPU_VCE_HARVEST_VCE1;
+ return ret;
+ }
+
+ /* Tonga and CZ are dual or single pipe */
+ if (adev->flags & AMD_IS_APU)
tmp = (RREG32_SMC(ixVCE_HARVEST_FUSE_MACRO__ADDRESS) &
VCE_HARVEST_FUSE_MACRO__MASK) >>
VCE_HARVEST_FUSE_MACRO__SHIFT;
@@ -419,17 +426,41 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
static bool vce_v3_0_is_idle(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ u32 mask = 0;
+ int idx;
+
+ for (idx = 0; idx < 2; ++idx) {
+ if (adev->vce.harvest_config & (1 << idx))
+ continue;
+
+ if (idx == 0)
+ mask |= SRBM_STATUS2__VCE0_BUSY_MASK;
+ else
+ mask |= SRBM_STATUS2__VCE1_BUSY_MASK;
+ }
- return !(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK);
+ return !(RREG32(mmSRBM_STATUS2) & mask);
}
static int vce_v3_0_wait_for_idle(void *handle)
{
unsigned i;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ u32 mask = 0;
+ int idx;
+
+ for (idx = 0; idx < 2; ++idx) {
+ if (adev->vce.harvest_config & (1 << idx))
+ continue;
+
+ if (idx == 0)
+ mask |= SRBM_STATUS2__VCE0_BUSY_MASK;
+ else
+ mask |= SRBM_STATUS2__VCE1_BUSY_MASK;
+ }
for (i = 0; i < adev->usec_timeout; i++) {
- if (!(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK))
+ if (!(RREG32(mmSRBM_STATUS2) & mask))
return 0;
}
return -ETIMEDOUT;
@@ -438,9 +469,21 @@ static int vce_v3_0_wait_for_idle(void *handle)
static int vce_v3_0_soft_reset(void *handle)
{
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+ u32 mask = 0;
+ int idx;
- WREG32_P(mmSRBM_SOFT_RESET, SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK,
- ~SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK);
+ for (idx = 0; idx < 2; ++idx) {
+ if (adev->vce.harvest_config & (1 << idx))
+ continue;
+
+ if (idx == 0)
+ mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK;
+ else
+ mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK;
+ }
+ WREG32_P(mmSRBM_SOFT_RESET, mask,
+ ~(SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK |
+ SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK));
mdelay(5);
return vce_v3_0_start(adev);
@@ -601,6 +644,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = {
.test_ring = amdgpu_vce_ring_test_ring,
.test_ib = amdgpu_vce_ring_test_ib,
.is_lockup = amdgpu_ring_test_lockup,
+ .insert_nop = amdgpu_ring_insert_nop,
};
static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 68552da40287..552d9e75ad1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -203,6 +203,17 @@ static const u32 tonga_mgcg_cgcg_init[] =
mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
};
+static const u32 fiji_mgcg_cgcg_init[] =
+{
+ mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
+ mmPCIE_INDEX, 0xffffffff, 0x0140001c,
+ mmPCIE_DATA, 0x000f0000, 0x00000000,
+ mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C,
+ mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100,
+ mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
+ mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
+};
+
static const u32 iceland_mgcg_cgcg_init[] =
{
mmPCIE_INDEX, 0xffffffff, ixPCIE_CNTL2,
@@ -232,6 +243,11 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
iceland_mgcg_cgcg_init,
(const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
break;
+ case CHIP_FIJI:
+ amdgpu_program_register_sequence(adev,
+ fiji_mgcg_cgcg_init,
+ (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+ break;
case CHIP_TONGA:
amdgpu_program_register_sequence(adev,
tonga_mgcg_cgcg_init,
@@ -261,7 +277,7 @@ static u32 vi_get_xclk(struct amdgpu_device *adev)
u32 reference_clock = adev->clock.spll.reference_freq;
u32 tmp;
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return reference_clock;
tmp = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
@@ -362,6 +378,26 @@ static struct amdgpu_allowed_register_entry cz_allowed_read_registers[] = {
static struct amdgpu_allowed_register_entry vi_allowed_read_registers[] = {
{mmGRBM_STATUS, false},
+ {mmGRBM_STATUS2, false},
+ {mmGRBM_STATUS_SE0, false},
+ {mmGRBM_STATUS_SE1, false},
+ {mmGRBM_STATUS_SE2, false},
+ {mmGRBM_STATUS_SE3, false},
+ {mmSRBM_STATUS, false},
+ {mmSRBM_STATUS2, false},
+ {mmSRBM_STATUS3, false},
+ {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET, false},
+ {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET, false},
+ {mmCP_STAT, false},
+ {mmCP_STALLED_STAT1, false},
+ {mmCP_STALLED_STAT2, false},
+ {mmCP_STALLED_STAT3, false},
+ {mmCP_CPF_BUSY_STAT, false},
+ {mmCP_CPF_STALLED_STAT1, false},
+ {mmCP_CPF_STATUS, false},
+ {mmCP_CPC_BUSY_STAT, false},
+ {mmCP_CPC_STALLED_STAT1, false},
+ {mmCP_CPC_STATUS, false},
{mmGB_ADDR_CONFIG, false},
{mmMC_ARB_RAMCFG, false},
{mmGB_TILE_MODE0, false},
@@ -449,6 +485,7 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
asic_register_table = tonga_allowed_read_registers;
size = ARRAY_SIZE(tonga_allowed_read_registers);
break;
+ case CHIP_FIJI:
case CHIP_TONGA:
case CHIP_CARRIZO:
asic_register_table = cz_allowed_read_registers;
@@ -751,7 +788,7 @@ static void vi_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask)
srbm_soft_reset =
REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
- if (!(adev->flags & AMDGPU_IS_APU)) {
+ if (!(adev->flags & AMD_IS_APU)) {
if (reset_mask & AMDGPU_RESET_MC)
srbm_soft_reset =
REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
@@ -971,7 +1008,7 @@ static void vi_pcie_gen3_enable(struct amdgpu_device *adev)
if (amdgpu_pcie_gen2 == 0)
return;
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return;
ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
@@ -999,7 +1036,7 @@ static void vi_enable_doorbell_aperture(struct amdgpu_device *adev,
u32 tmp;
/* not necessary on CZ */
- if (adev->flags & AMDGPU_IS_APU)
+ if (adev->flags & AMD_IS_APU)
return;
tmp = RREG32(mmBIF_DOORBELL_APER_EN);
@@ -1127,6 +1164,74 @@ static const struct amdgpu_ip_block_version tonga_ip_blocks[] =
},
};
+static const struct amdgpu_ip_block_version fiji_ip_blocks[] =
+{
+ /* ORDER MATTERS! */
+ {
+ .type = AMD_IP_BLOCK_TYPE_COMMON,
+ .major = 2,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &vi_common_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_GMC,
+ .major = 8,
+ .minor = 5,
+ .rev = 0,
+ .funcs = &gmc_v8_0_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_IH,
+ .major = 3,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &tonga_ih_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_SMC,
+ .major = 7,
+ .minor = 1,
+ .rev = 0,
+ .funcs = &fiji_dpm_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_DCE,
+ .major = 10,
+ .minor = 1,
+ .rev = 0,
+ .funcs = &dce_v10_0_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_GFX,
+ .major = 8,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &gfx_v8_0_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_SDMA,
+ .major = 3,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &sdma_v3_0_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_UVD,
+ .major = 6,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &uvd_v6_0_ip_funcs,
+ },
+ {
+ .type = AMD_IP_BLOCK_TYPE_VCE,
+ .major = 3,
+ .minor = 0,
+ .rev = 0,
+ .funcs = &vce_v3_0_ip_funcs,
+ },
+};
+
static const struct amdgpu_ip_block_version cz_ip_blocks[] =
{
/* ORDER MATTERS! */
@@ -1202,6 +1307,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
adev->ip_blocks = topaz_ip_blocks;
adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks);
break;
+ case CHIP_FIJI:
+ adev->ip_blocks = fiji_ip_blocks;
+ adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks);
+ break;
case CHIP_TONGA:
adev->ip_blocks = tonga_ip_blocks;
adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks);
@@ -1248,7 +1357,7 @@ static int vi_common_early_init(void *handle)
bool smc_enabled = false;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- if (adev->flags & AMDGPU_IS_APU) {
+ if (adev->flags & AMD_IS_APU) {
adev->smc_rreg = &cz_smc_rreg;
adev->smc_wreg = &cz_smc_wreg;
} else {
@@ -1279,6 +1388,7 @@ static int vi_common_early_init(void *handle)
if (amdgpu_smc_load_fw && smc_enabled)
adev->firmware.smu_load = true;
break;
+ case CHIP_FIJI:
case CHIP_TONGA:
adev->has_uvd = true;
adev->cg_flags = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
index 3b45332f5df4..fc120ba18aad 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h
@@ -30,7 +30,7 @@ int cz_smu_start(struct amdgpu_device *adev);
int cz_smu_fini(struct amdgpu_device *adev);
extern const struct amd_ip_funcs tonga_dpm_ip_funcs;
-
+extern const struct amd_ip_funcs fiji_dpm_ip_funcs;
extern const struct amd_ip_funcs iceland_dpm_ip_funcs;
#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index 31bb89452e12..d98aa9d82fa1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -66,6 +66,11 @@
#define AMDGPU_NUM_OF_VMIDS 8
+#define PIPEID(x) ((x) << 0)
+#define MEID(x) ((x) << 2)
+#define VMID(x) ((x) << 4)
+#define QUEUEID(x) ((x) << 8)
+
#define RB_BITMAP_WIDTH_PER_SH 2
#define MC_SEQ_MISC0__MT__MASK 0xf0000000
diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig
index 8dfac37ff327..e13c67c8d2c0 100644
--- a/drivers/gpu/drm/amd/amdkfd/Kconfig
+++ b/drivers/gpu/drm/amd/amdkfd/Kconfig
@@ -4,6 +4,6 @@
config HSA_AMD
tristate "HSA kernel driver for AMD GPU devices"
- depends on DRM_RADEON && AMD_IOMMU_V2 && X86_64
+ depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64
help
Enable this if you want to use HSA features on AMD GPU devices.
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 28551153ec6d..7fc9b0f444cb 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -2,7 +2,8 @@
# Makefile for Heterogenous System Architecture support for AMD GPU devices
#
-ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/
+ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/ \
+ -Idrivers/gpu/drm/amd/include/asic_reg
amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
diff --git a/drivers/gpu/drm/amd/amdkfd/cik_regs.h b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
index 183be5b8414f..48769d12dd7b 100644
--- a/drivers/gpu/drm/amd/amdkfd/cik_regs.h
+++ b/drivers/gpu/drm/amd/amdkfd/cik_regs.h
@@ -65,17 +65,6 @@
#define AQL_ENABLE 1
-#define SDMA_RB_VMID(x) (x << 24)
-#define SDMA_RB_ENABLE (1 << 0)
-#define SDMA_RB_SIZE(x) ((x) << 1) /* log2 */
-#define SDMA_RPTR_WRITEBACK_ENABLE (1 << 12)
-#define SDMA_RPTR_WRITEBACK_TIMER(x) ((x) << 16) /* log2 */
-#define SDMA_OFFSET(x) (x << 0)
-#define SDMA_DB_ENABLE (1 << 28)
-#define SDMA_ATC (1 << 0)
-#define SDMA_VA_PTR32 (1 << 4)
-#define SDMA_VA_SHARED_BASE(x) (x << 8)
-
#define GRBM_GFX_INDEX 0x30800
#define ATC_VMID_PASID_MAPPING_VALID (1U << 31)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index c991973019d0..c6a1b4cc6458 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -31,7 +31,7 @@
#include <uapi/linux/kfd_ioctl.h>
#include <linux/time.h>
#include <linux/mm.h>
-#include <uapi/asm-generic/mman-common.h>
+#include <linux/mman.h>
#include <asm/processor.h>
#include "kfd_priv.h"
#include "kfd_device_queue_manager.h"
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 75312c82969f..3f95f7cb4019 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -80,7 +80,12 @@ static const struct kfd_deviceid supported_devices[] = {
{ 0x1318, &kaveri_device_info }, /* Kaveri */
{ 0x131B, &kaveri_device_info }, /* Kaveri */
{ 0x131C, &kaveri_device_info }, /* Kaveri */
- { 0x131D, &kaveri_device_info } /* Kaveri */
+ { 0x131D, &kaveri_device_info }, /* Kaveri */
+ { 0x9870, &carrizo_device_info }, /* Carrizo */
+ { 0x9874, &carrizo_device_info }, /* Carrizo */
+ { 0x9875, &carrizo_device_info }, /* Carrizo */
+ { 0x9876, &carrizo_device_info }, /* Carrizo */
+ { 0x9877, &carrizo_device_info } /* Carrizo */
};
static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index 9ce8a20a7aff..c6f435aa803f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -23,6 +23,7 @@
#include "kfd_device_queue_manager.h"
#include "cik_regs.h"
+#include "oss/oss_2_4_sh_mask.h"
static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
@@ -135,13 +136,16 @@ static int register_process_cik(struct device_queue_manager *dqm,
static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd)
{
- uint32_t value = SDMA_ATC;
+ uint32_t value = (1 << SDMA0_RLC0_VIRTUAL_ADDR__ATC__SHIFT);
if (q->process->is_32bit_user_mode)
- value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd));
+ value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) |
+ get_sh_mem_bases_32(qpd_to_pdd(qpd));
else
- value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64(
- qpd_to_pdd(qpd)));
+ value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
+ SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
+ SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
+
q->properties.sdma_vm_addr = value;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index 4c15212a3899..7e9cae9d349b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -22,6 +22,10 @@
*/
#include "kfd_device_queue_manager.h"
+#include "gca/gfx_8_0_enum.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gca/gfx_8_0_enum.h"
+#include "oss/oss_3_0_sh_mask.h"
static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
@@ -37,14 +41,40 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops)
{
- pr_warn("amdkfd: VI DQM is not currently supported\n");
-
ops->set_cache_memory_policy = set_cache_memory_policy_vi;
ops->register_process = register_process_vi;
ops->initialize = initialize_cpsch_vi;
ops->init_sdma_vm = init_sdma_vm;
}
+static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+{
+ /* In 64-bit mode, we can only control the top 3 bits of the LDS,
+ * scratch and GPUVM apertures.
+ * The hardware fills in the remaining 59 bits according to the
+ * following pattern:
+ * LDS: X0000000'00000000 - X0000001'00000000 (4GB)
+ * Scratch: X0000001'00000000 - X0000002'00000000 (4GB)
+ * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB)
+ *
+ * (where X/Y is the configurable nybble with the low-bit 0)
+ *
+ * LDS and scratch will have the same top nybble programmed in the
+ * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
+ * GPUVM can have a different top nybble programmed in the
+ * top 3 bits of SH_MEM_BASES.SHARED_BASE.
+ * We don't bother to support different top nybbles
+ * for LDS/Scratch and GPUVM.
+ */
+
+ BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+ top_address_nybble == 0);
+
+ return top_address_nybble << 12 |
+ (top_address_nybble << 12) <<
+ SH_MEM_BASES__SHARED_BASE__SHIFT;
+}
+
static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
enum cache_policy default_policy,
@@ -52,18 +82,83 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
void __user *alternate_aperture_base,
uint64_t alternate_aperture_size)
{
- return false;
+ uint32_t default_mtype;
+ uint32_t ape1_mtype;
+
+ default_mtype = (default_policy == cache_policy_coherent) ?
+ MTYPE_CC :
+ MTYPE_NC;
+
+ ape1_mtype = (alternate_policy == cache_policy_coherent) ?
+ MTYPE_CC :
+ MTYPE_NC;
+
+ qpd->sh_mem_config = (qpd->sh_mem_config &
+ SH_MEM_CONFIG__ADDRESS_MODE_MASK) |
+ SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+ SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+ default_mtype << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+ ape1_mtype << SH_MEM_CONFIG__APE1_MTYPE__SHIFT |
+ SH_MEM_CONFIG__PRIVATE_ATC_MASK;
+
+ return true;
}
static int register_process_vi(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
- return -1;
+ struct kfd_process_device *pdd;
+ unsigned int temp;
+
+ BUG_ON(!dqm || !qpd);
+
+ pdd = qpd_to_pdd(qpd);
+
+ /* check if sh_mem_config register already configured */
+ if (qpd->sh_mem_config == 0) {
+ qpd->sh_mem_config =
+ SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
+ SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
+ MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
+ MTYPE_CC << SH_MEM_CONFIG__APE1_MTYPE__SHIFT |
+ SH_MEM_CONFIG__PRIVATE_ATC_MASK;
+
+ qpd->sh_mem_ape1_limit = 0;
+ qpd->sh_mem_ape1_base = 0;
+ }
+
+ if (qpd->pqm->process->is_32bit_user_mode) {
+ temp = get_sh_mem_bases_32(pdd);
+ qpd->sh_mem_bases = temp << SH_MEM_BASES__SHARED_BASE__SHIFT;
+ qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA32 <<
+ SH_MEM_CONFIG__ADDRESS_MODE__SHIFT;
+ } else {
+ temp = get_sh_mem_bases_nybble_64(pdd);
+ qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+ qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA64 <<
+ SH_MEM_CONFIG__ADDRESS_MODE__SHIFT;
+ }
+
+ pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+ qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
+
+ return 0;
}
static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd)
{
+ uint32_t value = (1 << SDMA0_RLC0_VIRTUAL_ADDR__ATC__SHIFT);
+
+ if (q->process->is_32bit_user_mode)
+ value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) |
+ get_sh_mem_bases_32(qpd_to_pdd(qpd));
+ else
+ value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
+ SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
+ SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
+
+ q->properties.sdma_vm_addr = value;
}
static int initialize_cpsch_vi(struct device_queue_manager *dqm)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 35b987574633..2b655103ba79 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -33,7 +33,7 @@
#include <linux/time.h>
#include "kfd_priv.h"
#include <linux/mm.h>
-#include <uapi/asm-generic/mman-common.h>
+#include <linux/mman.h>
#include <asm/processor.h>
/*
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 434979428fc0..d83de985e88c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -27,6 +27,7 @@
#include "kfd_mqd_manager.h"
#include "cik_regs.h"
#include "cik_structs.h"
+#include "oss/oss_2_4_sh_mask.h"
static inline struct cik_mqd *get_mqd(void *mqd)
{
@@ -214,17 +215,20 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
BUG_ON(!mm || !mqd || !q);
m = get_sdma_mqd(mqd);
- m->sdma_rlc_rb_cntl =
- SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) |
- SDMA_RB_VMID(q->vmid) |
- SDMA_RPTR_WRITEBACK_ENABLE |
- SDMA_RPTR_WRITEBACK_TIMER(6);
+ m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) <<
+ SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+ q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+ 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+ 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8);
m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8);
m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
- m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE;
+ m->sdma_rlc_doorbell = q->doorbell_off <<
+ SDMA0_RLC0_DOORBELL__OFFSET__SHIFT |
+ 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT;
+
m->sdma_rlc_virtual_addr = q->sdma_vm_addr;
m->sdma_engine_id = q->sdma_engine_id;
@@ -234,7 +238,9 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
if (q->queue_size > 0 &&
q->queue_address != 0 &&
q->queue_percent > 0) {
- m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE;
+ m->sdma_rlc_rb_cntl |=
+ 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT;
+
q->is_active = true;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index b3a7e3ba1e38..fa32c32fa1c2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -22,12 +22,255 @@
*/
#include <linux/printk.h>
+#include <linux/slab.h>
#include "kfd_priv.h"
#include "kfd_mqd_manager.h"
+#include "vi_structs.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "gca/gfx_8_0_enum.h"
+
+#define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
+
+static inline struct vi_mqd *get_mqd(void *mqd)
+{
+ return (struct vi_mqd *)mqd;
+}
+
+static int init_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ int retval;
+ uint64_t addr;
+ struct vi_mqd *m;
+
+ retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd),
+ mqd_mem_obj);
+ if (retval != 0)
+ return -ENOMEM;
+
+ m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr;
+ addr = (*mqd_mem_obj)->gpu_addr;
+
+ memset(m, 0, sizeof(struct vi_mqd));
+
+ m->header = 0xC0310800;
+ m->compute_pipelinestat_enable = 1;
+ m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
+ m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+
+ m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
+ 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
+
+ m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT |
+ MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT;
+
+ m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+ m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
+ m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
+ 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
+ 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
+
+ m->cp_hqd_pipe_priority = 1;
+ m->cp_hqd_queue_priority = 15;
+
+ m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL)
+ m->cp_hqd_iq_rptr = 1;
+
+ *mqd = m;
+ if (gart_addr != NULL)
+ *gart_addr = addr;
+ retval = mm->update_mqd(mm, m, q);
+
+ return retval;
+}
+
+static int load_mqd(struct mqd_manager *mm, void *mqd,
+ uint32_t pipe_id, uint32_t queue_id,
+ uint32_t __user *wptr)
+{
+ return mm->dev->kfd2kgd->hqd_load
+ (mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+}
+
+static int __update_mqd(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q, unsigned int mtype,
+ unsigned int atc_bit)
+{
+ struct vi_mqd *m;
+
+ BUG_ON(!mm || !q || !mqd);
+
+ pr_debug("kfd: In func %s\n", __func__);
+
+ m = get_mqd(mqd);
+
+ m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
+ atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT |
+ mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
+ m->cp_hqd_pq_control |=
+ ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+ pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+
+ m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
+ m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
+
+ m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+ m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+
+ m->cp_hqd_pq_doorbell_control =
+ 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT |
+ q->doorbell_off <<
+ CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+ pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n",
+ m->cp_hqd_pq_doorbell_control);
+
+ m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT |
+ mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT;
+
+ m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT |
+ 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
+ mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT;
+
+ m->cp_hqd_eop_control |=
+ ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1;
+ m->cp_hqd_eop_base_addr_lo =
+ lower_32_bits(q->eop_ring_buffer_address >> 8);
+ m->cp_hqd_eop_base_addr_hi =
+ upper_32_bits(q->eop_ring_buffer_address >> 8);
+
+ m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT |
+ mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT;
+
+ m->cp_hqd_vmid = q->vmid;
+
+ if (q->format == KFD_QUEUE_FORMAT_AQL) {
+ m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
+ 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
+ }
+
+ m->cp_hqd_active = 0;
+ q->is_active = false;
+ if (q->queue_size > 0 &&
+ q->queue_address != 0 &&
+ q->queue_percent > 0) {
+ m->cp_hqd_active = 1;
+ q->is_active = true;
+ }
+
+ return 0;
+}
+
+
+static int update_mqd(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
+}
+
+static int destroy_mqd(struct mqd_manager *mm, void *mqd,
+ enum kfd_preempt_type type,
+ unsigned int timeout, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_destroy
+ (mm->dev->kgd, type, timeout,
+ pipe_id, queue_id);
+}
+
+static void uninit_mqd(struct mqd_manager *mm, void *mqd,
+ struct kfd_mem_obj *mqd_mem_obj)
+{
+ BUG_ON(!mm || !mqd);
+ kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+}
+
+static bool is_occupied(struct mqd_manager *mm, void *mqd,
+ uint64_t queue_address, uint32_t pipe_id,
+ uint32_t queue_id)
+{
+ return mm->dev->kfd2kgd->hqd_is_occupied(
+ mm->dev->kgd, queue_address,
+ pipe_id, queue_id);
+}
+
+static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+ struct queue_properties *q)
+{
+ struct vi_mqd *m;
+ int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
+
+ if (retval != 0)
+ return retval;
+
+ m = get_mqd(*mqd);
+
+ m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
+ 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
+
+ return retval;
+}
+
+static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
+ struct queue_properties *q)
+{
+ struct vi_mqd *m;
+ int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0);
+
+ if (retval != 0)
+ return retval;
+
+ m = get_mqd(mqd);
+ m->cp_hqd_vmid = q->vmid;
+ return retval;
+}
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
- struct kfd_dev *dev)
+ struct kfd_dev *dev)
{
- pr_warn("amdkfd: VI MQD is not currently supported\n");
- return NULL;
+ struct mqd_manager *mqd;
+
+ BUG_ON(!dev);
+ BUG_ON(type >= KFD_MQD_TYPE_MAX);
+
+ pr_debug("kfd: In func %s\n", __func__);
+
+ mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
+ if (!mqd)
+ return NULL;
+
+ mqd->dev = dev;
+
+ switch (type) {
+ case KFD_MQD_TYPE_CP:
+ case KFD_MQD_TYPE_COMPUTE:
+ mqd->init_mqd = init_mqd;
+ mqd->uninit_mqd = uninit_mqd;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ break;
+ case KFD_MQD_TYPE_HIQ:
+ mqd->init_mqd = init_mqd_hiq;
+ mqd->uninit_mqd = uninit_mqd;
+ mqd->load_mqd = load_mqd;
+ mqd->update_mqd = update_mqd_hiq;
+ mqd->destroy_mqd = destroy_mqd;
+ mqd->is_occupied = is_occupied;
+ break;
+ case KFD_MQD_TYPE_SDMA:
+ break;
+ default:
+ kfree(mqd);
+ return NULL;
+ }
+
+ return mqd;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 99b6d28a11c3..90f391434fa3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -27,6 +27,7 @@
#include "kfd_kernel_queue.h"
#include "kfd_priv.h"
#include "kfd_pm4_headers.h"
+#include "kfd_pm4_headers_vi.h"
#include "kfd_pm4_opcodes.h"
static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
@@ -55,6 +56,7 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
bool *over_subscription)
{
unsigned int process_count, queue_count;
+ unsigned int map_queue_size;
BUG_ON(!pm || !rlib_size || !over_subscription);
@@ -69,9 +71,13 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
pr_debug("kfd: over subscribed runlist\n");
}
+ map_queue_size =
+ (pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ?
+ sizeof(struct pm4_mes_map_queues) :
+ sizeof(struct pm4_map_queues);
/* calculate run list ib allocation size */
*rlib_size = process_count * sizeof(struct pm4_map_process) +
- queue_count * sizeof(struct pm4_map_queues);
+ queue_count * map_queue_size;
/*
* Increase the allocation size in case we need a chained run list
@@ -176,6 +182,71 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
return 0;
}
+static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
+ struct queue *q, bool is_static)
+{
+ struct pm4_mes_map_queues *packet;
+ bool use_static = is_static;
+
+ BUG_ON(!pm || !buffer || !q);
+
+ pr_debug("kfd: In func %s\n", __func__);
+
+ packet = (struct pm4_mes_map_queues *)buffer;
+ memset(buffer, 0, sizeof(struct pm4_map_queues));
+
+ packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
+ sizeof(struct pm4_map_queues));
+ packet->bitfields2.alloc_format =
+ alloc_format__mes_map_queues__one_per_pipe_vi;
+ packet->bitfields2.num_queues = 1;
+ packet->bitfields2.queue_sel =
+ queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi;
+
+ packet->bitfields2.engine_sel =
+ engine_sel__mes_map_queues__compute_vi;
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__normal_compute_vi;
+
+ switch (q->properties.type) {
+ case KFD_QUEUE_TYPE_COMPUTE:
+ if (use_static)
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__normal_latency_static_queue_vi;
+ break;
+ case KFD_QUEUE_TYPE_DIQ:
+ packet->bitfields2.queue_type =
+ queue_type__mes_map_queues__debug_interface_queue_vi;
+ break;
+ case KFD_QUEUE_TYPE_SDMA:
+ packet->bitfields2.engine_sel =
+ engine_sel__mes_map_queues__sdma0_vi;
+ use_static = false; /* no static queues under SDMA */
+ break;
+ default:
+ pr_err("kfd: in %s queue type %d\n", __func__,
+ q->properties.type);
+ BUG();
+ break;
+ }
+ packet->bitfields3.doorbell_offset =
+ q->properties.doorbell_off;
+
+ packet->mqd_addr_lo =
+ lower_32_bits(q->gart_mqd_addr);
+
+ packet->mqd_addr_hi =
+ upper_32_bits(q->gart_mqd_addr);
+
+ packet->wptr_addr_lo =
+ lower_32_bits((uint64_t)q->properties.write_ptr);
+
+ packet->wptr_addr_hi =
+ upper_32_bits((uint64_t)q->properties.write_ptr);
+
+ return 0;
+}
+
static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
struct queue *q, bool is_static)
{
@@ -292,8 +363,17 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
pr_debug("kfd: static_queue, mapping kernel q %d, is debug status %d\n",
kq->queue->queue, qpd->is_debug);
- retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr],
- kq->queue, qpd->is_debug);
+ if (pm->dqm->dev->device_info->asic_family ==
+ CHIP_CARRIZO)
+ retval = pm_create_map_queue_vi(pm,
+ &rl_buffer[rl_wptr],
+ kq->queue,
+ qpd->is_debug);
+ else
+ retval = pm_create_map_queue(pm,
+ &rl_buffer[rl_wptr],
+ kq->queue,
+ qpd->is_debug);
if (retval != 0)
return retval;
@@ -309,8 +389,17 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
pr_debug("kfd: static_queue, mapping user queue %d, is debug status %d\n",
q->queue, qpd->is_debug);
- retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr],
- q, qpd->is_debug);
+ if (pm->dqm->dev->device_info->asic_family ==
+ CHIP_CARRIZO)
+ retval = pm_create_map_queue_vi(pm,
+ &rl_buffer[rl_wptr],
+ q,
+ qpd->is_debug);
+ else
+ retval = pm_create_map_queue(pm,
+ &rl_buffer[rl_wptr],
+ q,
+ qpd->is_debug);
if (retval != 0)
return retval;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
new file mode 100644
index 000000000000..08c721922812
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
@@ -0,0 +1,398 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef F32_MES_PM4_PACKETS_H
+#define F32_MES_PM4_PACKETS_H
+
+#ifndef PM4_MES_HEADER_DEFINED
+#define PM4_MES_HEADER_DEFINED
+union PM4_MES_TYPE_3_HEADER {
+ struct {
+ uint32_t reserved1 : 8; /* < reserved */
+ uint32_t opcode : 8; /* < IT opcode */
+ uint32_t count : 14;/* < number of DWORDs - 1 in the
+ information body. */
+ uint32_t type : 2; /* < packet identifier.
+ It should be 3 for type 3 packets */
+ };
+ uint32_t u32All;
+};
+#endif /* PM4_MES_HEADER_DEFINED */
+
+/*--------------------MES_SET_RESOURCES--------------------*/
+
+#ifndef PM4_MES_SET_RESOURCES_DEFINED
+#define PM4_MES_SET_RESOURCES_DEFINED
+enum mes_set_resources_queue_type_enum {
+ queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
+ queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
+ queue_type__mes_set_resources__hsa_debug_interface_queue = 4
+};
+
+
+struct pm4_mes_set_resources {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ uint32_t vmid_mask:16;
+ uint32_t unmap_latency:8;
+ uint32_t reserved1:5;
+ enum mes_set_resources_queue_type_enum queue_type:3;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
+ uint32_t queue_mask_lo;
+ uint32_t queue_mask_hi;
+ uint32_t gws_mask_lo;
+ uint32_t gws_mask_hi;
+
+ union {
+ struct {
+ uint32_t oac_mask:16;
+ uint32_t reserved2:16;
+ } bitfields7;
+ uint32_t ordinal7;
+ };
+
+ union {
+ struct {
+ uint32_t gds_heap_base:6;
+ uint32_t reserved3:5;
+ uint32_t gds_heap_size:6;
+ uint32_t reserved4:15;
+ } bitfields8;
+ uint32_t ordinal8;
+ };
+
+};
+#endif
+
+/*--------------------MES_RUN_LIST--------------------*/
+
+#ifndef PM4_MES_RUN_LIST_DEFINED
+#define PM4_MES_RUN_LIST_DEFINED
+
+struct pm4_mes_runlist {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ uint32_t reserved1:2;
+ uint32_t ib_base_lo:30;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
+ union {
+ struct {
+ uint32_t ib_base_hi:16;
+ uint32_t reserved2:16;
+ } bitfields3;
+ uint32_t ordinal3;
+ };
+
+ union {
+ struct {
+ uint32_t ib_size:20;
+ uint32_t chain:1;
+ uint32_t offload_polling:1;
+ uint32_t reserved3:1;
+ uint32_t valid:1;
+ uint32_t reserved4:8;
+ } bitfields4;
+ uint32_t ordinal4;
+ };
+
+};
+#endif
+
+/*--------------------MES_MAP_PROCESS--------------------*/
+
+#ifndef PM4_MES_MAP_PROCESS_DEFINED
+#define PM4_MES_MAP_PROCESS_DEFINED
+
+struct pm4_mes_map_process {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ uint32_t pasid:16;
+ uint32_t reserved1:8;
+ uint32_t diq_enable:1;
+ uint32_t process_quantum:7;
+ } bitfields2;
+ uint32_t ordinal2;
+};
+
+ union {
+ struct {
+ uint32_t page_table_base:28;
+ uint32_t reserved2:4;
+ } bitfields3;
+ uint32_t ordinal3;
+ };
+
+ uint32_t sh_mem_bases;
+ uint32_t sh_mem_ape1_base;
+ uint32_t sh_mem_ape1_limit;
+ uint32_t sh_mem_config;
+ uint32_t gds_addr_lo;
+ uint32_t gds_addr_hi;
+
+ union {
+ struct {
+ uint32_t num_gws:6;
+ uint32_t reserved3:2;
+ uint32_t num_oac:4;
+ uint32_t reserved4:4;
+ uint32_t gds_size:6;
+ uint32_t num_queues:10;
+ } bitfields10;
+ uint32_t ordinal10;
+ };
+
+};
+#endif
+
+/*--------------------MES_MAP_QUEUES--------------------*/
+
+#ifndef PM4_MES_MAP_QUEUES_VI_DEFINED
+#define PM4_MES_MAP_QUEUES_VI_DEFINED
+enum mes_map_queues_queue_sel_vi_enum {
+ queue_sel__mes_map_queues__map_to_specified_queue_slots_vi = 0,
+queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi = 1
+};
+
+enum mes_map_queues_queue_type_vi_enum {
+ queue_type__mes_map_queues__normal_compute_vi = 0,
+ queue_type__mes_map_queues__debug_interface_queue_vi = 1,
+ queue_type__mes_map_queues__normal_latency_static_queue_vi = 2,
+queue_type__mes_map_queues__low_latency_static_queue_vi = 3
+};
+
+enum mes_map_queues_alloc_format_vi_enum {
+ alloc_format__mes_map_queues__one_per_pipe_vi = 0,
+alloc_format__mes_map_queues__all_on_one_pipe_vi = 1
+};
+
+enum mes_map_queues_engine_sel_vi_enum {
+ engine_sel__mes_map_queues__compute_vi = 0,
+ engine_sel__mes_map_queues__sdma0_vi = 2,
+ engine_sel__mes_map_queues__sdma1_vi = 3
+};
+
+
+struct pm4_mes_map_queues {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ uint32_t reserved1:4;
+ enum mes_map_queues_queue_sel_vi_enum queue_sel:2;
+ uint32_t reserved2:15;
+ enum mes_map_queues_queue_type_vi_enum queue_type:3;
+ enum mes_map_queues_alloc_format_vi_enum alloc_format:2;
+ enum mes_map_queues_engine_sel_vi_enum engine_sel:3;
+ uint32_t num_queues:3;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
+ union {
+ struct {
+ uint32_t reserved3:1;
+ uint32_t check_disable:1;
+ uint32_t doorbell_offset:21;
+ uint32_t reserved4:3;
+ uint32_t queue:6;
+ } bitfields3;
+ uint32_t ordinal3;
+ };
+
+ uint32_t mqd_addr_lo;
+ uint32_t mqd_addr_hi;
+ uint32_t wptr_addr_lo;
+ uint32_t wptr_addr_hi;
+};
+#endif
+
+/*--------------------MES_QUERY_STATUS--------------------*/
+
+#ifndef PM4_MES_QUERY_STATUS_DEFINED
+#define PM4_MES_QUERY_STATUS_DEFINED
+enum mes_query_status_interrupt_sel_enum {
+ interrupt_sel__mes_query_status__completion_status = 0,
+ interrupt_sel__mes_query_status__process_status = 1,
+ interrupt_sel__mes_query_status__queue_status = 2
+};
+
+enum mes_query_status_command_enum {
+ command__mes_query_status__interrupt_only = 0,
+ command__mes_query_status__fence_only_immediate = 1,
+ command__mes_query_status__fence_only_after_write_ack = 2,
+ command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
+};
+
+enum mes_query_status_engine_sel_enum {
+ engine_sel__mes_query_status__compute = 0,
+ engine_sel__mes_query_status__sdma0_queue = 2,
+ engine_sel__mes_query_status__sdma1_queue = 3
+};
+
+struct pm4_mes_query_status {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ uint32_t context_id:28;
+ enum mes_query_status_interrupt_sel_enum
+ interrupt_sel:2;
+ enum mes_query_status_command_enum command:2;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
+ union {
+ struct {
+ uint32_t pasid:16;
+ uint32_t reserved1:16;
+ } bitfields3a;
+ struct {
+ uint32_t reserved2:2;
+ uint32_t doorbell_offset:21;
+ uint32_t reserved3:2;
+ enum mes_query_status_engine_sel_enum engine_sel:3;
+ uint32_t reserved4:4;
+ } bitfields3b;
+ uint32_t ordinal3;
+ };
+
+ uint32_t addr_lo;
+ uint32_t addr_hi;
+ uint32_t data_lo;
+ uint32_t data_hi;
+};
+#endif
+
+/*--------------------MES_UNMAP_QUEUES--------------------*/
+
+#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
+#define PM4_MES_UNMAP_QUEUES_DEFINED
+enum mes_unmap_queues_action_enum {
+ action__mes_unmap_queues__preempt_queues = 0,
+ action__mes_unmap_queues__reset_queues = 1,
+ action__mes_unmap_queues__disable_process_queues = 2,
+ action__mes_unmap_queues__reserved = 3
+};
+
+enum mes_unmap_queues_queue_sel_enum {
+ queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
+ queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
+ queue_sel__mes_unmap_queues__unmap_all_queues = 2,
+ queue_sel__mes_unmap_queues__unmap_all_non_static_queues = 3
+};
+
+enum mes_unmap_queues_engine_sel_enum {
+ engine_sel__mes_unmap_queues__compute = 0,
+ engine_sel__mes_unmap_queues__sdma0 = 2,
+ engine_sel__mes_unmap_queues__sdmal = 3
+};
+
+struct PM4_MES_UNMAP_QUEUES {
+ union {
+ union PM4_MES_TYPE_3_HEADER header; /* header */
+ uint32_t ordinal1;
+ };
+
+ union {
+ struct {
+ enum mes_unmap_queues_action_enum action:2;
+ uint32_t reserved1:2;
+ enum mes_unmap_queues_queue_sel_enum queue_sel:2;
+ uint32_t reserved2:20;
+ enum mes_unmap_queues_engine_sel_enum engine_sel:3;
+ uint32_t num_queues:3;
+ } bitfields2;
+ uint32_t ordinal2;
+ };
+
+ union {
+ struct {
+ uint32_t pasid:16;
+ uint32_t reserved3:16;
+ } bitfields3a;
+ struct {
+ uint32_t reserved4:2;
+ uint32_t doorbell_offset0:21;
+ uint32_t reserved5:9;
+ } bitfields3b;
+ uint32_t ordinal3;
+ };
+
+ union {
+ struct {
+ uint32_t reserved6:2;
+ uint32_t doorbell_offset1:21;
+ uint32_t reserved7:9;
+ } bitfields4;
+ uint32_t ordinal4;
+ };
+
+ union {
+ struct {
+ uint32_t reserved8:2;
+ uint32_t doorbell_offset2:21;
+ uint32_t reserved9:9;
+ } bitfields5;
+ uint32_t ordinal5;
+ };
+
+ union {
+ struct {
+ uint32_t reserved10:2;
+ uint32_t doorbell_offset3:21;
+ uint32_t reserved11:9;
+ } bitfields6;
+ uint32_t ordinal6;
+ };
+};
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index c25728bc388a..74909e72a009 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1186,6 +1186,11 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
* TODO: Retrieve max engine clock values from KGD
*/
+ if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
+ dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
+ pr_info("amdkfd: adding doorbell packet type capability\n");
+ }
+
res = 0;
err:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 989624b3cd14..c3ddb9b95ff8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -40,6 +40,7 @@
#define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00
#define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8
#define HSA_CAP_RESERVED 0xfffff000
+#define HSA_CAP_DOORBELL_PACKET_TYPE 0x00001000
struct kfd_node_properties {
uint32_t cpu_cores_count;
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 5bdf1b4397a0..68a8eaa1b7d0 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -23,6 +23,45 @@
#ifndef __AMD_SHARED_H__
#define __AMD_SHARED_H__
+#define AMD_MAX_USEC_TIMEOUT 100000 /* 100 ms */
+
+/*
+* Supported GPU families (aligned with amdgpu_drm.h)
+*/
+#define AMD_FAMILY_UNKNOWN 0
+#define AMD_FAMILY_CI 120 /* Bonaire, Hawaii */
+#define AMD_FAMILY_KV 125 /* Kaveri, Kabini, Mullins */
+#define AMD_FAMILY_VI 130 /* Iceland, Tonga */
+#define AMD_FAMILY_CZ 135 /* Carrizo */
+
+/*
+ * Supported ASIC types
+ */
+enum amd_asic_type {
+ CHIP_BONAIRE = 0,
+ CHIP_KAVERI,
+ CHIP_KABINI,
+ CHIP_HAWAII,
+ CHIP_MULLINS,
+ CHIP_TOPAZ,
+ CHIP_TONGA,
+ CHIP_FIJI,
+ CHIP_CARRIZO,
+ CHIP_LAST,
+};
+
+/*
+ * Chip flags
+ */
+enum amd_chip_flags {
+ AMD_ASIC_MASK = 0x0000ffffUL,
+ AMD_FLAGS_MASK = 0xffff0000UL,
+ AMD_IS_MOBILITY = 0x00010000UL,
+ AMD_IS_APU = 0x00020000UL,
+ AMD_IS_PX = 0x00040000UL,
+ AMD_EXP_HW_SUPPORT = 0x00080000UL,
+};
+
enum amd_ip_block_type {
AMD_IP_BLOCK_TYPE_COMMON,
AMD_IP_BLOCK_TYPE_GMC,
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h
new file mode 100644
index 000000000000..44b1855cb8df
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h
@@ -0,0 +1,1246 @@
+/*
+ * SMU_7_1_3 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SMU_7_1_3_D_H
+#define SMU_7_1_3_D_H
+
+#define mmGCK_SMC_IND_INDEX 0x80
+#define mmGCK0_GCK_SMC_IND_INDEX 0x80
+#define mmGCK1_GCK_SMC_IND_INDEX 0x82
+#define mmGCK2_GCK_SMC_IND_INDEX 0x84
+#define mmGCK3_GCK_SMC_IND_INDEX 0x86
+#define mmGCK_SMC_IND_DATA 0x81
+#define mmGCK0_GCK_SMC_IND_DATA 0x81
+#define mmGCK1_GCK_SMC_IND_DATA 0x83
+#define mmGCK2_GCK_SMC_IND_DATA 0x85
+#define mmGCK3_GCK_SMC_IND_DATA 0x87
+#define ixGCK_MCLK_FUSES 0xc0500008
+#define ixCG_DCLK_CNTL 0xc050009c
+#define ixCG_DCLK_STATUS 0xc05000a0
+#define ixCG_VCLK_CNTL 0xc05000a4
+#define ixCG_VCLK_STATUS 0xc05000a8
+#define ixCG_ECLK_CNTL 0xc05000ac
+#define ixCG_ECLK_STATUS 0xc05000b0
+#define ixCG_ACLK_CNTL 0xc05000dc
+#define ixCG_MCLK_CNTL 0xc0500120
+#define ixCG_MCLK_STATUS 0xc0500124
+#define ixGCK_DFS_BYPASS_CNTL 0xc0500118
+#define ixCG_SPLL_FUNC_CNTL 0xc0500140
+#define ixCG_SPLL_FUNC_CNTL_2 0xc0500144
+#define ixCG_SPLL_FUNC_CNTL_3 0xc0500148
+#define ixCG_SPLL_FUNC_CNTL_4 0xc050014c
+#define ixCG_SPLL_FUNC_CNTL_5 0xc0500150
+#define ixCG_SPLL_FUNC_CNTL_6 0xc0500154
+#define ixCG_SPLL_FUNC_CNTL_7 0xc0500158
+#define ixSPLL_CNTL_MODE 0xc0500160
+#define ixCG_SPLL_SPREAD_SPECTRUM 0xc0500164
+#define ixCG_SPLL_SPREAD_SPECTRUM_2 0xc0500168
+#define ixMPLL_BYPASSCLK_SEL 0xc050019c
+#define ixCG_CLKPIN_CNTL 0xc05001a0
+#define ixCG_CLKPIN_CNTL_2 0xc05001a4
+#define ixCG_CLKPIN_CNTL_DC 0xc0500204
+#define ixTHM_CLK_CNTL 0xc05001a8
+#define ixMISC_CLK_CTRL 0xc05001ac
+#define ixGCK_PLL_TEST_CNTL 0xc05001c0
+#define ixGCK_PLL_TEST_CNTL_2 0xc05001c4
+#define ixGCK_ADFS_CLK_BYPASS_CNTL1 0xc05001c8
+#define mmSMC_IND_INDEX 0x80
+#define mmSMC0_SMC_IND_INDEX 0x80
+#define mmSMC1_SMC_IND_INDEX 0x82
+#define mmSMC2_SMC_IND_INDEX 0x84
+#define mmSMC3_SMC_IND_INDEX 0x86
+#define mmSMC_IND_DATA 0x81
+#define mmSMC0_SMC_IND_DATA 0x81
+#define mmSMC1_SMC_IND_DATA 0x83
+#define mmSMC2_SMC_IND_DATA 0x85
+#define mmSMC3_SMC_IND_DATA 0x87
+#define mmSMC_IND_INDEX_0 0x80
+#define mmSMC_IND_DATA_0 0x81
+#define mmSMC_IND_INDEX_1 0x82
+#define mmSMC_IND_DATA_1 0x83
+#define mmSMC_IND_INDEX_2 0x84
+#define mmSMC_IND_DATA_2 0x85
+#define mmSMC_IND_INDEX_3 0x86
+#define mmSMC_IND_DATA_3 0x87
+#define mmSMC_IND_INDEX_4 0x88
+#define mmSMC_IND_DATA_4 0x89
+#define mmSMC_IND_INDEX_5 0x8a
+#define mmSMC_IND_DATA_5 0x8b
+#define mmSMC_IND_INDEX_6 0x8c
+#define mmSMC_IND_DATA_6 0x8d
+#define mmSMC_IND_INDEX_7 0x8e
+#define mmSMC_IND_DATA_7 0x8f
+#define mmSMC_IND_ACCESS_CNTL 0x92
+#define mmSMC_MESSAGE_0 0x94
+#define mmSMC_RESP_0 0x95
+#define mmSMC_MESSAGE_1 0x96
+#define mmSMC_RESP_1 0x97
+#define mmSMC_MESSAGE_2 0x98
+#define mmSMC_RESP_2 0x99
+#define mmSMC_MESSAGE_3 0x9a
+#define mmSMC_RESP_3 0x9b
+#define mmSMC_MESSAGE_4 0x9c
+#define mmSMC_RESP_4 0x9d
+#define mmSMC_MESSAGE_5 0x9e
+#define mmSMC_RESP_5 0x9f
+#define mmSMC_MESSAGE_6 0xa0
+#define mmSMC_RESP_6 0xa1
+#define mmSMC_MESSAGE_7 0xa2
+#define mmSMC_RESP_7 0xa3
+#define mmSMC_MSG_ARG_0 0xa4
+#define mmSMC_MSG_ARG_1 0xa5
+#define mmSMC_MSG_ARG_2 0xa6
+#define mmSMC_MSG_ARG_3 0xa7
+#define mmSMC_MSG_ARG_4 0xa8
+#define mmSMC_MSG_ARG_5 0xa9
+#define mmSMC_MSG_ARG_6 0xaa
+#define mmSMC_MSG_ARG_7 0xab
+#define mmSMC_MESSAGE_8 0xb5
+#define mmSMC_RESP_8 0xb6
+#define mmSMC_MESSAGE_9 0xb7
+#define mmSMC_RESP_9 0xb8
+#define mmSMC_MESSAGE_10 0xb9
+#define mmSMC_RESP_10 0xba
+#define mmSMC_MESSAGE_11 0xbb
+#define mmSMC_RESP_11 0xbc
+#define mmSMC_MSG_ARG_8 0xbd
+#define mmSMC_MSG_ARG_9 0xbe
+#define mmSMC_MSG_ARG_10 0xbf
+#define mmSMC_MSG_ARG_11 0x93
+#define ixSMC_SYSCON_RESET_CNTL 0x80000000
+#define ixSMC_SYSCON_CLOCK_CNTL_0 0x80000004
+#define ixSMC_SYSCON_CLOCK_CNTL_1 0x80000008
+#define ixSMC_SYSCON_CLOCK_CNTL_2 0x8000000c
+#define ixSMC_SYSCON_MISC_CNTL 0x80000010
+#define ixSMC_SYSCON_MSG_ARG_0 0x80000068
+#define ixSMC_PC_C 0x80000370
+#define ixSMC_SCRATCH9 0x80000424
+#define mmGPIOPAD_SW_INT_STAT 0x180
+#define mmGPIOPAD_STRENGTH 0x181
+#define mmGPIOPAD_MASK 0x182
+#define mmGPIOPAD_A 0x183
+#define mmGPIOPAD_EN 0x184
+#define mmGPIOPAD_Y 0x185
+#define mmGPIOPAD_PINSTRAPS 0x186
+#define mmGPIOPAD_INT_STAT_EN 0x187
+#define mmGPIOPAD_INT_STAT 0x188
+#define mmGPIOPAD_INT_STAT_AK 0x189
+#define mmGPIOPAD_INT_EN 0x18a
+#define mmGPIOPAD_INT_TYPE 0x18b
+#define mmGPIOPAD_INT_POLARITY 0x18c
+#define mmGPIOPAD_EXTERN_TRIG_CNTL 0x18d
+#define mmGPIOPAD_RCVR_SEL 0x191
+#define mmGPIOPAD_PU_EN 0x192
+#define mmGPIOPAD_PD_EN 0x193
+#define mmCG_FPS_CNT 0x1b6
+#define mmSMU_IND_INDEX_0 0x1a6
+#define mmSMU_IND_DATA_0 0x1a7
+#define mmSMU_IND_INDEX_1 0x1a8
+#define mmSMU_IND_DATA_1 0x1a9
+#define mmSMU_IND_INDEX_2 0x1aa
+#define mmSMU_IND_DATA_2 0x1ab
+#define mmSMU_IND_INDEX_3 0x1ac
+#define mmSMU_IND_DATA_3 0x1ad
+#define mmSMU_IND_INDEX_4 0x1ae
+#define mmSMU_IND_DATA_4 0x1af
+#define mmSMU_IND_INDEX_5 0x1b0
+#define mmSMU_IND_DATA_5 0x1b1
+#define mmSMU_IND_INDEX_6 0x1b2
+#define mmSMU_IND_DATA_6 0x1b3
+#define mmSMU_IND_INDEX_7 0x1b4
+#define mmSMU_IND_DATA_7 0x1b5
+#define mmSMU_SMC_IND_INDEX 0x80
+#define mmSMU0_SMU_SMC_IND_INDEX 0x80
+#define mmSMU1_SMU_SMC_IND_INDEX 0x82
+#define mmSMU2_SMU_SMC_IND_INDEX 0x84
+#define mmSMU3_SMU_SMC_IND_INDEX 0x86
+#define mmSMU_SMC_IND_DATA 0x81
+#define mmSMU0_SMU_SMC_IND_DATA 0x81
+#define mmSMU1_SMU_SMC_IND_DATA 0x83
+#define mmSMU2_SMU_SMC_IND_DATA 0x85
+#define mmSMU3_SMU_SMC_IND_DATA 0x87
+#define ixRCU_UC_EVENTS 0xc0000004
+#define ixRCU_MISC_CTRL 0xc0000010
+#define ixRCU_VIRT_RESET_REQ 0xc0000024
+#define ixCC_RCU_FUSES 0xc00c0000
+#define ixCC_SMU_MISC_FUSES 0xc00c0004
+#define ixCC_SCLK_VID_FUSES 0xc00c0008
+#define ixCC_GIO_IOCCFG_FUSES 0xc00c000c
+#define ixCC_GIO_IOC_FUSES 0xc00c0010
+#define ixCC_SMU_TST_EFUSE1_MISC 0xc00c001c
+#define ixCC_TST_ID_STRAPS 0xc00c0020
+#define ixCC_FCTRL_FUSES 0xc00c0024
+#define ixCC_HARVEST_FUSES 0xc00c0028
+#define ixSMU_MAIN_PLL_OP_FREQ 0xe0003020
+#define ixSMU_STATUS 0xe0003088
+#define ixSMU_FIRMWARE 0xe00030a4
+#define ixSMU_INPUT_DATA 0xe00030b8
+#define ixSMU_EFUSE_0 0xc0100000
+#define ixFIRMWARE_FLAGS 0x3f000
+#define ixTDC_STATUS 0x3f004
+#define ixTDC_MV_AVERAGE 0x3f008
+#define ixTDC_VRM_LIMIT 0x3f00c
+#define ixFEATURE_STATUS 0x3f010
+#define ixENTITY_TEMPERATURES_1 0x3f014
+#define ixMCARB_DRAM_TIMING_TABLE_1 0x3f018
+#define ixMCARB_DRAM_TIMING_TABLE_2 0x3f01c
+#define ixMCARB_DRAM_TIMING_TABLE_3 0x3f020
+#define ixMCARB_DRAM_TIMING_TABLE_4 0x3f024
+#define ixMCARB_DRAM_TIMING_TABLE_5 0x3f028
+#define ixMCARB_DRAM_TIMING_TABLE_6 0x3f02c
+#define ixMCARB_DRAM_TIMING_TABLE_7 0x3f030
+#define ixMCARB_DRAM_TIMING_TABLE_8 0x3f034
+#define ixMCARB_DRAM_TIMING_TABLE_9 0x3f038
+#define ixMCARB_DRAM_TIMING_TABLE_10 0x3f03c
+#define ixMCARB_DRAM_TIMING_TABLE_11 0x3f040
+#define ixMCARB_DRAM_TIMING_TABLE_12 0x3f044
+#define ixMCARB_DRAM_TIMING_TABLE_13 0x3f048
+#define ixMCARB_DRAM_TIMING_TABLE_14 0x3f04c
+#define ixMCARB_DRAM_TIMING_TABLE_15 0x3f050
+#define ixMCARB_DRAM_TIMING_TABLE_16 0x3f054
+#define ixMCARB_DRAM_TIMING_TABLE_17 0x3f058
+#define ixMCARB_DRAM_TIMING_TABLE_18 0x3f05c
+#define ixMCARB_DRAM_TIMING_TABLE_19 0x3f060
+#define ixMCARB_DRAM_TIMING_TABLE_20 0x3f064
+#define ixMCARB_DRAM_TIMING_TABLE_21 0x3f068
+#define ixMCARB_DRAM_TIMING_TABLE_22 0x3f06c
+#define ixMCARB_DRAM_TIMING_TABLE_23 0x3f070
+#define ixMCARB_DRAM_TIMING_TABLE_24 0x3f074
+#define ixMCARB_DRAM_TIMING_TABLE_25 0x3f078
+#define ixMCARB_DRAM_TIMING_TABLE_26 0x3f07c
+#define ixMCARB_DRAM_TIMING_TABLE_27 0x3f080
+#define ixMCARB_DRAM_TIMING_TABLE_28 0x3f084
+#define ixMCARB_DRAM_TIMING_TABLE_29 0x3f088
+#define ixMCARB_DRAM_TIMING_TABLE_30 0x3f08c
+#define ixMCARB_DRAM_TIMING_TABLE_31 0x3f090
+#define ixMCARB_DRAM_TIMING_TABLE_32 0x3f094
+#define ixMCARB_DRAM_TIMING_TABLE_33 0x3f098
+#define ixMCARB_DRAM_TIMING_TABLE_34 0x3f09c
+#define ixMCARB_DRAM_TIMING_TABLE_35 0x3f0a0
+#define ixMCARB_DRAM_TIMING_TABLE_36 0x3f0a4
+#define ixMCARB_DRAM_TIMING_TABLE_37 0x3f0a8
+#define ixMCARB_DRAM_TIMING_TABLE_38 0x3f0ac
+#define ixMCARB_DRAM_TIMING_TABLE_39 0x3f0b0
+#define ixMCARB_DRAM_TIMING_TABLE_40 0x3f0b4
+#define ixMCARB_DRAM_TIMING_TABLE_41 0x3f0b8
+#define ixMCARB_DRAM_TIMING_TABLE_42 0x3f0bc
+#define ixMCARB_DRAM_TIMING_TABLE_43 0x3f0c0
+#define ixMCARB_DRAM_TIMING_TABLE_44 0x3f0c4
+#define ixMCARB_DRAM_TIMING_TABLE_45 0x3f0c8
+#define ixMCARB_DRAM_TIMING_TABLE_46 0x3f0cc
+#define ixMCARB_DRAM_TIMING_TABLE_47 0x3f0d0
+#define ixMCARB_DRAM_TIMING_TABLE_48 0x3f0d4
+#define ixMCARB_DRAM_TIMING_TABLE_49 0x3f0d8
+#define ixMCARB_DRAM_TIMING_TABLE_50 0x3f0dc
+#define ixMCARB_DRAM_TIMING_TABLE_51 0x3f0e0
+#define ixMCARB_DRAM_TIMING_TABLE_52 0x3f0e4
+#define ixMCARB_DRAM_TIMING_TABLE_53 0x3f0e8
+#define ixMCARB_DRAM_TIMING_TABLE_54 0x3f0ec
+#define ixMCARB_DRAM_TIMING_TABLE_55 0x3f0f0
+#define ixMCARB_DRAM_TIMING_TABLE_56 0x3f0f4
+#define ixMCARB_DRAM_TIMING_TABLE_57 0x3f0f8
+#define ixMCARB_DRAM_TIMING_TABLE_58 0x3f0fc
+#define ixMCARB_DRAM_TIMING_TABLE_59 0x3f100
+#define ixMCARB_DRAM_TIMING_TABLE_60 0x3f104
+#define ixMCARB_DRAM_TIMING_TABLE_61 0x3f108
+#define ixMCARB_DRAM_TIMING_TABLE_62 0x3f10c
+#define ixMCARB_DRAM_TIMING_TABLE_63 0x3f110
+#define ixMCARB_DRAM_TIMING_TABLE_64 0x3f114
+#define ixMCARB_DRAM_TIMING_TABLE_65 0x3f118
+#define ixMCARB_DRAM_TIMING_TABLE_66 0x3f11c
+#define ixMCARB_DRAM_TIMING_TABLE_67 0x3f120
+#define ixMCARB_DRAM_TIMING_TABLE_68 0x3f124
+#define ixMCARB_DRAM_TIMING_TABLE_69 0x3f128
+#define ixMCARB_DRAM_TIMING_TABLE_70 0x3f12c
+#define ixMCARB_DRAM_TIMING_TABLE_71 0x3f130
+#define ixMCARB_DRAM_TIMING_TABLE_72 0x3f134
+#define ixMCARB_DRAM_TIMING_TABLE_73 0x3f138
+#define ixMCARB_DRAM_TIMING_TABLE_74 0x3f13c
+#define ixMCARB_DRAM_TIMING_TABLE_75 0x3f140
+#define ixMCARB_DRAM_TIMING_TABLE_76 0x3f144
+#define ixMCARB_DRAM_TIMING_TABLE_77 0x3f148
+#define ixMCARB_DRAM_TIMING_TABLE_78 0x3f14c
+#define ixMCARB_DRAM_TIMING_TABLE_79 0x3f150
+#define ixMCARB_DRAM_TIMING_TABLE_80 0x3f154
+#define ixMCARB_DRAM_TIMING_TABLE_81 0x3f158
+#define ixMCARB_DRAM_TIMING_TABLE_82 0x3f15c
+#define ixMCARB_DRAM_TIMING_TABLE_83 0x3f160
+#define ixMCARB_DRAM_TIMING_TABLE_84 0x3f164
+#define ixMCARB_DRAM_TIMING_TABLE_85 0x3f168
+#define ixMCARB_DRAM_TIMING_TABLE_86 0x3f16c
+#define ixMCARB_DRAM_TIMING_TABLE_87 0x3f170
+#define ixMCARB_DRAM_TIMING_TABLE_88 0x3f174
+#define ixMCARB_DRAM_TIMING_TABLE_89 0x3f178
+#define ixMCARB_DRAM_TIMING_TABLE_90 0x3f17c
+#define ixMCARB_DRAM_TIMING_TABLE_91 0x3f180
+#define ixMCARB_DRAM_TIMING_TABLE_92 0x3f184
+#define ixMCARB_DRAM_TIMING_TABLE_93 0x3f188
+#define ixMCARB_DRAM_TIMING_TABLE_94 0x3f18c
+#define ixMCARB_DRAM_TIMING_TABLE_95 0x3f190
+#define ixMCARB_DRAM_TIMING_TABLE_96 0x3f194
+#define ixDPM_TABLE_1 0x3f198
+#define ixDPM_TABLE_2 0x3f19c
+#define ixDPM_TABLE_3 0x3f1a0
+#define ixDPM_TABLE_4 0x3f1a4
+#define ixDPM_TABLE_5 0x3f1a8
+#define ixDPM_TABLE_6 0x3f1ac
+#define ixDPM_TABLE_7 0x3f1b0
+#define ixDPM_TABLE_8 0x3f1b4
+#define ixDPM_TABLE_9 0x3f1b8
+#define ixDPM_TABLE_10 0x3f1bc
+#define ixDPM_TABLE_11 0x3f1c0
+#define ixDPM_TABLE_12 0x3f1c4
+#define ixDPM_TABLE_13 0x3f1c8
+#define ixDPM_TABLE_14 0x3f1cc
+#define ixDPM_TABLE_15 0x3f1d0
+#define ixDPM_TABLE_16 0x3f1d4
+#define ixDPM_TABLE_17 0x3f1d8
+#define ixDPM_TABLE_18 0x3f1dc
+#define ixDPM_TABLE_19 0x3f1e0
+#define ixDPM_TABLE_20 0x3f1e4
+#define ixDPM_TABLE_21 0x3f1e8
+#define ixDPM_TABLE_22 0x3f1ec
+#define ixDPM_TABLE_23 0x3f1f0
+#define ixDPM_TABLE_24 0x3f1f4
+#define ixDPM_TABLE_25 0x3f1f8
+#define ixDPM_TABLE_26 0x3f1fc
+#define ixDPM_TABLE_27 0x3f200
+#define ixDPM_TABLE_28 0x3f204
+#define ixDPM_TABLE_29 0x3f208
+#define ixDPM_TABLE_30 0x3f20c
+#define ixDPM_TABLE_31 0x3f210
+#define ixDPM_TABLE_32 0x3f214
+#define ixDPM_TABLE_33 0x3f218
+#define ixDPM_TABLE_34 0x3f21c
+#define ixDPM_TABLE_35 0x3f220
+#define ixDPM_TABLE_36 0x3f224
+#define ixDPM_TABLE_37 0x3f228
+#define ixDPM_TABLE_38 0x3f22c
+#define ixDPM_TABLE_39 0x3f230
+#define ixDPM_TABLE_40 0x3f234
+#define ixDPM_TABLE_41 0x3f238
+#define ixDPM_TABLE_42 0x3f23c
+#define ixDPM_TABLE_43 0x3f240
+#define ixDPM_TABLE_44 0x3f244
+#define ixDPM_TABLE_45 0x3f248
+#define ixDPM_TABLE_46 0x3f24c
+#define ixDPM_TABLE_47 0x3f250
+#define ixDPM_TABLE_48 0x3f254
+#define ixDPM_TABLE_49 0x3f258
+#define ixDPM_TABLE_50 0x3f25c
+#define ixDPM_TABLE_51 0x3f260
+#define ixDPM_TABLE_52 0x3f264
+#define ixDPM_TABLE_53 0x3f268
+#define ixDPM_TABLE_54 0x3f26c
+#define ixDPM_TABLE_55 0x3f270
+#define ixDPM_TABLE_56 0x3f274
+#define ixDPM_TABLE_57 0x3f278
+#define ixDPM_TABLE_58 0x3f27c
+#define ixDPM_TABLE_59 0x3f280
+#define ixDPM_TABLE_60 0x3f284
+#define ixDPM_TABLE_61 0x3f288
+#define ixDPM_TABLE_62 0x3f28c
+#define ixDPM_TABLE_63 0x3f290
+#define ixDPM_TABLE_64 0x3f294
+#define ixDPM_TABLE_65 0x3f298
+#define ixDPM_TABLE_66 0x3f29c
+#define ixDPM_TABLE_67 0x3f2a0
+#define ixDPM_TABLE_68 0x3f2a4
+#define ixDPM_TABLE_69 0x3f2a8
+#define ixDPM_TABLE_70 0x3f2ac
+#define ixDPM_TABLE_71 0x3f2b0
+#define ixDPM_TABLE_72 0x3f2b4
+#define ixDPM_TABLE_73 0x3f2b8
+#define ixDPM_TABLE_74 0x3f2bc
+#define ixDPM_TABLE_75 0x3f2c0
+#define ixDPM_TABLE_76 0x3f2c4
+#define ixDPM_TABLE_77 0x3f2c8
+#define ixDPM_TABLE_78 0x3f2cc
+#define ixDPM_TABLE_79 0x3f2d0
+#define ixDPM_TABLE_80 0x3f2d4
+#define ixDPM_TABLE_81 0x3f2d8
+#define ixDPM_TABLE_82 0x3f2dc
+#define ixDPM_TABLE_83 0x3f2e0
+#define ixDPM_TABLE_84 0x3f2e4
+#define ixDPM_TABLE_85 0x3f2e8
+#define ixDPM_TABLE_86 0x3f2ec
+#define ixDPM_TABLE_87 0x3f2f0
+#define ixDPM_TABLE_88 0x3f2f4
+#define ixDPM_TABLE_89 0x3f2f8
+#define ixDPM_TABLE_90 0x3f2fc
+#define ixDPM_TABLE_91 0x3f300
+#define ixDPM_TABLE_92 0x3f304
+#define ixDPM_TABLE_93 0x3f308
+#define ixDPM_TABLE_94 0x3f30c
+#define ixDPM_TABLE_95 0x3f310
+#define ixDPM_TABLE_96 0x3f314
+#define ixDPM_TABLE_97 0x3f318
+#define ixDPM_TABLE_98 0x3f31c
+#define ixDPM_TABLE_99 0x3f320
+#define ixDPM_TABLE_100 0x3f324
+#define ixDPM_TABLE_101 0x3f328
+#define ixDPM_TABLE_102 0x3f32c
+#define ixDPM_TABLE_103 0x3f330
+#define ixDPM_TABLE_104 0x3f334
+#define ixDPM_TABLE_105 0x3f338
+#define ixDPM_TABLE_106 0x3f33c
+#define ixDPM_TABLE_107 0x3f340
+#define ixDPM_TABLE_108 0x3f344
+#define ixDPM_TABLE_109 0x3f348
+#define ixDPM_TABLE_110 0x3f34c
+#define ixDPM_TABLE_111 0x3f350
+#define ixDPM_TABLE_112 0x3f354
+#define ixDPM_TABLE_113 0x3f358
+#define ixDPM_TABLE_114 0x3f35c
+#define ixDPM_TABLE_115 0x3f360
+#define ixDPM_TABLE_116 0x3f364
+#define ixDPM_TABLE_117 0x3f368
+#define ixDPM_TABLE_118 0x3f36c
+#define ixDPM_TABLE_119 0x3f370
+#define ixDPM_TABLE_120 0x3f374
+#define ixDPM_TABLE_121 0x3f378
+#define ixDPM_TABLE_122 0x3f37c
+#define ixDPM_TABLE_123 0x3f380
+#define ixDPM_TABLE_124 0x3f384
+#define ixDPM_TABLE_125 0x3f388
+#define ixDPM_TABLE_126 0x3f38c
+#define ixDPM_TABLE_127 0x3f390
+#define ixDPM_TABLE_128 0x3f394
+#define ixDPM_TABLE_129 0x3f398
+#define ixDPM_TABLE_130 0x3f39c
+#define ixDPM_TABLE_131 0x3f3a0
+#define ixDPM_TABLE_132 0x3f3a4
+#define ixDPM_TABLE_133 0x3f3a8
+#define ixDPM_TABLE_134 0x3f3ac
+#define ixDPM_TABLE_135 0x3f3b0
+#define ixDPM_TABLE_136 0x3f3b4
+#define ixDPM_TABLE_137 0x3f3b8
+#define ixDPM_TABLE_138 0x3f3bc
+#define ixDPM_TABLE_139 0x3f3c0
+#define ixDPM_TABLE_140 0x3f3c4
+#define ixDPM_TABLE_141 0x3f3c8
+#define ixDPM_TABLE_142 0x3f3cc
+#define ixDPM_TABLE_143 0x3f3d0
+#define ixDPM_TABLE_144 0x3f3d4
+#define ixDPM_TABLE_145 0x3f3d8
+#define ixDPM_TABLE_146 0x3f3dc
+#define ixDPM_TABLE_147 0x3f3e0
+#define ixDPM_TABLE_148 0x3f3e4
+#define ixDPM_TABLE_149 0x3f3e8
+#define ixDPM_TABLE_150 0x3f3ec
+#define ixDPM_TABLE_151 0x3f3f0
+#define ixDPM_TABLE_152 0x3f3f4
+#define ixDPM_TABLE_153 0x3f3f8
+#define ixDPM_TABLE_154 0x3f3fc
+#define ixDPM_TABLE_155 0x3f400
+#define ixDPM_TABLE_156 0x3f404
+#define ixDPM_TABLE_157 0x3f408
+#define ixDPM_TABLE_158 0x3f40c
+#define ixDPM_TABLE_159 0x3f410
+#define ixDPM_TABLE_160 0x3f414
+#define ixDPM_TABLE_161 0x3f418
+#define ixDPM_TABLE_162 0x3f41c
+#define ixDPM_TABLE_163 0x3f420
+#define ixDPM_TABLE_164 0x3f424
+#define ixDPM_TABLE_165 0x3f428
+#define ixDPM_TABLE_166 0x3f42c
+#define ixDPM_TABLE_167 0x3f430
+#define ixDPM_TABLE_168 0x3f434
+#define ixDPM_TABLE_169 0x3f438
+#define ixDPM_TABLE_170 0x3f43c
+#define ixDPM_TABLE_171 0x3f440
+#define ixDPM_TABLE_172 0x3f444
+#define ixDPM_TABLE_173 0x3f448
+#define ixDPM_TABLE_174 0x3f44c
+#define ixDPM_TABLE_175 0x3f450
+#define ixDPM_TABLE_176 0x3f454
+#define ixDPM_TABLE_177 0x3f458
+#define ixDPM_TABLE_178 0x3f45c
+#define ixDPM_TABLE_179 0x3f460
+#define ixDPM_TABLE_180 0x3f464
+#define ixDPM_TABLE_181 0x3f468
+#define ixDPM_TABLE_182 0x3f46c
+#define ixDPM_TABLE_183 0x3f470
+#define ixDPM_TABLE_184 0x3f474
+#define ixDPM_TABLE_185 0x3f478
+#define ixDPM_TABLE_186 0x3f47c
+#define ixDPM_TABLE_187 0x3f480
+#define ixDPM_TABLE_188 0x3f484
+#define ixDPM_TABLE_189 0x3f488
+#define ixDPM_TABLE_190 0x3f48c
+#define ixDPM_TABLE_191 0x3f490
+#define ixDPM_TABLE_192 0x3f494
+#define ixDPM_TABLE_193 0x3f498
+#define ixDPM_TABLE_194 0x3f49c
+#define ixDPM_TABLE_195 0x3f4a0
+#define ixDPM_TABLE_196 0x3f4a4
+#define ixDPM_TABLE_197 0x3f4a8
+#define ixDPM_TABLE_198 0x3f4ac
+#define ixDPM_TABLE_199 0x3f4b0
+#define ixDPM_TABLE_200 0x3f4b4
+#define ixDPM_TABLE_201 0x3f4b8
+#define ixDPM_TABLE_202 0x3f4bc
+#define ixDPM_TABLE_203 0x3f4c0
+#define ixDPM_TABLE_204 0x3f4c4
+#define ixDPM_TABLE_205 0x3f4c8
+#define ixDPM_TABLE_206 0x3f4cc
+#define ixDPM_TABLE_207 0x3f4d0
+#define ixDPM_TABLE_208 0x3f4d4
+#define ixDPM_TABLE_209 0x3f4d8
+#define ixDPM_TABLE_210 0x3f4dc
+#define ixDPM_TABLE_211 0x3f4e0
+#define ixDPM_TABLE_212 0x3f4e4
+#define ixDPM_TABLE_213 0x3f4e8
+#define ixDPM_TABLE_214 0x3f4ec
+#define ixDPM_TABLE_215 0x3f4f0
+#define ixDPM_TABLE_216 0x3f4f4
+#define ixDPM_TABLE_217 0x3f4f8
+#define ixDPM_TABLE_218 0x3f4fc
+#define ixDPM_TABLE_219 0x3f500
+#define ixDPM_TABLE_220 0x3f504
+#define ixDPM_TABLE_221 0x3f508
+#define ixDPM_TABLE_222 0x3f50c
+#define ixDPM_TABLE_223 0x3f510
+#define ixDPM_TABLE_224 0x3f514
+#define ixDPM_TABLE_225 0x3f518
+#define ixDPM_TABLE_226 0x3f51c
+#define ixDPM_TABLE_227 0x3f520
+#define ixDPM_TABLE_228 0x3f524
+#define ixDPM_TABLE_229 0x3f528
+#define ixDPM_TABLE_230 0x3f52c
+#define ixDPM_TABLE_231 0x3f530
+#define ixDPM_TABLE_232 0x3f534
+#define ixDPM_TABLE_233 0x3f538
+#define ixDPM_TABLE_234 0x3f53c
+#define ixDPM_TABLE_235 0x3f540
+#define ixDPM_TABLE_236 0x3f544
+#define ixDPM_TABLE_237 0x3f548
+#define ixDPM_TABLE_238 0x3f54c
+#define ixDPM_TABLE_239 0x3f550
+#define ixDPM_TABLE_240 0x3f554
+#define ixDPM_TABLE_241 0x3f558
+#define ixDPM_TABLE_242 0x3f55c
+#define ixDPM_TABLE_243 0x3f560
+#define ixDPM_TABLE_244 0x3f564
+#define ixDPM_TABLE_245 0x3f568
+#define ixDPM_TABLE_246 0x3f56c
+#define ixDPM_TABLE_247 0x3f570
+#define ixDPM_TABLE_248 0x3f574
+#define ixDPM_TABLE_249 0x3f578
+#define ixDPM_TABLE_250 0x3f57c
+#define ixDPM_TABLE_251 0x3f580
+#define ixDPM_TABLE_252 0x3f584
+#define ixDPM_TABLE_253 0x3f588
+#define ixDPM_TABLE_254 0x3f58c
+#define ixDPM_TABLE_255 0x3f590
+#define ixDPM_TABLE_256 0x3f594
+#define ixDPM_TABLE_257 0x3f598
+#define ixDPM_TABLE_258 0x3f59c
+#define ixDPM_TABLE_259 0x3f5a0
+#define ixDPM_TABLE_260 0x3f5a4
+#define ixDPM_TABLE_261 0x3f5a8
+#define ixDPM_TABLE_262 0x3f5ac
+#define ixDPM_TABLE_263 0x3f5b0
+#define ixDPM_TABLE_264 0x3f5b4
+#define ixDPM_TABLE_265 0x3f5b8
+#define ixDPM_TABLE_266 0x3f5bc
+#define ixDPM_TABLE_267 0x3f5c0
+#define ixDPM_TABLE_268 0x3f5c4
+#define ixDPM_TABLE_269 0x3f5c8
+#define ixDPM_TABLE_270 0x3f5cc
+#define ixDPM_TABLE_271 0x3f5d0
+#define ixDPM_TABLE_272 0x3f5d4
+#define ixDPM_TABLE_273 0x3f5d8
+#define ixDPM_TABLE_274 0x3f5dc
+#define ixDPM_TABLE_275 0x3f5e0
+#define ixDPM_TABLE_276 0x3f5e4
+#define ixDPM_TABLE_277 0x3f5e8
+#define ixDPM_TABLE_278 0x3f5ec
+#define ixDPM_TABLE_279 0x3f5f0
+#define ixDPM_TABLE_280 0x3f5f4
+#define ixDPM_TABLE_281 0x3f5f8
+#define ixDPM_TABLE_282 0x3f5fc
+#define ixDPM_TABLE_283 0x3f600
+#define ixDPM_TABLE_284 0x3f604
+#define ixDPM_TABLE_285 0x3f608
+#define ixDPM_TABLE_286 0x3f60c
+#define ixDPM_TABLE_287 0x3f610
+#define ixDPM_TABLE_288 0x3f614
+#define ixDPM_TABLE_289 0x3f618
+#define ixDPM_TABLE_290 0x3f61c
+#define ixDPM_TABLE_291 0x3f620
+#define ixDPM_TABLE_292 0x3f624
+#define ixDPM_TABLE_293 0x3f628
+#define ixDPM_TABLE_294 0x3f62c
+#define ixDPM_TABLE_295 0x3f630
+#define ixDPM_TABLE_296 0x3f634
+#define ixDPM_TABLE_297 0x3f638
+#define ixDPM_TABLE_298 0x3f63c
+#define ixDPM_TABLE_299 0x3f640
+#define ixDPM_TABLE_300 0x3f644
+#define ixDPM_TABLE_301 0x3f648
+#define ixDPM_TABLE_302 0x3f64c
+#define ixDPM_TABLE_303 0x3f650
+#define ixDPM_TABLE_304 0x3f654
+#define ixDPM_TABLE_305 0x3f658
+#define ixDPM_TABLE_306 0x3f65c
+#define ixDPM_TABLE_307 0x3f660
+#define ixDPM_TABLE_308 0x3f664
+#define ixDPM_TABLE_309 0x3f668
+#define ixDPM_TABLE_310 0x3f66c
+#define ixDPM_TABLE_311 0x3f670
+#define ixDPM_TABLE_312 0x3f674
+#define ixDPM_TABLE_313 0x3f678
+#define ixDPM_TABLE_314 0x3f67c
+#define ixDPM_TABLE_315 0x3f680
+#define ixDPM_TABLE_316 0x3f684
+#define ixDPM_TABLE_317 0x3f688
+#define ixDPM_TABLE_318 0x3f68c
+#define ixDPM_TABLE_319 0x3f690
+#define ixDPM_TABLE_320 0x3f694
+#define ixDPM_TABLE_321 0x3f698
+#define ixDPM_TABLE_322 0x3f69c
+#define ixDPM_TABLE_323 0x3f6a0
+#define ixDPM_TABLE_324 0x3f6a4
+#define ixDPM_TABLE_325 0x3f6a8
+#define ixDPM_TABLE_326 0x3f6ac
+#define ixDPM_TABLE_327 0x3f6b0
+#define ixDPM_TABLE_328 0x3f6b4
+#define ixDPM_TABLE_329 0x3f6b8
+#define ixDPM_TABLE_330 0x3f6bc
+#define ixDPM_TABLE_331 0x3f6c0
+#define ixDPM_TABLE_332 0x3f6c4
+#define ixDPM_TABLE_333 0x3f6c8
+#define ixDPM_TABLE_334 0x3f6cc
+#define ixDPM_TABLE_335 0x3f6d0
+#define ixDPM_TABLE_336 0x3f6d4
+#define ixDPM_TABLE_337 0x3f6d8
+#define ixDPM_TABLE_338 0x3f6dc
+#define ixDPM_TABLE_339 0x3f6e0
+#define ixDPM_TABLE_340 0x3f6e4
+#define ixDPM_TABLE_341 0x3f6e8
+#define ixDPM_TABLE_342 0x3f6ec
+#define ixDPM_TABLE_343 0x3f6f0
+#define ixDPM_TABLE_344 0x3f6f4
+#define ixDPM_TABLE_345 0x3f6f8
+#define ixDPM_TABLE_346 0x3f6fc
+#define ixDPM_TABLE_347 0x3f700
+#define ixDPM_TABLE_348 0x3f704
+#define ixDPM_TABLE_349 0x3f708
+#define ixDPM_TABLE_350 0x3f70c
+#define ixDPM_TABLE_351 0x3f710
+#define ixDPM_TABLE_352 0x3f714
+#define ixDPM_TABLE_353 0x3f718
+#define ixDPM_TABLE_354 0x3f71c
+#define ixDPM_TABLE_355 0x3f720
+#define ixDPM_TABLE_356 0x3f724
+#define ixDPM_TABLE_357 0x3f728
+#define ixDPM_TABLE_358 0x3f72c
+#define ixDPM_TABLE_359 0x3f730
+#define ixDPM_TABLE_360 0x3f734
+#define ixDPM_TABLE_361 0x3f738
+#define ixDPM_TABLE_362 0x3f73c
+#define ixDPM_TABLE_363 0x3f740
+#define ixDPM_TABLE_364 0x3f744
+#define ixDPM_TABLE_365 0x3f748
+#define ixDPM_TABLE_366 0x3f74c
+#define ixDPM_TABLE_367 0x3f750
+#define ixDPM_TABLE_368 0x3f754
+#define ixDPM_TABLE_369 0x3f758
+#define ixDPM_TABLE_370 0x3f75c
+#define ixDPM_TABLE_371 0x3f760
+#define ixDPM_TABLE_372 0x3f764
+#define ixDPM_TABLE_373 0x3f768
+#define ixDPM_TABLE_374 0x3f76c
+#define ixDPM_TABLE_375 0x3f770
+#define ixDPM_TABLE_376 0x3f774
+#define ixDPM_TABLE_377 0x3f778
+#define ixDPM_TABLE_378 0x3f77c
+#define ixDPM_TABLE_379 0x3f780
+#define ixDPM_TABLE_380 0x3f784
+#define ixDPM_TABLE_381 0x3f788
+#define ixDPM_TABLE_382 0x3f78c
+#define ixDPM_TABLE_383 0x3f790
+#define ixDPM_TABLE_384 0x3f794
+#define ixDPM_TABLE_385 0x3f798
+#define ixDPM_TABLE_386 0x3f79c
+#define ixDPM_TABLE_387 0x3f7a0
+#define ixDPM_TABLE_388 0x3f7a4
+#define ixDPM_TABLE_389 0x3f7a8
+#define ixDPM_TABLE_390 0x3f7ac
+#define ixDPM_TABLE_391 0x3f7b0
+#define ixDPM_TABLE_392 0x3f7b4
+#define ixDPM_TABLE_393 0x3f7b8
+#define ixDPM_TABLE_394 0x3f7bc
+#define ixDPM_TABLE_395 0x3f7c0
+#define ixDPM_TABLE_396 0x3f7c4
+#define ixDPM_TABLE_397 0x3f7c8
+#define ixDPM_TABLE_398 0x3f7cc
+#define ixDPM_TABLE_399 0x3f7d0
+#define ixDPM_TABLE_400 0x3f7d4
+#define ixDPM_TABLE_401 0x3f7d8
+#define ixDPM_TABLE_402 0x3f7dc
+#define ixDPM_TABLE_403 0x3f7e0
+#define ixDPM_TABLE_404 0x3f7e4
+#define ixDPM_TABLE_405 0x3f7e8
+#define ixDPM_TABLE_406 0x3f7ec
+#define ixDPM_TABLE_407 0x3f7f0
+#define ixDPM_TABLE_408 0x3f7f4
+#define ixDPM_TABLE_409 0x3f7f8
+#define ixDPM_TABLE_410 0x3f7fc
+#define ixDPM_TABLE_411 0x3f800
+#define ixDPM_TABLE_412 0x3f804
+#define ixDPM_TABLE_413 0x3f808
+#define ixDPM_TABLE_414 0x3f80c
+#define ixDPM_TABLE_415 0x3f810
+#define ixDPM_TABLE_416 0x3f814
+#define ixDPM_TABLE_417 0x3f818
+#define ixDPM_TABLE_418 0x3f81c
+#define ixDPM_TABLE_419 0x3f820
+#define ixDPM_TABLE_420 0x3f824
+#define ixDPM_TABLE_421 0x3f828
+#define ixDPM_TABLE_422 0x3f82c
+#define ixDPM_TABLE_423 0x3f830
+#define ixDPM_TABLE_424 0x3f834
+#define ixDPM_TABLE_425 0x3f838
+#define ixDPM_TABLE_426 0x3f83c
+#define ixDPM_TABLE_427 0x3f840
+#define ixDPM_TABLE_428 0x3f844
+#define ixDPM_TABLE_429 0x3f848
+#define ixDPM_TABLE_430 0x3f84c
+#define ixDPM_TABLE_431 0x3f850
+#define ixDPM_TABLE_432 0x3f854
+#define ixDPM_TABLE_433 0x3f858
+#define ixDPM_TABLE_434 0x3f85c
+#define ixDPM_TABLE_435 0x3f860
+#define ixDPM_TABLE_436 0x3f864
+#define ixDPM_TABLE_437 0x3f868
+#define ixDPM_TABLE_438 0x3f86c
+#define ixDPM_TABLE_439 0x3f870
+#define ixDPM_TABLE_440 0x3f874
+#define ixSOFT_REGISTERS_TABLE_1 0x3f89c
+#define ixSOFT_REGISTERS_TABLE_2 0x3f8a0
+#define ixSOFT_REGISTERS_TABLE_3 0x3f8a4
+#define ixSOFT_REGISTERS_TABLE_4 0x3f8a8
+#define ixSOFT_REGISTERS_TABLE_5 0x3f8ac
+#define ixSOFT_REGISTERS_TABLE_6 0x3f8b0
+#define ixSOFT_REGISTERS_TABLE_7 0x3f8b4
+#define ixSOFT_REGISTERS_TABLE_8 0x3f8b8
+#define ixSOFT_REGISTERS_TABLE_9 0x3f8bc
+#define ixSOFT_REGISTERS_TABLE_10 0x3f8c0
+#define ixSOFT_REGISTERS_TABLE_11 0x3f8c4
+#define ixSOFT_REGISTERS_TABLE_12 0x3f8c8
+#define ixSOFT_REGISTERS_TABLE_13 0x3f8cc
+#define ixSOFT_REGISTERS_TABLE_14 0x3f8d0
+#define ixSOFT_REGISTERS_TABLE_15 0x3f8d4
+#define ixSOFT_REGISTERS_TABLE_16 0x3f8d8
+#define ixSOFT_REGISTERS_TABLE_17 0x3f8dc
+#define ixSOFT_REGISTERS_TABLE_18 0x3f8e0
+#define ixSOFT_REGISTERS_TABLE_19 0x3f8e4
+#define ixSOFT_REGISTERS_TABLE_20 0x3f8e8
+#define ixSOFT_REGISTERS_TABLE_21 0x3f8ec
+#define ixSOFT_REGISTERS_TABLE_22 0x3f8f0
+#define ixSOFT_REGISTERS_TABLE_23 0x3f8f4
+#define ixSOFT_REGISTERS_TABLE_24 0x3f8f8
+#define ixSOFT_REGISTERS_TABLE_25 0x3f8fc
+#define ixSOFT_REGISTERS_TABLE_26 0x3f900
+#define ixSOFT_REGISTERS_TABLE_27 0x3f904
+#define ixSOFT_REGISTERS_TABLE_28 0x3f888
+#define ixSOFT_REGISTERS_TABLE_29 0x3f90c
+#define ixSOFT_REGISTERS_TABLE_30 0x3f910
+#define ixPM_FUSES_1 0x3f914
+#define ixPM_FUSES_2 0x3f918
+#define ixPM_FUSES_3 0x3f91c
+#define ixPM_FUSES_4 0x3f920
+#define ixPM_FUSES_5 0x3f924
+#define ixPM_FUSES_6 0x3f928
+#define ixPM_FUSES_7 0x3f92c
+#define ixPM_FUSES_8 0x3f930
+#define ixPM_FUSES_9 0x3f934
+#define ixPM_FUSES_10 0x3f938
+#define ixPM_FUSES_11 0x3f93c
+#define ixPM_FUSES_12 0x3f940
+#define ixPM_FUSES_13 0x3f944
+#define ixPM_FUSES_14 0x3f948
+#define ixPM_FUSES_15 0x3f94c
+#define ixSMU_PM_STATUS_0 0x3fe00
+#define ixSMU_PM_STATUS_1 0x3fe04
+#define ixSMU_PM_STATUS_2 0x3fe08
+#define ixSMU_PM_STATUS_3 0x3fe0c
+#define ixSMU_PM_STATUS_4 0x3fe10
+#define ixSMU_PM_STATUS_5 0x3fe14
+#define ixSMU_PM_STATUS_6 0x3fe18
+#define ixSMU_PM_STATUS_7 0x3fe1c
+#define ixSMU_PM_STATUS_8 0x3fe20
+#define ixSMU_PM_STATUS_9 0x3fe24
+#define ixSMU_PM_STATUS_10 0x3fe28
+#define ixSMU_PM_STATUS_11 0x3fe2c
+#define ixSMU_PM_STATUS_12 0x3fe30
+#define ixSMU_PM_STATUS_13 0x3fe34
+#define ixSMU_PM_STATUS_14 0x3fe38
+#define ixSMU_PM_STATUS_15 0x3fe3c
+#define ixSMU_PM_STATUS_16 0x3fe40
+#define ixSMU_PM_STATUS_17 0x3fe44
+#define ixSMU_PM_STATUS_18 0x3fe48
+#define ixSMU_PM_STATUS_19 0x3fe4c
+#define ixSMU_PM_STATUS_20 0x3fe50
+#define ixSMU_PM_STATUS_21 0x3fe54
+#define ixSMU_PM_STATUS_22 0x3fe58
+#define ixSMU_PM_STATUS_23 0x3fe5c
+#define ixSMU_PM_STATUS_24 0x3fe60
+#define ixSMU_PM_STATUS_25 0x3fe64
+#define ixSMU_PM_STATUS_26 0x3fe68
+#define ixSMU_PM_STATUS_27 0x3fe6c
+#define ixSMU_PM_STATUS_28 0x3fe70
+#define ixSMU_PM_STATUS_29 0x3fe74
+#define ixSMU_PM_STATUS_30 0x3fe78
+#define ixSMU_PM_STATUS_31 0x3fe7c
+#define ixSMU_PM_STATUS_32 0x3fe80
+#define ixSMU_PM_STATUS_33 0x3fe84
+#define ixSMU_PM_STATUS_34 0x3fe88
+#define ixSMU_PM_STATUS_35 0x3fe8c
+#define ixSMU_PM_STATUS_36 0x3fe90
+#define ixSMU_PM_STATUS_37 0x3fe94
+#define ixSMU_PM_STATUS_38 0x3fe98
+#define ixSMU_PM_STATUS_39 0x3fe9c
+#define ixSMU_PM_STATUS_40 0x3fea0
+#define ixSMU_PM_STATUS_41 0x3fea4
+#define ixSMU_PM_STATUS_42 0x3fea8
+#define ixSMU_PM_STATUS_43 0x3feac
+#define ixSMU_PM_STATUS_44 0x3feb0
+#define ixSMU_PM_STATUS_45 0x3feb4
+#define ixSMU_PM_STATUS_46 0x3feb8
+#define ixSMU_PM_STATUS_47 0x3febc
+#define ixSMU_PM_STATUS_48 0x3fec0
+#define ixSMU_PM_STATUS_49 0x3fec4
+#define ixSMU_PM_STATUS_50 0x3fec8
+#define ixSMU_PM_STATUS_51 0x3fecc
+#define ixSMU_PM_STATUS_52 0x3fed0
+#define ixSMU_PM_STATUS_53 0x3fed4
+#define ixSMU_PM_STATUS_54 0x3fed8
+#define ixSMU_PM_STATUS_55 0x3fedc
+#define ixSMU_PM_STATUS_56 0x3fee0
+#define ixSMU_PM_STATUS_57 0x3fee4
+#define ixSMU_PM_STATUS_58 0x3fee8
+#define ixSMU_PM_STATUS_59 0x3feec
+#define ixSMU_PM_STATUS_60 0x3fef0
+#define ixSMU_PM_STATUS_61 0x3fef4
+#define ixSMU_PM_STATUS_62 0x3fef8
+#define ixSMU_PM_STATUS_63 0x3fefc
+#define ixSMU_PM_STATUS_64 0x3ff00
+#define ixSMU_PM_STATUS_65 0x3ff04
+#define ixSMU_PM_STATUS_66 0x3ff08
+#define ixSMU_PM_STATUS_67 0x3ff0c
+#define ixSMU_PM_STATUS_68 0x3ff10
+#define ixSMU_PM_STATUS_69 0x3ff14
+#define ixSMU_PM_STATUS_70 0x3ff18
+#define ixSMU_PM_STATUS_71 0x3ff1c
+#define ixSMU_PM_STATUS_72 0x3ff20
+#define ixSMU_PM_STATUS_73 0x3ff24
+#define ixSMU_PM_STATUS_74 0x3ff28
+#define ixSMU_PM_STATUS_75 0x3ff2c
+#define ixSMU_PM_STATUS_76 0x3ff30
+#define ixSMU_PM_STATUS_77 0x3ff34
+#define ixSMU_PM_STATUS_78 0x3ff38
+#define ixSMU_PM_STATUS_79 0x3ff3c
+#define ixSMU_PM_STATUS_80 0x3ff40
+#define ixSMU_PM_STATUS_81 0x3ff44
+#define ixSMU_PM_STATUS_82 0x3ff48
+#define ixSMU_PM_STATUS_83 0x3ff4c
+#define ixSMU_PM_STATUS_84 0x3ff50
+#define ixSMU_PM_STATUS_85 0x3ff54
+#define ixSMU_PM_STATUS_86 0x3ff58
+#define ixSMU_PM_STATUS_87 0x3ff5c
+#define ixSMU_PM_STATUS_88 0x3ff60
+#define ixSMU_PM_STATUS_89 0x3ff64
+#define ixSMU_PM_STATUS_90 0x3ff68
+#define ixSMU_PM_STATUS_91 0x3ff6c
+#define ixSMU_PM_STATUS_92 0x3ff70
+#define ixSMU_PM_STATUS_93 0x3ff74
+#define ixSMU_PM_STATUS_94 0x3ff78
+#define ixSMU_PM_STATUS_95 0x3ff7c
+#define ixSMU_PM_STATUS_96 0x3ff80
+#define ixSMU_PM_STATUS_97 0x3ff84
+#define ixSMU_PM_STATUS_98 0x3ff88
+#define ixSMU_PM_STATUS_99 0x3ff8c
+#define ixSMU_PM_STATUS_100 0x3ff90
+#define ixSMU_PM_STATUS_101 0x3ff94
+#define ixSMU_PM_STATUS_102 0x3ff98
+#define ixSMU_PM_STATUS_103 0x3ff9c
+#define ixSMU_PM_STATUS_104 0x3ffa0
+#define ixSMU_PM_STATUS_105 0x3ffa4
+#define ixSMU_PM_STATUS_106 0x3ffa8
+#define ixSMU_PM_STATUS_107 0x3ffac
+#define ixSMU_PM_STATUS_108 0x3ffb0
+#define ixSMU_PM_STATUS_109 0x3ffb4
+#define ixSMU_PM_STATUS_110 0x3ffb8
+#define ixSMU_PM_STATUS_111 0x3ffbc
+#define ixSMU_PM_STATUS_112 0x3ffc0
+#define ixSMU_PM_STATUS_113 0x3ffc4
+#define ixSMU_PM_STATUS_114 0x3ffc8
+#define ixSMU_PM_STATUS_115 0x3ffcc
+#define ixSMU_PM_STATUS_116 0x3ffd0
+#define ixSMU_PM_STATUS_117 0x3ffd4
+#define ixSMU_PM_STATUS_118 0x3ffd8
+#define ixSMU_PM_STATUS_119 0x3ffdc
+#define ixSMU_PM_STATUS_120 0x3ffe0
+#define ixSMU_PM_STATUS_121 0x3ffe4
+#define ixSMU_PM_STATUS_122 0x3ffe8
+#define ixSMU_PM_STATUS_123 0x3ffec
+#define ixSMU_PM_STATUS_124 0x3fff0
+#define ixSMU_PM_STATUS_125 0x3fff4
+#define ixSMU_PM_STATUS_126 0x3fff8
+#define ixSMU_PM_STATUS_127 0x3fffc
+#define ixCG_THERMAL_INT_ENA 0xc2100024
+#define ixCG_THERMAL_INT_CTRL 0xc2100028
+#define ixCG_THERMAL_INT_STATUS 0xc210002c
+#define ixCG_THERMAL_CTRL 0xc0300004
+#define ixCG_THERMAL_STATUS 0xc0300008
+#define ixCG_THERMAL_INT 0xc030000c
+#define ixCG_MULT_THERMAL_CTRL 0xc0300010
+#define ixCG_MULT_THERMAL_STATUS 0xc0300014
+#define ixTHM_TMON2_CTRL 0xc0300034
+#define ixTHM_TMON2_CTRL2 0xc0300038
+#define ixTHM_TMON2_CSR_WR 0xc0300054
+#define ixTHM_TMON2_CSR_RD 0xc0300058
+#define ixCG_FDO_CTRL0 0xc0300064
+#define ixCG_FDO_CTRL1 0xc0300068
+#define ixCG_FDO_CTRL2 0xc030006c
+#define ixCG_TACH_CTRL 0xc0300070
+#define ixCG_TACH_STATUS 0xc0300074
+#define ixCC_THM_STRAPS0 0xc0300080
+#define ixTHM_TMON0_RDIL0_DATA 0xc0300100
+#define ixTHM_TMON0_RDIL1_DATA 0xc0300104
+#define ixTHM_TMON0_RDIL2_DATA 0xc0300108
+#define ixTHM_TMON0_RDIL3_DATA 0xc030010c
+#define ixTHM_TMON0_RDIL4_DATA 0xc0300110
+#define ixTHM_TMON0_RDIL5_DATA 0xc0300114
+#define ixTHM_TMON0_RDIL6_DATA 0xc0300118
+#define ixTHM_TMON0_RDIL7_DATA 0xc030011c
+#define ixTHM_TMON0_RDIL8_DATA 0xc0300120
+#define ixTHM_TMON0_RDIL9_DATA 0xc0300124
+#define ixTHM_TMON0_RDIL10_DATA 0xc0300128
+#define ixTHM_TMON0_RDIL11_DATA 0xc030012c
+#define ixTHM_TMON0_RDIL12_DATA 0xc0300130
+#define ixTHM_TMON0_RDIL13_DATA 0xc0300134
+#define ixTHM_TMON0_RDIL14_DATA 0xc0300138
+#define ixTHM_TMON0_RDIL15_DATA 0xc030013c
+#define ixTHM_TMON0_RDIR0_DATA 0xc0300140
+#define ixTHM_TMON0_RDIR1_DATA 0xc0300144
+#define ixTHM_TMON0_RDIR2_DATA 0xc0300148
+#define ixTHM_TMON0_RDIR3_DATA 0xc030014c
+#define ixTHM_TMON0_RDIR4_DATA 0xc0300150
+#define ixTHM_TMON0_RDIR5_DATA 0xc0300154
+#define ixTHM_TMON0_RDIR6_DATA 0xc0300158
+#define ixTHM_TMON0_RDIR7_DATA 0xc030015c
+#define ixTHM_TMON0_RDIR8_DATA 0xc0300160
+#define ixTHM_TMON0_RDIR9_DATA 0xc0300164
+#define ixTHM_TMON0_RDIR10_DATA 0xc0300168
+#define ixTHM_TMON0_RDIR11_DATA 0xc030016c
+#define ixTHM_TMON0_RDIR12_DATA 0xc0300170
+#define ixTHM_TMON0_RDIR13_DATA 0xc0300174
+#define ixTHM_TMON0_RDIR14_DATA 0xc0300178
+#define ixTHM_TMON0_RDIR15_DATA 0xc030017c
+#define ixTHM_TMON1_RDIL0_DATA 0xc0300180
+#define ixTHM_TMON1_RDIL1_DATA 0xc0300184
+#define ixTHM_TMON1_RDIL2_DATA 0xc0300188
+#define ixTHM_TMON1_RDIL3_DATA 0xc030018c
+#define ixTHM_TMON1_RDIL4_DATA 0xc0300190
+#define ixTHM_TMON1_RDIL5_DATA 0xc0300194
+#define ixTHM_TMON1_RDIL6_DATA 0xc0300198
+#define ixTHM_TMON1_RDIL7_DATA 0xc030019c
+#define ixTHM_TMON1_RDIL8_DATA 0xc03001a0
+#define ixTHM_TMON1_RDIL9_DATA 0xc03001a4
+#define ixTHM_TMON1_RDIL10_DATA 0xc03001a8
+#define ixTHM_TMON1_RDIL11_DATA 0xc03001ac
+#define ixTHM_TMON1_RDIL12_DATA 0xc03001b0
+#define ixTHM_TMON1_RDIL13_DATA 0xc03001b4
+#define ixTHM_TMON1_RDIL14_DATA 0xc03001b8
+#define ixTHM_TMON1_RDIL15_DATA 0xc03001bc
+#define ixTHM_TMON1_RDIR0_DATA 0xc03001c0
+#define ixTHM_TMON1_RDIR1_DATA 0xc03001c4
+#define ixTHM_TMON1_RDIR2_DATA 0xc03001c8
+#define ixTHM_TMON1_RDIR3_DATA 0xc03001cc
+#define ixTHM_TMON1_RDIR4_DATA 0xc03001d0
+#define ixTHM_TMON1_RDIR5_DATA 0xc03001d4
+#define ixTHM_TMON1_RDIR6_DATA 0xc03001d8
+#define ixTHM_TMON1_RDIR7_DATA 0xc03001dc
+#define ixTHM_TMON1_RDIR8_DATA 0xc03001e0
+#define ixTHM_TMON1_RDIR9_DATA 0xc03001e4
+#define ixTHM_TMON1_RDIR10_DATA 0xc03001e8
+#define ixTHM_TMON1_RDIR11_DATA 0xc03001ec
+#define ixTHM_TMON1_RDIR12_DATA 0xc03001f0
+#define ixTHM_TMON1_RDIR13_DATA 0xc03001f4
+#define ixTHM_TMON1_RDIR14_DATA 0xc03001f8
+#define ixTHM_TMON1_RDIR15_DATA 0xc03001fc
+#define ixTHM_TMON2_RDIL0_DATA 0xc0300200
+#define ixTHM_TMON2_RDIL1_DATA 0xc0300204
+#define ixTHM_TMON2_RDIL2_DATA 0xc0300208
+#define ixTHM_TMON2_RDIL3_DATA 0xc030020c
+#define ixTHM_TMON2_RDIL4_DATA 0xc0300210
+#define ixTHM_TMON2_RDIL5_DATA 0xc0300214
+#define ixTHM_TMON2_RDIL6_DATA 0xc0300218
+#define ixTHM_TMON2_RDIL7_DATA 0xc030021c
+#define ixTHM_TMON2_RDIL8_DATA 0xc0300220
+#define ixTHM_TMON2_RDIL9_DATA 0xc0300224
+#define ixTHM_TMON2_RDIL10_DATA 0xc0300228
+#define ixTHM_TMON2_RDIL11_DATA 0xc030022c
+#define ixTHM_TMON2_RDIL12_DATA 0xc0300230
+#define ixTHM_TMON2_RDIL13_DATA 0xc0300234
+#define ixTHM_TMON2_RDIL14_DATA 0xc0300238
+#define ixTHM_TMON2_RDIL15_DATA 0xc030023c
+#define ixTHM_TMON2_RDIR0_DATA 0xc0300240
+#define ixTHM_TMON2_RDIR1_DATA 0xc0300244
+#define ixTHM_TMON2_RDIR2_DATA 0xc0300248
+#define ixTHM_TMON2_RDIR3_DATA 0xc030024c
+#define ixTHM_TMON2_RDIR4_DATA 0xc0300250
+#define ixTHM_TMON2_RDIR5_DATA 0xc0300254
+#define ixTHM_TMON2_RDIR6_DATA 0xc0300258
+#define ixTHM_TMON2_RDIR7_DATA 0xc030025c
+#define ixTHM_TMON2_RDIR8_DATA 0xc0300260
+#define ixTHM_TMON2_RDIR9_DATA 0xc0300264
+#define ixTHM_TMON2_RDIR10_DATA 0xc0300268
+#define ixTHM_TMON2_RDIR11_DATA 0xc030026c
+#define ixTHM_TMON2_RDIR12_DATA 0xc0300270
+#define ixTHM_TMON2_RDIR13_DATA 0xc0300274
+#define ixTHM_TMON2_RDIR14_DATA 0xc0300278
+#define ixTHM_TMON2_RDIR15_DATA 0xc030027c
+#define ixTHM_TMON0_INT_DATA 0xc0300300
+#define ixTHM_TMON1_INT_DATA 0xc0300304
+#define ixTHM_TMON2_INT_DATA 0xc0300308
+#define ixTHM_TMON0_DEBUG 0xc0300310
+#define ixTHM_TMON1_DEBUG 0xc0300314
+#define ixTHM_TMON2_DEBUG 0xc0300318
+#define ixTHM_TMON0_STATUS 0xc0300320
+#define ixTHM_TMON1_STATUS 0xc0300324
+#define ixTHM_TMON2_STATUS 0xc0300328
+#define ixGENERAL_PWRMGT 0xc0200000
+#define ixCNB_PWRMGT_CNTL 0xc0200004
+#define ixSCLK_PWRMGT_CNTL 0xc0200008
+#define ixTARGET_AND_CURRENT_PROFILE_INDEX 0xc0200014
+#define ixPWR_PCC_CONTROL 0xc0200018
+#define ixPWR_PCC_GPIO_SELECT 0xc020001c
+#define ixCG_FREQ_TRAN_VOTING_0 0xc02001a8
+#define ixCG_FREQ_TRAN_VOTING_1 0xc02001ac
+#define ixCG_FREQ_TRAN_VOTING_2 0xc02001b0
+#define ixCG_FREQ_TRAN_VOTING_3 0xc02001b4
+#define ixCG_FREQ_TRAN_VOTING_4 0xc02001b8
+#define ixCG_FREQ_TRAN_VOTING_5 0xc02001bc
+#define ixCG_FREQ_TRAN_VOTING_6 0xc02001c0
+#define ixCG_FREQ_TRAN_VOTING_7 0xc02001c4
+#define ixPLL_TEST_CNTL 0xc020003c
+#define ixCG_STATIC_SCREEN_PARAMETER 0xc0200044
+#define ixCG_DISPLAY_GAP_CNTL 0xc0200060
+#define ixCG_DISPLAY_GAP_CNTL2 0xc0200230
+#define ixCG_ACPI_CNTL 0xc0200064
+#define ixSCLK_DEEP_SLEEP_CNTL 0xc0200080
+#define ixSCLK_DEEP_SLEEP_CNTL2 0xc0200084
+#define ixSCLK_DEEP_SLEEP_CNTL3 0xc020009c
+#define ixSCLK_DEEP_SLEEP_MISC_CNTL 0xc0200088
+#define ixLCLK_DEEP_SLEEP_CNTL 0xc020008c
+#define ixLCLK_DEEP_SLEEP_CNTL2 0xc0200310
+#define ixTARGET_AND_CURRENT_PROFILE_INDEX_1 0xc02000f0
+#define ixCG_ULV_PARAMETER 0xc020015c
+#define ixSCLK_MIN_DIV 0xc02003ac
+#define ixPWR_AVFS_SEL 0xc0200384
+#define ixPWR_AVFS_CNTL 0xc0200388
+#define ixPWR_AVFS0_CNTL_STATUS 0xc0200400
+#define ixPWR_AVFS1_CNTL_STATUS 0xc0200404
+#define ixPWR_AVFS2_CNTL_STATUS 0xc0200408
+#define ixPWR_AVFS3_CNTL_STATUS 0xc020040c
+#define ixPWR_AVFS4_CNTL_STATUS 0xc0200410
+#define ixPWR_AVFS5_CNTL_STATUS 0xc0200414
+#define ixPWR_AVFS6_CNTL_STATUS 0xc0200418
+#define ixPWR_AVFS7_CNTL_STATUS 0xc020041c
+#define ixPWR_AVFS8_CNTL_STATUS 0xc0200420
+#define ixPWR_AVFS9_CNTL_STATUS 0xc0200424
+#define ixPWR_AVFS10_CNTL_STATUS 0xc0200428
+#define ixPWR_AVFS11_CNTL_STATUS 0xc020042c
+#define ixPWR_AVFS12_CNTL_STATUS 0xc0200430
+#define ixPWR_AVFS13_CNTL_STATUS 0xc0200434
+#define ixPWR_AVFS14_CNTL_STATUS 0xc0200438
+#define ixPWR_AVFS15_CNTL_STATUS 0xc020043c
+#define ixPWR_AVFS16_CNTL_STATUS 0xc0200440
+#define ixPWR_AVFS17_CNTL_STATUS 0xc0200444
+#define ixPWR_AVFS18_CNTL_STATUS 0xc0200448
+#define ixPWR_AVFS19_CNTL_STATUS 0xc020044c
+#define ixPWR_AVFS20_CNTL_STATUS 0xc0200450
+#define ixPWR_AVFS21_CNTL_STATUS 0xc0200454
+#define ixPWR_AVFS22_CNTL_STATUS 0xc0200458
+#define ixPWR_AVFS23_CNTL_STATUS 0xc020045c
+#define ixPWR_AVFS24_CNTL_STATUS 0xc0200460
+#define ixPWR_AVFS25_CNTL_STATUS 0xc0200464
+#define ixPWR_AVFS26_CNTL_STATUS 0xc0200468
+#define ixPWR_AVFS27_CNTL_STATUS 0xc020046c
+#define ixPWR_CKS_ENABLE 0xc020034c
+#define ixPWR_CKS_CNTL 0xc0200350
+#define ixPWR_DISP_TIMER_CONTROL 0xc02003c0
+#define ixPWR_DISP_TIMER_DEBUG 0xc02003c4
+#define ixPWR_DISP_TIMER2_CONTROL 0xc02003c8
+#define ixPWR_DISP_TIMER2_DEBUG 0xc02003cc
+#define ixPWR_DISP_TIMER_CONTROL2 0xc0200378
+#define ixVDDGFX_IDLE_PARAMETER 0xc020036c
+#define ixVDDGFX_IDLE_CONTROL 0xc0200370
+#define ixVDDGFX_IDLE_EXIT 0xc0200374
+#define ixLCAC_MC0_CNTL 0xc0400130
+#define ixLCAC_MC0_OVR_SEL 0xc0400134
+#define ixLCAC_MC0_OVR_VAL 0xc0400138
+#define ixLCAC_MC1_CNTL 0xc040013c
+#define ixLCAC_MC1_OVR_SEL 0xc0400140
+#define ixLCAC_MC1_OVR_VAL 0xc0400144
+#define ixLCAC_MC2_CNTL 0xc0400148
+#define ixLCAC_MC2_OVR_SEL 0xc040014c
+#define ixLCAC_MC2_OVR_VAL 0xc0400150
+#define ixLCAC_MC3_CNTL 0xc0400154
+#define ixLCAC_MC3_OVR_SEL 0xc0400158
+#define ixLCAC_MC3_OVR_VAL 0xc040015c
+#define ixLCAC_MC4_CNTL 0xc0400d60
+#define ixLCAC_MC4_OVR_SEL 0xc0400d64
+#define ixLCAC_MC4_OVR_VAL 0xc0400d68
+#define ixLCAC_MC5_CNTL 0xc0400d6c
+#define ixLCAC_MC5_OVR_SEL 0xc0400d70
+#define ixLCAC_MC5_OVR_VAL 0xc0400d74
+#define ixLCAC_MC6_CNTL 0xc0400d78
+#define ixLCAC_MC6_OVR_SEL 0xc0400d7c
+#define ixLCAC_MC6_OVR_VAL 0xc0400d80
+#define ixLCAC_MC7_CNTL 0xc0400d84
+#define ixLCAC_MC7_OVR_SEL 0xc0400d88
+#define ixLCAC_MC7_OVR_VAL 0xc0400d8c
+#define ixLCAC_CPL_CNTL 0xc0400160
+#define ixLCAC_CPL_OVR_SEL 0xc0400164
+#define ixLCAC_CPL_OVR_VAL 0xc0400168
+#define mmROM_SMC_IND_INDEX 0x80
+#define mmROM0_ROM_SMC_IND_INDEX 0x80
+#define mmROM1_ROM_SMC_IND_INDEX 0x82
+#define mmROM2_ROM_SMC_IND_INDEX 0x84
+#define mmROM3_ROM_SMC_IND_INDEX 0x86
+#define mmROM_SMC_IND_DATA 0x81
+#define mmROM0_ROM_SMC_IND_DATA 0x81
+#define mmROM1_ROM_SMC_IND_DATA 0x83
+#define mmROM2_ROM_SMC_IND_DATA 0x85
+#define mmROM3_ROM_SMC_IND_DATA 0x87
+#define ixROM_CNTL 0xc0600000
+#define ixPAGE_MIRROR_CNTL 0xc0600004
+#define ixROM_STATUS 0xc0600008
+#define ixCGTT_ROM_CLK_CTRL0 0xc060000c
+#define ixROM_INDEX 0xc0600010
+#define ixROM_DATA 0xc0600014
+#define ixROM_START 0xc0600018
+#define ixROM_SW_CNTL 0xc060001c
+#define ixROM_SW_STATUS 0xc0600020
+#define ixROM_SW_COMMAND 0xc0600024
+#define ixROM_SW_DATA_1 0xc0600028
+#define ixROM_SW_DATA_2 0xc060002c
+#define ixROM_SW_DATA_3 0xc0600030
+#define ixROM_SW_DATA_4 0xc0600034
+#define ixROM_SW_DATA_5 0xc0600038
+#define ixROM_SW_DATA_6 0xc060003c
+#define ixROM_SW_DATA_7 0xc0600040
+#define ixROM_SW_DATA_8 0xc0600044
+#define ixROM_SW_DATA_9 0xc0600048
+#define ixROM_SW_DATA_10 0xc060004c
+#define ixROM_SW_DATA_11 0xc0600050
+#define ixROM_SW_DATA_12 0xc0600054
+#define ixROM_SW_DATA_13 0xc0600058
+#define ixROM_SW_DATA_14 0xc060005c
+#define ixROM_SW_DATA_15 0xc0600060
+#define ixROM_SW_DATA_16 0xc0600064
+#define ixROM_SW_DATA_17 0xc0600068
+#define ixROM_SW_DATA_18 0xc060006c
+#define ixROM_SW_DATA_19 0xc0600070
+#define ixROM_SW_DATA_20 0xc0600074
+#define ixROM_SW_DATA_21 0xc0600078
+#define ixROM_SW_DATA_22 0xc060007c
+#define ixROM_SW_DATA_23 0xc0600080
+#define ixROM_SW_DATA_24 0xc0600084
+#define ixROM_SW_DATA_25 0xc0600088
+#define ixROM_SW_DATA_26 0xc060008c
+#define ixROM_SW_DATA_27 0xc0600090
+#define ixROM_SW_DATA_28 0xc0600094
+#define ixROM_SW_DATA_29 0xc0600098
+#define ixROM_SW_DATA_30 0xc060009c
+#define ixROM_SW_DATA_31 0xc06000a0
+#define ixROM_SW_DATA_32 0xc06000a4
+#define ixROM_SW_DATA_33 0xc06000a8
+#define ixROM_SW_DATA_34 0xc06000ac
+#define ixROM_SW_DATA_35 0xc06000b0
+#define ixROM_SW_DATA_36 0xc06000b4
+#define ixROM_SW_DATA_37 0xc06000b8
+#define ixROM_SW_DATA_38 0xc06000bc
+#define ixROM_SW_DATA_39 0xc06000c0
+#define ixROM_SW_DATA_40 0xc06000c4
+#define ixROM_SW_DATA_41 0xc06000c8
+#define ixROM_SW_DATA_42 0xc06000cc
+#define ixROM_SW_DATA_43 0xc06000d0
+#define ixROM_SW_DATA_44 0xc06000d4
+#define ixROM_SW_DATA_45 0xc06000d8
+#define ixROM_SW_DATA_46 0xc06000dc
+#define ixROM_SW_DATA_47 0xc06000e0
+#define ixROM_SW_DATA_48 0xc06000e4
+#define ixROM_SW_DATA_49 0xc06000e8
+#define ixROM_SW_DATA_50 0xc06000ec
+#define ixROM_SW_DATA_51 0xc06000f0
+#define ixROM_SW_DATA_52 0xc06000f4
+#define ixROM_SW_DATA_53 0xc06000f8
+#define ixROM_SW_DATA_54 0xc06000fc
+#define ixROM_SW_DATA_55 0xc0600100
+#define ixROM_SW_DATA_56 0xc0600104
+#define ixROM_SW_DATA_57 0xc0600108
+#define ixROM_SW_DATA_58 0xc060010c
+#define ixROM_SW_DATA_59 0xc0600110
+#define ixROM_SW_DATA_60 0xc0600114
+#define ixROM_SW_DATA_61 0xc0600118
+#define ixROM_SW_DATA_62 0xc060011c
+#define ixROM_SW_DATA_63 0xc0600120
+#define ixROM_SW_DATA_64 0xc0600124
+#define mmGC_CAC_CGTT_CLK_CTRL 0x3292
+#define mmSE_CAC_CGTT_CLK_CTRL 0x3293
+#define mmGC_CAC_LKG_AGGR_LOWER 0x3296
+#define mmGC_CAC_LKG_AGGR_UPPER 0x3297
+#define ixGC_CAC_WEIGHT_CU_0 0x32
+#define ixGC_CAC_WEIGHT_CU_1 0x33
+#define ixGC_CAC_WEIGHT_CU_2 0x34
+#define ixGC_CAC_WEIGHT_CU_3 0x35
+#define ixGC_CAC_WEIGHT_CU_4 0x36
+#define ixGC_CAC_WEIGHT_CU_5 0x37
+#define ixGC_CAC_WEIGHT_CU_6 0x38
+#define ixGC_CAC_WEIGHT_CU_7 0x39
+#define ixGC_CAC_ACC_CU0 0xba
+#define ixGC_CAC_ACC_CU1 0xbb
+#define ixGC_CAC_ACC_CU2 0xbc
+#define ixGC_CAC_ACC_CU3 0xbd
+#define ixGC_CAC_ACC_CU4 0xbe
+#define ixGC_CAC_ACC_CU5 0xbf
+#define ixGC_CAC_ACC_CU6 0xc0
+#define ixGC_CAC_ACC_CU7 0xc1
+#define ixGC_CAC_ACC_CU8 0xc2
+#define ixGC_CAC_ACC_CU9 0xc3
+#define ixGC_CAC_ACC_CU10 0xc4
+#define ixGC_CAC_ACC_CU11 0xc5
+#define ixGC_CAC_ACC_CU12 0xc6
+#define ixGC_CAC_ACC_CU13 0xc7
+#define ixGC_CAC_ACC_CU14 0xc8
+#define ixGC_CAC_ACC_CU15 0xc9
+#define ixGC_CAC_OVRD_CU 0xe7
+
+#endif /* SMU_7_1_3_D_H */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h
new file mode 100644
index 000000000000..f19c4208d963
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h
@@ -0,0 +1,1282 @@
+/*
+ * SMU_7_1_3 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SMU_7_1_3_ENUM_H
+#define SMU_7_1_3_ENUM_H
+
+#define CG_SRBM_START_ADDR 0x600
+#define CG_SRBM_END_ADDR 0x8ff
+#define RCU_CCF_DWORDS0 0xa0
+#define RCU_CCF_BITS0 0x1400
+#define RCU_SAM_BYTES 0x2c
+#define RCU_SAM_RTL_BYTES 0x2c
+#define RCU_SMU_BYTES 0x14
+#define RCU_SMU_RTL_BYTES 0x14
+#define SFP_CHAIN_ADDR 0x1
+#define SFP_SADR 0x0
+#define SFP_EADR 0x37f
+#define SAMU_KEY_CHAIN_ADR 0x0
+#define SAMU_KEY_SADR 0x280
+#define SAMU_KEY_EADR 0x2ab
+#define SMU_KEY_CHAIN_ADR 0x0
+#define SMU_KEY_SADR 0x2ac
+#define SMU_KEY_EADR 0x2bf
+#define SMC_MSG_TEST 0x1
+#define SMC_MSG_PHY_LN_OFF 0x2
+#define SMC_MSG_PHY_LN_ON 0x3
+#define SMC_MSG_DDI_PHY_OFF 0x4
+#define SMC_MSG_DDI_PHY_ON 0x5
+#define SMC_MSG_CASCADE_PLL_OFF 0x6
+#define SMC_MSG_CASCADE_PLL_ON 0x7
+#define SMC_MSG_PWR_OFF_x16 0x8
+#define SMC_MSG_CONFIG_LCLK_DPM 0x9
+#define SMC_MSG_FLUSH_DATA_CACHE 0xa
+#define SMC_MSG_FLUSH_INSTRUCTION_CACHE 0xb
+#define SMC_MSG_CONFIG_VPC_ACCUMULATOR 0xc
+#define SMC_MSG_CONFIG_BAPM 0xd
+#define SMC_MSG_CONFIG_TDC_LIMIT 0xe
+#define SMC_MSG_CONFIG_LPMx 0xf
+#define SMC_MSG_CONFIG_HTC_LIMIT 0x10
+#define SMC_MSG_CONFIG_THERMAL_CNTL 0x11
+#define SMC_MSG_CONFIG_VOLTAGE_CNTL 0x12
+#define SMC_MSG_CONFIG_TDP_CNTL 0x13
+#define SMC_MSG_EN_PM_CNTL 0x14
+#define SMC_MSG_DIS_PM_CNTL 0x15
+#define SMC_MSG_CONFIG_NBDPM 0x16
+#define SMC_MSG_CONFIG_LOADLINE 0x17
+#define SMC_MSG_ADJUST_LOADLINE 0x18
+#define SMC_MSG_RESET 0x20
+#define SMC_MSG_VOLTAGE 0x25
+#define SMC_VERSION_MAJOR 0x7
+#define SMC_VERSION_MINOR 0x0
+#define SMC_HEADER_SIZE 0x40
+#define ROM_SIGNATURE 0xaa55
+typedef enum SurfaceEndian {
+ ENDIAN_NONE = 0x0,
+ ENDIAN_8IN16 = 0x1,
+ ENDIAN_8IN32 = 0x2,
+ ENDIAN_8IN64 = 0x3,
+} SurfaceEndian;
+typedef enum ArrayMode {
+ ARRAY_LINEAR_GENERAL = 0x0,
+ ARRAY_LINEAR_ALIGNED = 0x1,
+ ARRAY_1D_TILED_THIN1 = 0x2,
+ ARRAY_1D_TILED_THICK = 0x3,
+ ARRAY_2D_TILED_THIN1 = 0x4,
+ ARRAY_PRT_TILED_THIN1 = 0x5,
+ ARRAY_PRT_2D_TILED_THIN1 = 0x6,
+ ARRAY_2D_TILED_THICK = 0x7,
+ ARRAY_2D_TILED_XTHICK = 0x8,
+ ARRAY_PRT_TILED_THICK = 0x9,
+ ARRAY_PRT_2D_TILED_THICK = 0xa,
+ ARRAY_PRT_3D_TILED_THIN1 = 0xb,
+ ARRAY_3D_TILED_THIN1 = 0xc,
+ ARRAY_3D_TILED_THICK = 0xd,
+ ARRAY_3D_TILED_XTHICK = 0xe,
+ ARRAY_PRT_3D_TILED_THICK = 0xf,
+} ArrayMode;
+typedef enum PipeTiling {
+ CONFIG_1_PIPE = 0x0,
+ CONFIG_2_PIPE = 0x1,
+ CONFIG_4_PIPE = 0x2,
+ CONFIG_8_PIPE = 0x3,
+} PipeTiling;
+typedef enum BankTiling {
+ CONFIG_4_BANK = 0x0,
+ CONFIG_8_BANK = 0x1,
+} BankTiling;
+typedef enum GroupInterleave {
+ CONFIG_256B_GROUP = 0x0,
+ CONFIG_512B_GROUP = 0x1,
+} GroupInterleave;
+typedef enum RowTiling {
+ CONFIG_1KB_ROW = 0x0,
+ CONFIG_2KB_ROW = 0x1,
+ CONFIG_4KB_ROW = 0x2,
+ CONFIG_8KB_ROW = 0x3,
+ CONFIG_1KB_ROW_OPT = 0x4,
+ CONFIG_2KB_ROW_OPT = 0x5,
+ CONFIG_4KB_ROW_OPT = 0x6,
+ CONFIG_8KB_ROW_OPT = 0x7,
+} RowTiling;
+typedef enum BankSwapBytes {
+ CONFIG_128B_SWAPS = 0x0,
+ CONFIG_256B_SWAPS = 0x1,
+ CONFIG_512B_SWAPS = 0x2,
+ CONFIG_1KB_SWAPS = 0x3,
+} BankSwapBytes;
+typedef enum SampleSplitBytes {
+ CONFIG_1KB_SPLIT = 0x0,
+ CONFIG_2KB_SPLIT = 0x1,
+ CONFIG_4KB_SPLIT = 0x2,
+ CONFIG_8KB_SPLIT = 0x3,
+} SampleSplitBytes;
+typedef enum NumPipes {
+ ADDR_CONFIG_1_PIPE = 0x0,
+ ADDR_CONFIG_2_PIPE = 0x1,
+ ADDR_CONFIG_4_PIPE = 0x2,
+ ADDR_CONFIG_8_PIPE = 0x3,
+} NumPipes;
+typedef enum PipeInterleaveSize {
+ ADDR_CONFIG_PIPE_INTERLEAVE_256B = 0x0,
+ ADDR_CONFIG_PIPE_INTERLEAVE_512B = 0x1,
+} PipeInterleaveSize;
+typedef enum BankInterleaveSize {
+ ADDR_CONFIG_BANK_INTERLEAVE_1 = 0x0,
+ ADDR_CONFIG_BANK_INTERLEAVE_2 = 0x1,
+ ADDR_CONFIG_BANK_INTERLEAVE_4 = 0x2,
+ ADDR_CONFIG_BANK_INTERLEAVE_8 = 0x3,
+} BankInterleaveSize;
+typedef enum NumShaderEngines {
+ ADDR_CONFIG_1_SHADER_ENGINE = 0x0,
+ ADDR_CONFIG_2_SHADER_ENGINE = 0x1,
+} NumShaderEngines;
+typedef enum ShaderEngineTileSize {
+ ADDR_CONFIG_SE_TILE_16 = 0x0,
+ ADDR_CONFIG_SE_TILE_32 = 0x1,
+} ShaderEngineTileSize;
+typedef enum NumGPUs {
+ ADDR_CONFIG_1_GPU = 0x0,
+ ADDR_CONFIG_2_GPU = 0x1,
+ ADDR_CONFIG_4_GPU = 0x2,
+} NumGPUs;
+typedef enum MultiGPUTileSize {
+ ADDR_CONFIG_GPU_TILE_16 = 0x0,
+ ADDR_CONFIG_GPU_TILE_32 = 0x1,
+ ADDR_CONFIG_GPU_TILE_64 = 0x2,
+ ADDR_CONFIG_GPU_TILE_128 = 0x3,
+} MultiGPUTileSize;
+typedef enum RowSize {
+ ADDR_CONFIG_1KB_ROW = 0x0,
+ ADDR_CONFIG_2KB_ROW = 0x1,
+ ADDR_CONFIG_4KB_ROW = 0x2,
+} RowSize;
+typedef enum NumLowerPipes {
+ ADDR_CONFIG_1_LOWER_PIPES = 0x0,
+ ADDR_CONFIG_2_LOWER_PIPES = 0x1,
+} NumLowerPipes;
+typedef enum DebugBlockId {
+ DBG_CLIENT_BLKID_RESERVED = 0x0,
+ DBG_CLIENT_BLKID_dbg = 0x1,
+ DBG_CLIENT_BLKID_scf2 = 0x2,
+ DBG_CLIENT_BLKID_mcd5_0 = 0x3,
+ DBG_CLIENT_BLKID_mcd5_1 = 0x4,
+ DBG_CLIENT_BLKID_mcd6_0 = 0x5,
+ DBG_CLIENT_BLKID_mcd6_1 = 0x6,
+ DBG_CLIENT_BLKID_mcd7_0 = 0x7,
+ DBG_CLIENT_BLKID_mcd7_1 = 0x8,
+ DBG_CLIENT_BLKID_vmc = 0x9,
+ DBG_CLIENT_BLKID_sx30 = 0xa,
+ DBG_CLIENT_BLKID_mcd2_0 = 0xb,
+ DBG_CLIENT_BLKID_mcd2_1 = 0xc,
+ DBG_CLIENT_BLKID_bci1 = 0xd,
+ DBG_CLIENT_BLKID_xdma_dbg_client_wrapper = 0xe,
+ DBG_CLIENT_BLKID_mcc0 = 0xf,
+ DBG_CLIENT_BLKID_uvdf_0 = 0x10,
+ DBG_CLIENT_BLKID_uvdf_1 = 0x11,
+ DBG_CLIENT_BLKID_uvdf_2 = 0x12,
+ DBG_CLIENT_BLKID_bci0 = 0x13,
+ DBG_CLIENT_BLKID_vcec0_0 = 0x14,
+ DBG_CLIENT_BLKID_cb100 = 0x15,
+ DBG_CLIENT_BLKID_cb001 = 0x16,
+ DBG_CLIENT_BLKID_cb002 = 0x17,
+ DBG_CLIENT_BLKID_cb003 = 0x18,
+ DBG_CLIENT_BLKID_mcd4_0 = 0x19,
+ DBG_CLIENT_BLKID_mcd4_1 = 0x1a,
+ DBG_CLIENT_BLKID_tmonw00 = 0x1b,
+ DBG_CLIENT_BLKID_cb101 = 0x1c,
+ DBG_CLIENT_BLKID_cb102 = 0x1d,
+ DBG_CLIENT_BLKID_cb103 = 0x1e,
+ DBG_CLIENT_BLKID_sx10 = 0x1f,
+ DBG_CLIENT_BLKID_cb301 = 0x20,
+ DBG_CLIENT_BLKID_cb302 = 0x21,
+ DBG_CLIENT_BLKID_cb303 = 0x22,
+ DBG_CLIENT_BLKID_tmonw01 = 0x23,
+ DBG_CLIENT_BLKID_tmonw02 = 0x24,
+ DBG_CLIENT_BLKID_vcea0_0 = 0x25,
+ DBG_CLIENT_BLKID_vcea0_1 = 0x26,
+ DBG_CLIENT_BLKID_vcea0_2 = 0x27,
+ DBG_CLIENT_BLKID_vcea0_3 = 0x28,
+ DBG_CLIENT_BLKID_scf1 = 0x29,
+ DBG_CLIENT_BLKID_sx20 = 0x2a,
+ DBG_CLIENT_BLKID_spim1 = 0x2b,
+ DBG_CLIENT_BLKID_scb1 = 0x2c,
+ DBG_CLIENT_BLKID_pa10 = 0x2d,
+ DBG_CLIENT_BLKID_pa00 = 0x2e,
+ DBG_CLIENT_BLKID_gmcon = 0x2f,
+ DBG_CLIENT_BLKID_mcb = 0x30,
+ DBG_CLIENT_BLKID_vgt0 = 0x31,
+ DBG_CLIENT_BLKID_pc0 = 0x32,
+ DBG_CLIENT_BLKID_bci2 = 0x33,
+ DBG_CLIENT_BLKID_uvdb_0 = 0x34,
+ DBG_CLIENT_BLKID_spim3 = 0x35,
+ DBG_CLIENT_BLKID_scb3 = 0x36,
+ DBG_CLIENT_BLKID_cpc_0 = 0x37,
+ DBG_CLIENT_BLKID_cpc_1 = 0x38,
+ DBG_CLIENT_BLKID_uvdm_0 = 0x39,
+ DBG_CLIENT_BLKID_uvdm_1 = 0x3a,
+ DBG_CLIENT_BLKID_uvdm_2 = 0x3b,
+ DBG_CLIENT_BLKID_uvdm_3 = 0x3c,
+ DBG_CLIENT_BLKID_cb000 = 0x3d,
+ DBG_CLIENT_BLKID_spim0 = 0x3e,
+ DBG_CLIENT_BLKID_scb0 = 0x3f,
+ DBG_CLIENT_BLKID_mcc2 = 0x40,
+ DBG_CLIENT_BLKID_ds0 = 0x41,
+ DBG_CLIENT_BLKID_srbm = 0x42,
+ DBG_CLIENT_BLKID_ih = 0x43,
+ DBG_CLIENT_BLKID_sem = 0x44,
+ DBG_CLIENT_BLKID_sdma_0 = 0x45,
+ DBG_CLIENT_BLKID_sdma_1 = 0x46,
+ DBG_CLIENT_BLKID_hdp = 0x47,
+ DBG_CLIENT_BLKID_acp_0 = 0x48,
+ DBG_CLIENT_BLKID_acp_1 = 0x49,
+ DBG_CLIENT_BLKID_cb200 = 0x4a,
+ DBG_CLIENT_BLKID_scf3 = 0x4b,
+ DBG_CLIENT_BLKID_bci3 = 0x4c,
+ DBG_CLIENT_BLKID_mcd0_0 = 0x4d,
+ DBG_CLIENT_BLKID_mcd0_1 = 0x4e,
+ DBG_CLIENT_BLKID_pa11 = 0x4f,
+ DBG_CLIENT_BLKID_pa01 = 0x50,
+ DBG_CLIENT_BLKID_cb201 = 0x51,
+ DBG_CLIENT_BLKID_cb202 = 0x52,
+ DBG_CLIENT_BLKID_cb203 = 0x53,
+ DBG_CLIENT_BLKID_spim2 = 0x54,
+ DBG_CLIENT_BLKID_scb2 = 0x55,
+ DBG_CLIENT_BLKID_vgt2 = 0x56,
+ DBG_CLIENT_BLKID_pc2 = 0x57,
+ DBG_CLIENT_BLKID_smu_0 = 0x58,
+ DBG_CLIENT_BLKID_smu_1 = 0x59,
+ DBG_CLIENT_BLKID_smu_2 = 0x5a,
+ DBG_CLIENT_BLKID_cb1 = 0x5b,
+ DBG_CLIENT_BLKID_ia0 = 0x5c,
+ DBG_CLIENT_BLKID_wd = 0x5d,
+ DBG_CLIENT_BLKID_ia1 = 0x5e,
+ DBG_CLIENT_BLKID_scf0 = 0x5f,
+ DBG_CLIENT_BLKID_vgt1 = 0x60,
+ DBG_CLIENT_BLKID_pc1 = 0x61,
+ DBG_CLIENT_BLKID_cb0 = 0x62,
+ DBG_CLIENT_BLKID_gdc_one_0 = 0x63,
+ DBG_CLIENT_BLKID_gdc_one_1 = 0x64,
+ DBG_CLIENT_BLKID_gdc_one_2 = 0x65,
+ DBG_CLIENT_BLKID_gdc_one_3 = 0x66,
+ DBG_CLIENT_BLKID_gdc_one_4 = 0x67,
+ DBG_CLIENT_BLKID_gdc_one_5 = 0x68,
+ DBG_CLIENT_BLKID_gdc_one_6 = 0x69,
+ DBG_CLIENT_BLKID_gdc_one_7 = 0x6a,
+ DBG_CLIENT_BLKID_gdc_one_8 = 0x6b,
+ DBG_CLIENT_BLKID_gdc_one_9 = 0x6c,
+ DBG_CLIENT_BLKID_gdc_one_10 = 0x6d,
+ DBG_CLIENT_BLKID_gdc_one_11 = 0x6e,
+ DBG_CLIENT_BLKID_gdc_one_12 = 0x6f,
+ DBG_CLIENT_BLKID_gdc_one_13 = 0x70,
+ DBG_CLIENT_BLKID_gdc_one_14 = 0x71,
+ DBG_CLIENT_BLKID_gdc_one_15 = 0x72,
+ DBG_CLIENT_BLKID_gdc_one_16 = 0x73,
+ DBG_CLIENT_BLKID_gdc_one_17 = 0x74,
+ DBG_CLIENT_BLKID_gdc_one_18 = 0x75,
+ DBG_CLIENT_BLKID_gdc_one_19 = 0x76,
+ DBG_CLIENT_BLKID_gdc_one_20 = 0x77,
+ DBG_CLIENT_BLKID_gdc_one_21 = 0x78,
+ DBG_CLIENT_BLKID_gdc_one_22 = 0x79,
+ DBG_CLIENT_BLKID_gdc_one_23 = 0x7a,
+ DBG_CLIENT_BLKID_gdc_one_24 = 0x7b,
+ DBG_CLIENT_BLKID_gdc_one_25 = 0x7c,
+ DBG_CLIENT_BLKID_gdc_one_26 = 0x7d,
+ DBG_CLIENT_BLKID_gdc_one_27 = 0x7e,
+ DBG_CLIENT_BLKID_gdc_one_28 = 0x7f,
+ DBG_CLIENT_BLKID_gdc_one_29 = 0x80,
+ DBG_CLIENT_BLKID_gdc_one_30 = 0x81,
+ DBG_CLIENT_BLKID_gdc_one_31 = 0x82,
+ DBG_CLIENT_BLKID_gdc_one_32 = 0x83,
+ DBG_CLIENT_BLKID_gdc_one_33 = 0x84,
+ DBG_CLIENT_BLKID_gdc_one_34 = 0x85,
+ DBG_CLIENT_BLKID_gdc_one_35 = 0x86,
+ DBG_CLIENT_BLKID_vceb0_0 = 0x87,
+ DBG_CLIENT_BLKID_vgt3 = 0x88,
+ DBG_CLIENT_BLKID_pc3 = 0x89,
+ DBG_CLIENT_BLKID_mcd3_0 = 0x8a,
+ DBG_CLIENT_BLKID_mcd3_1 = 0x8b,
+ DBG_CLIENT_BLKID_uvdu_0 = 0x8c,
+ DBG_CLIENT_BLKID_uvdu_1 = 0x8d,
+ DBG_CLIENT_BLKID_uvdu_2 = 0x8e,
+ DBG_CLIENT_BLKID_uvdu_3 = 0x8f,
+ DBG_CLIENT_BLKID_uvdu_4 = 0x90,
+ DBG_CLIENT_BLKID_uvdu_5 = 0x91,
+ DBG_CLIENT_BLKID_uvdu_6 = 0x92,
+ DBG_CLIENT_BLKID_cb300 = 0x93,
+ DBG_CLIENT_BLKID_mcd1_0 = 0x94,
+ DBG_CLIENT_BLKID_mcd1_1 = 0x95,
+ DBG_CLIENT_BLKID_sx00 = 0x96,
+ DBG_CLIENT_BLKID_uvdc_0 = 0x97,
+ DBG_CLIENT_BLKID_uvdc_1 = 0x98,
+ DBG_CLIENT_BLKID_mcc3 = 0x99,
+ DBG_CLIENT_BLKID_mcc4 = 0x9a,
+ DBG_CLIENT_BLKID_mcc5 = 0x9b,
+ DBG_CLIENT_BLKID_mcc6 = 0x9c,
+ DBG_CLIENT_BLKID_mcc7 = 0x9d,
+ DBG_CLIENT_BLKID_cpg_0 = 0x9e,
+ DBG_CLIENT_BLKID_cpg_1 = 0x9f,
+ DBG_CLIENT_BLKID_gck = 0xa0,
+ DBG_CLIENT_BLKID_mcc1 = 0xa1,
+ DBG_CLIENT_BLKID_cpf_0 = 0xa2,
+ DBG_CLIENT_BLKID_cpf_1 = 0xa3,
+ DBG_CLIENT_BLKID_rlc = 0xa4,
+ DBG_CLIENT_BLKID_grbm = 0xa5,
+ DBG_CLIENT_BLKID_sammsp = 0xa6,
+ DBG_CLIENT_BLKID_dci_pg = 0xa7,
+ DBG_CLIENT_BLKID_dci_0 = 0xa8,
+ DBG_CLIENT_BLKID_dccg0_0 = 0xa9,
+ DBG_CLIENT_BLKID_dccg0_1 = 0xaa,
+ DBG_CLIENT_BLKID_dcfe01_0 = 0xab,
+ DBG_CLIENT_BLKID_dcfe02_0 = 0xac,
+ DBG_CLIENT_BLKID_dcfe03_0 = 0xad,
+ DBG_CLIENT_BLKID_dcfe04_0 = 0xae,
+ DBG_CLIENT_BLKID_dcfe05_0 = 0xaf,
+ DBG_CLIENT_BLKID_dcfe06_0 = 0xb0,
+ DBG_CLIENT_BLKID_mcq0_0 = 0xb1,
+ DBG_CLIENT_BLKID_mcq0_1 = 0xb2,
+ DBG_CLIENT_BLKID_mcq1_0 = 0xb3,
+ DBG_CLIENT_BLKID_mcq1_1 = 0xb4,
+ DBG_CLIENT_BLKID_mcq2_0 = 0xb5,
+ DBG_CLIENT_BLKID_mcq2_1 = 0xb6,
+ DBG_CLIENT_BLKID_mcq3_0 = 0xb7,
+ DBG_CLIENT_BLKID_mcq3_1 = 0xb8,
+ DBG_CLIENT_BLKID_mcq4_0 = 0xb9,
+ DBG_CLIENT_BLKID_mcq4_1 = 0xba,
+ DBG_CLIENT_BLKID_mcq5_0 = 0xbb,
+ DBG_CLIENT_BLKID_mcq5_1 = 0xbc,
+ DBG_CLIENT_BLKID_mcq6_0 = 0xbd,
+ DBG_CLIENT_BLKID_mcq6_1 = 0xbe,
+ DBG_CLIENT_BLKID_mcq7_0 = 0xbf,
+ DBG_CLIENT_BLKID_mcq7_1 = 0xc0,
+ DBG_CLIENT_BLKID_uvdi_0 = 0xc1,
+ DBG_CLIENT_BLKID_RESERVED_LAST = 0xc2,
+} DebugBlockId;
+typedef enum DebugBlockId_OLD {
+ DBG_BLOCK_ID_RESERVED = 0x0,
+ DBG_BLOCK_ID_DBG = 0x1,
+ DBG_BLOCK_ID_VMC = 0x2,
+ DBG_BLOCK_ID_PDMA = 0x3,
+ DBG_BLOCK_ID_CG = 0x4,
+ DBG_BLOCK_ID_SRBM = 0x5,
+ DBG_BLOCK_ID_GRBM = 0x6,
+ DBG_BLOCK_ID_RLC = 0x7,
+ DBG_BLOCK_ID_CSC = 0x8,
+ DBG_BLOCK_ID_SEM = 0x9,
+ DBG_BLOCK_ID_IH = 0xa,
+ DBG_BLOCK_ID_SC = 0xb,
+ DBG_BLOCK_ID_SQ = 0xc,
+ DBG_BLOCK_ID_AVP = 0xd,
+ DBG_BLOCK_ID_GMCON = 0xe,
+ DBG_BLOCK_ID_SMU = 0xf,
+ DBG_BLOCK_ID_DMA0 = 0x10,
+ DBG_BLOCK_ID_DMA1 = 0x11,
+ DBG_BLOCK_ID_SPIM = 0x12,
+ DBG_BLOCK_ID_GDS = 0x13,
+ DBG_BLOCK_ID_SPIS = 0x14,
+ DBG_BLOCK_ID_UNUSED0 = 0x15,
+ DBG_BLOCK_ID_PA0 = 0x16,
+ DBG_BLOCK_ID_PA1 = 0x17,
+ DBG_BLOCK_ID_CP0 = 0x18,
+ DBG_BLOCK_ID_CP1 = 0x19,
+ DBG_BLOCK_ID_CP2 = 0x1a,
+ DBG_BLOCK_ID_UNUSED1 = 0x1b,
+ DBG_BLOCK_ID_UVDU = 0x1c,
+ DBG_BLOCK_ID_UVDM = 0x1d,
+ DBG_BLOCK_ID_VCE = 0x1e,
+ DBG_BLOCK_ID_UNUSED2 = 0x1f,
+ DBG_BLOCK_ID_VGT0 = 0x20,
+ DBG_BLOCK_ID_VGT1 = 0x21,
+ DBG_BLOCK_ID_IA = 0x22,
+ DBG_BLOCK_ID_UNUSED3 = 0x23,
+ DBG_BLOCK_ID_SCT0 = 0x24,
+ DBG_BLOCK_ID_SCT1 = 0x25,
+ DBG_BLOCK_ID_SPM0 = 0x26,
+ DBG_BLOCK_ID_SPM1 = 0x27,
+ DBG_BLOCK_ID_TCAA = 0x28,
+ DBG_BLOCK_ID_TCAB = 0x29,
+ DBG_BLOCK_ID_TCCA = 0x2a,
+ DBG_BLOCK_ID_TCCB = 0x2b,
+ DBG_BLOCK_ID_MCC0 = 0x2c,
+ DBG_BLOCK_ID_MCC1 = 0x2d,
+ DBG_BLOCK_ID_MCC2 = 0x2e,
+ DBG_BLOCK_ID_MCC3 = 0x2f,
+ DBG_BLOCK_ID_SX0 = 0x30,
+ DBG_BLOCK_ID_SX1 = 0x31,
+ DBG_BLOCK_ID_SX2 = 0x32,
+ DBG_BLOCK_ID_SX3 = 0x33,
+ DBG_BLOCK_ID_UNUSED4 = 0x34,
+ DBG_BLOCK_ID_UNUSED5 = 0x35,
+ DBG_BLOCK_ID_UNUSED6 = 0x36,
+ DBG_BLOCK_ID_UNUSED7 = 0x37,
+ DBG_BLOCK_ID_PC0 = 0x38,
+ DBG_BLOCK_ID_PC1 = 0x39,
+ DBG_BLOCK_ID_UNUSED8 = 0x3a,
+ DBG_BLOCK_ID_UNUSED9 = 0x3b,
+ DBG_BLOCK_ID_UNUSED10 = 0x3c,
+ DBG_BLOCK_ID_UNUSED11 = 0x3d,
+ DBG_BLOCK_ID_MCB = 0x3e,
+ DBG_BLOCK_ID_UNUSED12 = 0x3f,
+ DBG_BLOCK_ID_SCB0 = 0x40,
+ DBG_BLOCK_ID_SCB1 = 0x41,
+ DBG_BLOCK_ID_UNUSED13 = 0x42,
+ DBG_BLOCK_ID_UNUSED14 = 0x43,
+ DBG_BLOCK_ID_SCF0 = 0x44,
+ DBG_BLOCK_ID_SCF1 = 0x45,
+ DBG_BLOCK_ID_UNUSED15 = 0x46,
+ DBG_BLOCK_ID_UNUSED16 = 0x47,
+ DBG_BLOCK_ID_BCI0 = 0x48,
+ DBG_BLOCK_ID_BCI1 = 0x49,
+ DBG_BLOCK_ID_BCI2 = 0x4a,
+ DBG_BLOCK_ID_BCI3 = 0x4b,
+ DBG_BLOCK_ID_UNUSED17 = 0x4c,
+ DBG_BLOCK_ID_UNUSED18 = 0x4d,
+ DBG_BLOCK_ID_UNUSED19 = 0x4e,
+ DBG_BLOCK_ID_UNUSED20 = 0x4f,
+ DBG_BLOCK_ID_CB00 = 0x50,
+ DBG_BLOCK_ID_CB01 = 0x51,
+ DBG_BLOCK_ID_CB02 = 0x52,
+ DBG_BLOCK_ID_CB03 = 0x53,
+ DBG_BLOCK_ID_CB04 = 0x54,
+ DBG_BLOCK_ID_UNUSED21 = 0x55,
+ DBG_BLOCK_ID_UNUSED22 = 0x56,
+ DBG_BLOCK_ID_UNUSED23 = 0x57,
+ DBG_BLOCK_ID_CB10 = 0x58,
+ DBG_BLOCK_ID_CB11 = 0x59,
+ DBG_BLOCK_ID_CB12 = 0x5a,
+ DBG_BLOCK_ID_CB13 = 0x5b,
+ DBG_BLOCK_ID_CB14 = 0x5c,
+ DBG_BLOCK_ID_UNUSED24 = 0x5d,
+ DBG_BLOCK_ID_UNUSED25 = 0x5e,
+ DBG_BLOCK_ID_UNUSED26 = 0x5f,
+ DBG_BLOCK_ID_TCP0 = 0x60,
+ DBG_BLOCK_ID_TCP1 = 0x61,
+ DBG_BLOCK_ID_TCP2 = 0x62,
+ DBG_BLOCK_ID_TCP3 = 0x63,
+ DBG_BLOCK_ID_TCP4 = 0x64,
+ DBG_BLOCK_ID_TCP5 = 0x65,
+ DBG_BLOCK_ID_TCP6 = 0x66,
+ DBG_BLOCK_ID_TCP7 = 0x67,
+ DBG_BLOCK_ID_TCP8 = 0x68,
+ DBG_BLOCK_ID_TCP9 = 0x69,
+ DBG_BLOCK_ID_TCP10 = 0x6a,
+ DBG_BLOCK_ID_TCP11 = 0x6b,
+ DBG_BLOCK_ID_TCP12 = 0x6c,
+ DBG_BLOCK_ID_TCP13 = 0x6d,
+ DBG_BLOCK_ID_TCP14 = 0x6e,
+ DBG_BLOCK_ID_TCP15 = 0x6f,
+ DBG_BLOCK_ID_TCP16 = 0x70,
+ DBG_BLOCK_ID_TCP17 = 0x71,
+ DBG_BLOCK_ID_TCP18 = 0x72,
+ DBG_BLOCK_ID_TCP19 = 0x73,
+ DBG_BLOCK_ID_TCP20 = 0x74,
+ DBG_BLOCK_ID_TCP21 = 0x75,
+ DBG_BLOCK_ID_TCP22 = 0x76,
+ DBG_BLOCK_ID_TCP23 = 0x77,
+ DBG_BLOCK_ID_TCP_RESERVED0 = 0x78,
+ DBG_BLOCK_ID_TCP_RESERVED1 = 0x79,
+ DBG_BLOCK_ID_TCP_RESERVED2 = 0x7a,
+ DBG_BLOCK_ID_TCP_RESERVED3 = 0x7b,
+ DBG_BLOCK_ID_TCP_RESERVED4 = 0x7c,
+ DBG_BLOCK_ID_TCP_RESERVED5 = 0x7d,
+ DBG_BLOCK_ID_TCP_RESERVED6 = 0x7e,
+ DBG_BLOCK_ID_TCP_RESERVED7 = 0x7f,
+ DBG_BLOCK_ID_DB00 = 0x80,
+ DBG_BLOCK_ID_DB01 = 0x81,
+ DBG_BLOCK_ID_DB02 = 0x82,
+ DBG_BLOCK_ID_DB03 = 0x83,
+ DBG_BLOCK_ID_DB04 = 0x84,
+ DBG_BLOCK_ID_UNUSED27 = 0x85,
+ DBG_BLOCK_ID_UNUSED28 = 0x86,
+ DBG_BLOCK_ID_UNUSED29 = 0x87,
+ DBG_BLOCK_ID_DB10 = 0x88,
+ DBG_BLOCK_ID_DB11 = 0x89,
+ DBG_BLOCK_ID_DB12 = 0x8a,
+ DBG_BLOCK_ID_DB13 = 0x8b,
+ DBG_BLOCK_ID_DB14 = 0x8c,
+ DBG_BLOCK_ID_UNUSED30 = 0x8d,
+ DBG_BLOCK_ID_UNUSED31 = 0x8e,
+ DBG_BLOCK_ID_UNUSED32 = 0x8f,
+ DBG_BLOCK_ID_TCC0 = 0x90,
+ DBG_BLOCK_ID_TCC1 = 0x91,
+ DBG_BLOCK_ID_TCC2 = 0x92,
+ DBG_BLOCK_ID_TCC3 = 0x93,
+ DBG_BLOCK_ID_TCC4 = 0x94,
+ DBG_BLOCK_ID_TCC5 = 0x95,
+ DBG_BLOCK_ID_TCC6 = 0x96,
+ DBG_BLOCK_ID_TCC7 = 0x97,
+ DBG_BLOCK_ID_SPS00 = 0x98,
+ DBG_BLOCK_ID_SPS01 = 0x99,
+ DBG_BLOCK_ID_SPS02 = 0x9a,
+ DBG_BLOCK_ID_SPS10 = 0x9b,
+ DBG_BLOCK_ID_SPS11 = 0x9c,
+ DBG_BLOCK_ID_SPS12 = 0x9d,
+ DBG_BLOCK_ID_UNUSED33 = 0x9e,
+ DBG_BLOCK_ID_UNUSED34 = 0x9f,
+ DBG_BLOCK_ID_TA00 = 0xa0,
+ DBG_BLOCK_ID_TA01 = 0xa1,
+ DBG_BLOCK_ID_TA02 = 0xa2,
+ DBG_BLOCK_ID_TA03 = 0xa3,
+ DBG_BLOCK_ID_TA04 = 0xa4,
+ DBG_BLOCK_ID_TA05 = 0xa5,
+ DBG_BLOCK_ID_TA06 = 0xa6,
+ DBG_BLOCK_ID_TA07 = 0xa7,
+ DBG_BLOCK_ID_TA08 = 0xa8,
+ DBG_BLOCK_ID_TA09 = 0xa9,
+ DBG_BLOCK_ID_TA0A = 0xaa,
+ DBG_BLOCK_ID_TA0B = 0xab,
+ DBG_BLOCK_ID_UNUSED35 = 0xac,
+ DBG_BLOCK_ID_UNUSED36 = 0xad,
+ DBG_BLOCK_ID_UNUSED37 = 0xae,
+ DBG_BLOCK_ID_UNUSED38 = 0xaf,
+ DBG_BLOCK_ID_TA10 = 0xb0,
+ DBG_BLOCK_ID_TA11 = 0xb1,
+ DBG_BLOCK_ID_TA12 = 0xb2,
+ DBG_BLOCK_ID_TA13 = 0xb3,
+ DBG_BLOCK_ID_TA14 = 0xb4,
+ DBG_BLOCK_ID_TA15 = 0xb5,
+ DBG_BLOCK_ID_TA16 = 0xb6,
+ DBG_BLOCK_ID_TA17 = 0xb7,
+ DBG_BLOCK_ID_TA18 = 0xb8,
+ DBG_BLOCK_ID_TA19 = 0xb9,
+ DBG_BLOCK_ID_TA1A = 0xba,
+ DBG_BLOCK_ID_TA1B = 0xbb,
+ DBG_BLOCK_ID_UNUSED39 = 0xbc,
+ DBG_BLOCK_ID_UNUSED40 = 0xbd,
+ DBG_BLOCK_ID_UNUSED41 = 0xbe,
+ DBG_BLOCK_ID_UNUSED42 = 0xbf,
+ DBG_BLOCK_ID_TD00 = 0xc0,
+ DBG_BLOCK_ID_TD01 = 0xc1,
+ DBG_BLOCK_ID_TD02 = 0xc2,
+ DBG_BLOCK_ID_TD03 = 0xc3,
+ DBG_BLOCK_ID_TD04 = 0xc4,
+ DBG_BLOCK_ID_TD05 = 0xc5,
+ DBG_BLOCK_ID_TD06 = 0xc6,
+ DBG_BLOCK_ID_TD07 = 0xc7,
+ DBG_BLOCK_ID_TD08 = 0xc8,
+ DBG_BLOCK_ID_TD09 = 0xc9,
+ DBG_BLOCK_ID_TD0A = 0xca,
+ DBG_BLOCK_ID_TD0B = 0xcb,
+ DBG_BLOCK_ID_UNUSED43 = 0xcc,
+ DBG_BLOCK_ID_UNUSED44 = 0xcd,
+ DBG_BLOCK_ID_UNUSED45 = 0xce,
+ DBG_BLOCK_ID_UNUSED46 = 0xcf,
+ DBG_BLOCK_ID_TD10 = 0xd0,
+ DBG_BLOCK_ID_TD11 = 0xd1,
+ DBG_BLOCK_ID_TD12 = 0xd2,
+ DBG_BLOCK_ID_TD13 = 0xd3,
+ DBG_BLOCK_ID_TD14 = 0xd4,
+ DBG_BLOCK_ID_TD15 = 0xd5,
+ DBG_BLOCK_ID_TD16 = 0xd6,
+ DBG_BLOCK_ID_TD17 = 0xd7,
+ DBG_BLOCK_ID_TD18 = 0xd8,
+ DBG_BLOCK_ID_TD19 = 0xd9,
+ DBG_BLOCK_ID_TD1A = 0xda,
+ DBG_BLOCK_ID_TD1B = 0xdb,
+ DBG_BLOCK_ID_UNUSED47 = 0xdc,
+ DBG_BLOCK_ID_UNUSED48 = 0xdd,
+ DBG_BLOCK_ID_UNUSED49 = 0xde,
+ DBG_BLOCK_ID_UNUSED50 = 0xdf,
+ DBG_BLOCK_ID_MCD0 = 0xe0,
+ DBG_BLOCK_ID_MCD1 = 0xe1,
+ DBG_BLOCK_ID_MCD2 = 0xe2,
+ DBG_BLOCK_ID_MCD3 = 0xe3,
+ DBG_BLOCK_ID_MCD4 = 0xe4,
+ DBG_BLOCK_ID_MCD5 = 0xe5,
+ DBG_BLOCK_ID_UNUSED51 = 0xe6,
+ DBG_BLOCK_ID_UNUSED52 = 0xe7,
+} DebugBlockId_OLD;
+typedef enum DebugBlockId_BY2 {
+ DBG_BLOCK_ID_RESERVED_BY2 = 0x0,
+ DBG_BLOCK_ID_VMC_BY2 = 0x1,
+ DBG_BLOCK_ID_CG_BY2 = 0x2,
+ DBG_BLOCK_ID_GRBM_BY2 = 0x3,
+ DBG_BLOCK_ID_CSC_BY2 = 0x4,
+ DBG_BLOCK_ID_IH_BY2 = 0x5,
+ DBG_BLOCK_ID_SQ_BY2 = 0x6,
+ DBG_BLOCK_ID_GMCON_BY2 = 0x7,
+ DBG_BLOCK_ID_DMA0_BY2 = 0x8,
+ DBG_BLOCK_ID_SPIM_BY2 = 0x9,
+ DBG_BLOCK_ID_SPIS_BY2 = 0xa,
+ DBG_BLOCK_ID_PA0_BY2 = 0xb,
+ DBG_BLOCK_ID_CP0_BY2 = 0xc,
+ DBG_BLOCK_ID_CP2_BY2 = 0xd,
+ DBG_BLOCK_ID_UVDU_BY2 = 0xe,
+ DBG_BLOCK_ID_VCE_BY2 = 0xf,
+ DBG_BLOCK_ID_VGT0_BY2 = 0x10,
+ DBG_BLOCK_ID_IA_BY2 = 0x11,
+ DBG_BLOCK_ID_SCT0_BY2 = 0x12,
+ DBG_BLOCK_ID_SPM0_BY2 = 0x13,
+ DBG_BLOCK_ID_TCAA_BY2 = 0x14,
+ DBG_BLOCK_ID_TCCA_BY2 = 0x15,
+ DBG_BLOCK_ID_MCC0_BY2 = 0x16,
+ DBG_BLOCK_ID_MCC2_BY2 = 0x17,
+ DBG_BLOCK_ID_SX0_BY2 = 0x18,
+ DBG_BLOCK_ID_SX2_BY2 = 0x19,
+ DBG_BLOCK_ID_UNUSED4_BY2 = 0x1a,
+ DBG_BLOCK_ID_UNUSED6_BY2 = 0x1b,
+ DBG_BLOCK_ID_PC0_BY2 = 0x1c,
+ DBG_BLOCK_ID_UNUSED8_BY2 = 0x1d,
+ DBG_BLOCK_ID_UNUSED10_BY2 = 0x1e,
+ DBG_BLOCK_ID_MCB_BY2 = 0x1f,
+ DBG_BLOCK_ID_SCB0_BY2 = 0x20,
+ DBG_BLOCK_ID_UNUSED13_BY2 = 0x21,
+ DBG_BLOCK_ID_SCF0_BY2 = 0x22,
+ DBG_BLOCK_ID_UNUSED15_BY2 = 0x23,
+ DBG_BLOCK_ID_BCI0_BY2 = 0x24,
+ DBG_BLOCK_ID_BCI2_BY2 = 0x25,
+ DBG_BLOCK_ID_UNUSED17_BY2 = 0x26,
+ DBG_BLOCK_ID_UNUSED19_BY2 = 0x27,
+ DBG_BLOCK_ID_CB00_BY2 = 0x28,
+ DBG_BLOCK_ID_CB02_BY2 = 0x29,
+ DBG_BLOCK_ID_CB04_BY2 = 0x2a,
+ DBG_BLOCK_ID_UNUSED22_BY2 = 0x2b,
+ DBG_BLOCK_ID_CB10_BY2 = 0x2c,
+ DBG_BLOCK_ID_CB12_BY2 = 0x2d,
+ DBG_BLOCK_ID_CB14_BY2 = 0x2e,
+ DBG_BLOCK_ID_UNUSED25_BY2 = 0x2f,
+ DBG_BLOCK_ID_TCP0_BY2 = 0x30,
+ DBG_BLOCK_ID_TCP2_BY2 = 0x31,
+ DBG_BLOCK_ID_TCP4_BY2 = 0x32,
+ DBG_BLOCK_ID_TCP6_BY2 = 0x33,
+ DBG_BLOCK_ID_TCP8_BY2 = 0x34,
+ DBG_BLOCK_ID_TCP10_BY2 = 0x35,
+ DBG_BLOCK_ID_TCP12_BY2 = 0x36,
+ DBG_BLOCK_ID_TCP14_BY2 = 0x37,
+ DBG_BLOCK_ID_TCP16_BY2 = 0x38,
+ DBG_BLOCK_ID_TCP18_BY2 = 0x39,
+ DBG_BLOCK_ID_TCP20_BY2 = 0x3a,
+ DBG_BLOCK_ID_TCP22_BY2 = 0x3b,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY2 = 0x3c,
+ DBG_BLOCK_ID_TCP_RESERVED2_BY2 = 0x3d,
+ DBG_BLOCK_ID_TCP_RESERVED4_BY2 = 0x3e,
+ DBG_BLOCK_ID_TCP_RESERVED6_BY2 = 0x3f,
+ DBG_BLOCK_ID_DB00_BY2 = 0x40,
+ DBG_BLOCK_ID_DB02_BY2 = 0x41,
+ DBG_BLOCK_ID_DB04_BY2 = 0x42,
+ DBG_BLOCK_ID_UNUSED28_BY2 = 0x43,
+ DBG_BLOCK_ID_DB10_BY2 = 0x44,
+ DBG_BLOCK_ID_DB12_BY2 = 0x45,
+ DBG_BLOCK_ID_DB14_BY2 = 0x46,
+ DBG_BLOCK_ID_UNUSED31_BY2 = 0x47,
+ DBG_BLOCK_ID_TCC0_BY2 = 0x48,
+ DBG_BLOCK_ID_TCC2_BY2 = 0x49,
+ DBG_BLOCK_ID_TCC4_BY2 = 0x4a,
+ DBG_BLOCK_ID_TCC6_BY2 = 0x4b,
+ DBG_BLOCK_ID_SPS00_BY2 = 0x4c,
+ DBG_BLOCK_ID_SPS02_BY2 = 0x4d,
+ DBG_BLOCK_ID_SPS11_BY2 = 0x4e,
+ DBG_BLOCK_ID_UNUSED33_BY2 = 0x4f,
+ DBG_BLOCK_ID_TA00_BY2 = 0x50,
+ DBG_BLOCK_ID_TA02_BY2 = 0x51,
+ DBG_BLOCK_ID_TA04_BY2 = 0x52,
+ DBG_BLOCK_ID_TA06_BY2 = 0x53,
+ DBG_BLOCK_ID_TA08_BY2 = 0x54,
+ DBG_BLOCK_ID_TA0A_BY2 = 0x55,
+ DBG_BLOCK_ID_UNUSED35_BY2 = 0x56,
+ DBG_BLOCK_ID_UNUSED37_BY2 = 0x57,
+ DBG_BLOCK_ID_TA10_BY2 = 0x58,
+ DBG_BLOCK_ID_TA12_BY2 = 0x59,
+ DBG_BLOCK_ID_TA14_BY2 = 0x5a,
+ DBG_BLOCK_ID_TA16_BY2 = 0x5b,
+ DBG_BLOCK_ID_TA18_BY2 = 0x5c,
+ DBG_BLOCK_ID_TA1A_BY2 = 0x5d,
+ DBG_BLOCK_ID_UNUSED39_BY2 = 0x5e,
+ DBG_BLOCK_ID_UNUSED41_BY2 = 0x5f,
+ DBG_BLOCK_ID_TD00_BY2 = 0x60,
+ DBG_BLOCK_ID_TD02_BY2 = 0x61,
+ DBG_BLOCK_ID_TD04_BY2 = 0x62,
+ DBG_BLOCK_ID_TD06_BY2 = 0x63,
+ DBG_BLOCK_ID_TD08_BY2 = 0x64,
+ DBG_BLOCK_ID_TD0A_BY2 = 0x65,
+ DBG_BLOCK_ID_UNUSED43_BY2 = 0x66,
+ DBG_BLOCK_ID_UNUSED45_BY2 = 0x67,
+ DBG_BLOCK_ID_TD10_BY2 = 0x68,
+ DBG_BLOCK_ID_TD12_BY2 = 0x69,
+ DBG_BLOCK_ID_TD14_BY2 = 0x6a,
+ DBG_BLOCK_ID_TD16_BY2 = 0x6b,
+ DBG_BLOCK_ID_TD18_BY2 = 0x6c,
+ DBG_BLOCK_ID_TD1A_BY2 = 0x6d,
+ DBG_BLOCK_ID_UNUSED47_BY2 = 0x6e,
+ DBG_BLOCK_ID_UNUSED49_BY2 = 0x6f,
+ DBG_BLOCK_ID_MCD0_BY2 = 0x70,
+ DBG_BLOCK_ID_MCD2_BY2 = 0x71,
+ DBG_BLOCK_ID_MCD4_BY2 = 0x72,
+ DBG_BLOCK_ID_UNUSED51_BY2 = 0x73,
+} DebugBlockId_BY2;
+typedef enum DebugBlockId_BY4 {
+ DBG_BLOCK_ID_RESERVED_BY4 = 0x0,
+ DBG_BLOCK_ID_CG_BY4 = 0x1,
+ DBG_BLOCK_ID_CSC_BY4 = 0x2,
+ DBG_BLOCK_ID_SQ_BY4 = 0x3,
+ DBG_BLOCK_ID_DMA0_BY4 = 0x4,
+ DBG_BLOCK_ID_SPIS_BY4 = 0x5,
+ DBG_BLOCK_ID_CP0_BY4 = 0x6,
+ DBG_BLOCK_ID_UVDU_BY4 = 0x7,
+ DBG_BLOCK_ID_VGT0_BY4 = 0x8,
+ DBG_BLOCK_ID_SCT0_BY4 = 0x9,
+ DBG_BLOCK_ID_TCAA_BY4 = 0xa,
+ DBG_BLOCK_ID_MCC0_BY4 = 0xb,
+ DBG_BLOCK_ID_SX0_BY4 = 0xc,
+ DBG_BLOCK_ID_UNUSED4_BY4 = 0xd,
+ DBG_BLOCK_ID_PC0_BY4 = 0xe,
+ DBG_BLOCK_ID_UNUSED10_BY4 = 0xf,
+ DBG_BLOCK_ID_SCB0_BY4 = 0x10,
+ DBG_BLOCK_ID_SCF0_BY4 = 0x11,
+ DBG_BLOCK_ID_BCI0_BY4 = 0x12,
+ DBG_BLOCK_ID_UNUSED17_BY4 = 0x13,
+ DBG_BLOCK_ID_CB00_BY4 = 0x14,
+ DBG_BLOCK_ID_CB04_BY4 = 0x15,
+ DBG_BLOCK_ID_CB10_BY4 = 0x16,
+ DBG_BLOCK_ID_CB14_BY4 = 0x17,
+ DBG_BLOCK_ID_TCP0_BY4 = 0x18,
+ DBG_BLOCK_ID_TCP4_BY4 = 0x19,
+ DBG_BLOCK_ID_TCP8_BY4 = 0x1a,
+ DBG_BLOCK_ID_TCP12_BY4 = 0x1b,
+ DBG_BLOCK_ID_TCP16_BY4 = 0x1c,
+ DBG_BLOCK_ID_TCP20_BY4 = 0x1d,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY4 = 0x1e,
+ DBG_BLOCK_ID_TCP_RESERVED4_BY4 = 0x1f,
+ DBG_BLOCK_ID_DB_BY4 = 0x20,
+ DBG_BLOCK_ID_DB04_BY4 = 0x21,
+ DBG_BLOCK_ID_DB10_BY4 = 0x22,
+ DBG_BLOCK_ID_DB14_BY4 = 0x23,
+ DBG_BLOCK_ID_TCC0_BY4 = 0x24,
+ DBG_BLOCK_ID_TCC4_BY4 = 0x25,
+ DBG_BLOCK_ID_SPS00_BY4 = 0x26,
+ DBG_BLOCK_ID_SPS11_BY4 = 0x27,
+ DBG_BLOCK_ID_TA00_BY4 = 0x28,
+ DBG_BLOCK_ID_TA04_BY4 = 0x29,
+ DBG_BLOCK_ID_TA08_BY4 = 0x2a,
+ DBG_BLOCK_ID_UNUSED35_BY4 = 0x2b,
+ DBG_BLOCK_ID_TA10_BY4 = 0x2c,
+ DBG_BLOCK_ID_TA14_BY4 = 0x2d,
+ DBG_BLOCK_ID_TA18_BY4 = 0x2e,
+ DBG_BLOCK_ID_UNUSED39_BY4 = 0x2f,
+ DBG_BLOCK_ID_TD00_BY4 = 0x30,
+ DBG_BLOCK_ID_TD04_BY4 = 0x31,
+ DBG_BLOCK_ID_TD08_BY4 = 0x32,
+ DBG_BLOCK_ID_UNUSED43_BY4 = 0x33,
+ DBG_BLOCK_ID_TD10_BY4 = 0x34,
+ DBG_BLOCK_ID_TD14_BY4 = 0x35,
+ DBG_BLOCK_ID_TD18_BY4 = 0x36,
+ DBG_BLOCK_ID_UNUSED47_BY4 = 0x37,
+ DBG_BLOCK_ID_MCD0_BY4 = 0x38,
+ DBG_BLOCK_ID_MCD4_BY4 = 0x39,
+} DebugBlockId_BY4;
+typedef enum DebugBlockId_BY8 {
+ DBG_BLOCK_ID_RESERVED_BY8 = 0x0,
+ DBG_BLOCK_ID_CSC_BY8 = 0x1,
+ DBG_BLOCK_ID_DMA0_BY8 = 0x2,
+ DBG_BLOCK_ID_CP0_BY8 = 0x3,
+ DBG_BLOCK_ID_VGT0_BY8 = 0x4,
+ DBG_BLOCK_ID_TCAA_BY8 = 0x5,
+ DBG_BLOCK_ID_SX0_BY8 = 0x6,
+ DBG_BLOCK_ID_PC0_BY8 = 0x7,
+ DBG_BLOCK_ID_SCB0_BY8 = 0x8,
+ DBG_BLOCK_ID_BCI0_BY8 = 0x9,
+ DBG_BLOCK_ID_CB00_BY8 = 0xa,
+ DBG_BLOCK_ID_CB10_BY8 = 0xb,
+ DBG_BLOCK_ID_TCP0_BY8 = 0xc,
+ DBG_BLOCK_ID_TCP8_BY8 = 0xd,
+ DBG_BLOCK_ID_TCP16_BY8 = 0xe,
+ DBG_BLOCK_ID_TCP_RESERVED0_BY8 = 0xf,
+ DBG_BLOCK_ID_DB00_BY8 = 0x10,
+ DBG_BLOCK_ID_DB10_BY8 = 0x11,
+ DBG_BLOCK_ID_TCC0_BY8 = 0x12,
+ DBG_BLOCK_ID_SPS00_BY8 = 0x13,
+ DBG_BLOCK_ID_TA00_BY8 = 0x14,
+ DBG_BLOCK_ID_TA08_BY8 = 0x15,
+ DBG_BLOCK_ID_TA10_BY8 = 0x16,
+ DBG_BLOCK_ID_TA18_BY8 = 0x17,
+ DBG_BLOCK_ID_TD00_BY8 = 0x18,
+ DBG_BLOCK_ID_TD08_BY8 = 0x19,
+ DBG_BLOCK_ID_TD10_BY8 = 0x1a,
+ DBG_BLOCK_ID_TD18_BY8 = 0x1b,
+ DBG_BLOCK_ID_MCD0_BY8 = 0x1c,
+} DebugBlockId_BY8;
+typedef enum DebugBlockId_BY16 {
+ DBG_BLOCK_ID_RESERVED_BY16 = 0x0,
+ DBG_BLOCK_ID_DMA0_BY16 = 0x1,
+ DBG_BLOCK_ID_VGT0_BY16 = 0x2,
+ DBG_BLOCK_ID_SX0_BY16 = 0x3,
+ DBG_BLOCK_ID_SCB0_BY16 = 0x4,
+ DBG_BLOCK_ID_CB00_BY16 = 0x5,
+ DBG_BLOCK_ID_TCP0_BY16 = 0x6,
+ DBG_BLOCK_ID_TCP16_BY16 = 0x7,
+ DBG_BLOCK_ID_DB00_BY16 = 0x8,
+ DBG_BLOCK_ID_TCC0_BY16 = 0x9,
+ DBG_BLOCK_ID_TA00_BY16 = 0xa,
+ DBG_BLOCK_ID_TA10_BY16 = 0xb,
+ DBG_BLOCK_ID_TD00_BY16 = 0xc,
+ DBG_BLOCK_ID_TD10_BY16 = 0xd,
+ DBG_BLOCK_ID_MCD0_BY16 = 0xe,
+} DebugBlockId_BY16;
+typedef enum ColorTransform {
+ DCC_CT_AUTO = 0x0,
+ DCC_CT_NONE = 0x1,
+ ABGR_TO_A_BG_G_RB = 0x2,
+ BGRA_TO_BG_G_RB_A = 0x3,
+} ColorTransform;
+typedef enum CompareRef {
+ REF_NEVER = 0x0,
+ REF_LESS = 0x1,
+ REF_EQUAL = 0x2,
+ REF_LEQUAL = 0x3,
+ REF_GREATER = 0x4,
+ REF_NOTEQUAL = 0x5,
+ REF_GEQUAL = 0x6,
+ REF_ALWAYS = 0x7,
+} CompareRef;
+typedef enum ReadSize {
+ READ_256_BITS = 0x0,
+ READ_512_BITS = 0x1,
+} ReadSize;
+typedef enum DepthFormat {
+ DEPTH_INVALID = 0x0,
+ DEPTH_16 = 0x1,
+ DEPTH_X8_24 = 0x2,
+ DEPTH_8_24 = 0x3,
+ DEPTH_X8_24_FLOAT = 0x4,
+ DEPTH_8_24_FLOAT = 0x5,
+ DEPTH_32_FLOAT = 0x6,
+ DEPTH_X24_8_32_FLOAT = 0x7,
+} DepthFormat;
+typedef enum ZFormat {
+ Z_INVALID = 0x0,
+ Z_16 = 0x1,
+ Z_24 = 0x2,
+ Z_32_FLOAT = 0x3,
+} ZFormat;
+typedef enum StencilFormat {
+ STENCIL_INVALID = 0x0,
+ STENCIL_8 = 0x1,
+} StencilFormat;
+typedef enum CmaskMode {
+ CMASK_CLEAR_NONE = 0x0,
+ CMASK_CLEAR_ONE = 0x1,
+ CMASK_CLEAR_ALL = 0x2,
+ CMASK_ANY_EXPANDED = 0x3,
+ CMASK_ALPHA0_FRAG1 = 0x4,
+ CMASK_ALPHA0_FRAG2 = 0x5,
+ CMASK_ALPHA0_FRAG4 = 0x6,
+ CMASK_ALPHA0_FRAGS = 0x7,
+ CMASK_ALPHA1_FRAG1 = 0x8,
+ CMASK_ALPHA1_FRAG2 = 0x9,
+ CMASK_ALPHA1_FRAG4 = 0xa,
+ CMASK_ALPHA1_FRAGS = 0xb,
+ CMASK_ALPHAX_FRAG1 = 0xc,
+ CMASK_ALPHAX_FRAG2 = 0xd,
+ CMASK_ALPHAX_FRAG4 = 0xe,
+ CMASK_ALPHAX_FRAGS = 0xf,
+} CmaskMode;
+typedef enum QuadExportFormat {
+ EXPORT_UNUSED = 0x0,
+ EXPORT_32_R = 0x1,
+ EXPORT_32_GR = 0x2,
+ EXPORT_32_AR = 0x3,
+ EXPORT_FP16_ABGR = 0x4,
+ EXPORT_UNSIGNED16_ABGR = 0x5,
+ EXPORT_SIGNED16_ABGR = 0x6,
+ EXPORT_32_ABGR = 0x7,
+} QuadExportFormat;
+typedef enum QuadExportFormatOld {
+ EXPORT_4P_32BPC_ABGR = 0x0,
+ EXPORT_4P_16BPC_ABGR = 0x1,
+ EXPORT_4P_32BPC_GR = 0x2,
+ EXPORT_4P_32BPC_AR = 0x3,
+ EXPORT_2P_32BPC_ABGR = 0x4,
+ EXPORT_8P_32BPC_R = 0x5,
+} QuadExportFormatOld;
+typedef enum ColorFormat {
+ COLOR_INVALID = 0x0,
+ COLOR_8 = 0x1,
+ COLOR_16 = 0x2,
+ COLOR_8_8 = 0x3,
+ COLOR_32 = 0x4,
+ COLOR_16_16 = 0x5,
+ COLOR_10_11_11 = 0x6,
+ COLOR_11_11_10 = 0x7,
+ COLOR_10_10_10_2 = 0x8,
+ COLOR_2_10_10_10 = 0x9,
+ COLOR_8_8_8_8 = 0xa,
+ COLOR_32_32 = 0xb,
+ COLOR_16_16_16_16 = 0xc,
+ COLOR_RESERVED_13 = 0xd,
+ COLOR_32_32_32_32 = 0xe,
+ COLOR_RESERVED_15 = 0xf,
+ COLOR_5_6_5 = 0x10,
+ COLOR_1_5_5_5 = 0x11,
+ COLOR_5_5_5_1 = 0x12,
+ COLOR_4_4_4_4 = 0x13,
+ COLOR_8_24 = 0x14,
+ COLOR_24_8 = 0x15,
+ COLOR_X24_8_32_FLOAT = 0x16,
+ COLOR_RESERVED_23 = 0x17,
+} ColorFormat;
+typedef enum SurfaceFormat {
+ FMT_INVALID = 0x0,
+ FMT_8 = 0x1,
+ FMT_16 = 0x2,
+ FMT_8_8 = 0x3,
+ FMT_32 = 0x4,
+ FMT_16_16 = 0x5,
+ FMT_10_11_11 = 0x6,
+ FMT_11_11_10 = 0x7,
+ FMT_10_10_10_2 = 0x8,
+ FMT_2_10_10_10 = 0x9,
+ FMT_8_8_8_8 = 0xa,
+ FMT_32_32 = 0xb,
+ FMT_16_16_16_16 = 0xc,
+ FMT_32_32_32 = 0xd,
+ FMT_32_32_32_32 = 0xe,
+ FMT_RESERVED_4 = 0xf,
+ FMT_5_6_5 = 0x10,
+ FMT_1_5_5_5 = 0x11,
+ FMT_5_5_5_1 = 0x12,
+ FMT_4_4_4_4 = 0x13,
+ FMT_8_24 = 0x14,
+ FMT_24_8 = 0x15,
+ FMT_X24_8_32_FLOAT = 0x16,
+ FMT_RESERVED_33 = 0x17,
+ FMT_11_11_10_FLOAT = 0x18,
+ FMT_16_FLOAT = 0x19,
+ FMT_32_FLOAT = 0x1a,
+ FMT_16_16_FLOAT = 0x1b,
+ FMT_8_24_FLOAT = 0x1c,
+ FMT_24_8_FLOAT = 0x1d,
+ FMT_32_32_FLOAT = 0x1e,
+ FMT_10_11_11_FLOAT = 0x1f,
+ FMT_16_16_16_16_FLOAT = 0x20,
+ FMT_3_3_2 = 0x21,
+ FMT_6_5_5 = 0x22,
+ FMT_32_32_32_32_FLOAT = 0x23,
+ FMT_RESERVED_36 = 0x24,
+ FMT_1 = 0x25,
+ FMT_1_REVERSED = 0x26,
+ FMT_GB_GR = 0x27,
+ FMT_BG_RG = 0x28,
+ FMT_32_AS_8 = 0x29,
+ FMT_32_AS_8_8 = 0x2a,
+ FMT_5_9_9_9_SHAREDEXP = 0x2b,
+ FMT_8_8_8 = 0x2c,
+ FMT_16_16_16 = 0x2d,
+ FMT_16_16_16_FLOAT = 0x2e,
+ FMT_4_4 = 0x2f,
+ FMT_32_32_32_FLOAT = 0x30,
+ FMT_BC1 = 0x31,
+ FMT_BC2 = 0x32,
+ FMT_BC3 = 0x33,
+ FMT_BC4 = 0x34,
+ FMT_BC5 = 0x35,
+ FMT_BC6 = 0x36,
+ FMT_BC7 = 0x37,
+ FMT_32_AS_32_32_32_32 = 0x38,
+ FMT_APC3 = 0x39,
+ FMT_APC4 = 0x3a,
+ FMT_APC5 = 0x3b,
+ FMT_APC6 = 0x3c,
+ FMT_APC7 = 0x3d,
+ FMT_CTX1 = 0x3e,
+ FMT_RESERVED_63 = 0x3f,
+} SurfaceFormat;
+typedef enum BUF_DATA_FORMAT {
+ BUF_DATA_FORMAT_INVALID = 0x0,
+ BUF_DATA_FORMAT_8 = 0x1,
+ BUF_DATA_FORMAT_16 = 0x2,
+ BUF_DATA_FORMAT_8_8 = 0x3,
+ BUF_DATA_FORMAT_32 = 0x4,
+ BUF_DATA_FORMAT_16_16 = 0x5,
+ BUF_DATA_FORMAT_10_11_11 = 0x6,
+ BUF_DATA_FORMAT_11_11_10 = 0x7,
+ BUF_DATA_FORMAT_10_10_10_2 = 0x8,
+ BUF_DATA_FORMAT_2_10_10_10 = 0x9,
+ BUF_DATA_FORMAT_8_8_8_8 = 0xa,
+ BUF_DATA_FORMAT_32_32 = 0xb,
+ BUF_DATA_FORMAT_16_16_16_16 = 0xc,
+ BUF_DATA_FORMAT_32_32_32 = 0xd,
+ BUF_DATA_FORMAT_32_32_32_32 = 0xe,
+ BUF_DATA_FORMAT_RESERVED_15 = 0xf,
+} BUF_DATA_FORMAT;
+typedef enum IMG_DATA_FORMAT {
+ IMG_DATA_FORMAT_INVALID = 0x0,
+ IMG_DATA_FORMAT_8 = 0x1,
+ IMG_DATA_FORMAT_16 = 0x2,
+ IMG_DATA_FORMAT_8_8 = 0x3,
+ IMG_DATA_FORMAT_32 = 0x4,
+ IMG_DATA_FORMAT_16_16 = 0x5,
+ IMG_DATA_FORMAT_10_11_11 = 0x6,
+ IMG_DATA_FORMAT_11_11_10 = 0x7,
+ IMG_DATA_FORMAT_10_10_10_2 = 0x8,
+ IMG_DATA_FORMAT_2_10_10_10 = 0x9,
+ IMG_DATA_FORMAT_8_8_8_8 = 0xa,
+ IMG_DATA_FORMAT_32_32 = 0xb,
+ IMG_DATA_FORMAT_16_16_16_16 = 0xc,
+ IMG_DATA_FORMAT_32_32_32 = 0xd,
+ IMG_DATA_FORMAT_32_32_32_32 = 0xe,
+ IMG_DATA_FORMAT_RESERVED_15 = 0xf,
+ IMG_DATA_FORMAT_5_6_5 = 0x10,
+ IMG_DATA_FORMAT_1_5_5_5 = 0x11,
+ IMG_DATA_FORMAT_5_5_5_1 = 0x12,
+ IMG_DATA_FORMAT_4_4_4_4 = 0x13,
+ IMG_DATA_FORMAT_8_24 = 0x14,
+ IMG_DATA_FORMAT_24_8 = 0x15,
+ IMG_DATA_FORMAT_X24_8_32 = 0x16,
+ IMG_DATA_FORMAT_RESERVED_23 = 0x17,
+ IMG_DATA_FORMAT_RESERVED_24 = 0x18,
+ IMG_DATA_FORMAT_RESERVED_25 = 0x19,
+ IMG_DATA_FORMAT_RESERVED_26 = 0x1a,
+ IMG_DATA_FORMAT_RESERVED_27 = 0x1b,
+ IMG_DATA_FORMAT_RESERVED_28 = 0x1c,
+ IMG_DATA_FORMAT_RESERVED_29 = 0x1d,
+ IMG_DATA_FORMAT_RESERVED_30 = 0x1e,
+ IMG_DATA_FORMAT_RESERVED_31 = 0x1f,
+ IMG_DATA_FORMAT_GB_GR = 0x20,
+ IMG_DATA_FORMAT_BG_RG = 0x21,
+ IMG_DATA_FORMAT_5_9_9_9 = 0x22,
+ IMG_DATA_FORMAT_BC1 = 0x23,
+ IMG_DATA_FORMAT_BC2 = 0x24,
+ IMG_DATA_FORMAT_BC3 = 0x25,
+ IMG_DATA_FORMAT_BC4 = 0x26,
+ IMG_DATA_FORMAT_BC5 = 0x27,
+ IMG_DATA_FORMAT_BC6 = 0x28,
+ IMG_DATA_FORMAT_BC7 = 0x29,
+ IMG_DATA_FORMAT_RESERVED_42 = 0x2a,
+ IMG_DATA_FORMAT_RESERVED_43 = 0x2b,
+ IMG_DATA_FORMAT_FMASK8_S2_F1 = 0x2c,
+ IMG_DATA_FORMAT_FMASK8_S4_F1 = 0x2d,
+ IMG_DATA_FORMAT_FMASK8_S8_F1 = 0x2e,
+ IMG_DATA_FORMAT_FMASK8_S2_F2 = 0x2f,
+ IMG_DATA_FORMAT_FMASK8_S4_F2 = 0x30,
+ IMG_DATA_FORMAT_FMASK8_S4_F4 = 0x31,
+ IMG_DATA_FORMAT_FMASK16_S16_F1 = 0x32,
+ IMG_DATA_FORMAT_FMASK16_S8_F2 = 0x33,
+ IMG_DATA_FORMAT_FMASK32_S16_F2 = 0x34,
+ IMG_DATA_FORMAT_FMASK32_S8_F4 = 0x35,
+ IMG_DATA_FORMAT_FMASK32_S8_F8 = 0x36,
+ IMG_DATA_FORMAT_FMASK64_S16_F4 = 0x37,
+ IMG_DATA_FORMAT_FMASK64_S16_F8 = 0x38,
+ IMG_DATA_FORMAT_4_4 = 0x39,
+ IMG_DATA_FORMAT_6_5_5 = 0x3a,
+ IMG_DATA_FORMAT_1 = 0x3b,
+ IMG_DATA_FORMAT_1_REVERSED = 0x3c,
+ IMG_DATA_FORMAT_32_AS_8 = 0x3d,
+ IMG_DATA_FORMAT_32_AS_8_8 = 0x3e,
+ IMG_DATA_FORMAT_32_AS_32_32_32_32 = 0x3f,
+} IMG_DATA_FORMAT;
+typedef enum BUF_NUM_FORMAT {
+ BUF_NUM_FORMAT_UNORM = 0x0,
+ BUF_NUM_FORMAT_SNORM = 0x1,
+ BUF_NUM_FORMAT_USCALED = 0x2,
+ BUF_NUM_FORMAT_SSCALED = 0x3,
+ BUF_NUM_FORMAT_UINT = 0x4,
+ BUF_NUM_FORMAT_SINT = 0x5,
+ BUF_NUM_FORMAT_RESERVED_6 = 0x6,
+ BUF_NUM_FORMAT_FLOAT = 0x7,
+} BUF_NUM_FORMAT;
+typedef enum IMG_NUM_FORMAT {
+ IMG_NUM_FORMAT_UNORM = 0x0,
+ IMG_NUM_FORMAT_SNORM = 0x1,
+ IMG_NUM_FORMAT_USCALED = 0x2,
+ IMG_NUM_FORMAT_SSCALED = 0x3,
+ IMG_NUM_FORMAT_UINT = 0x4,
+ IMG_NUM_FORMAT_SINT = 0x5,
+ IMG_NUM_FORMAT_RESERVED_6 = 0x6,
+ IMG_NUM_FORMAT_FLOAT = 0x7,
+ IMG_NUM_FORMAT_RESERVED_8 = 0x8,
+ IMG_NUM_FORMAT_SRGB = 0x9,
+ IMG_NUM_FORMAT_RESERVED_10 = 0xa,
+ IMG_NUM_FORMAT_RESERVED_11 = 0xb,
+ IMG_NUM_FORMAT_RESERVED_12 = 0xc,
+ IMG_NUM_FORMAT_RESERVED_13 = 0xd,
+ IMG_NUM_FORMAT_RESERVED_14 = 0xe,
+ IMG_NUM_FORMAT_RESERVED_15 = 0xf,
+} IMG_NUM_FORMAT;
+typedef enum TileType {
+ ARRAY_COLOR_TILE = 0x0,
+ ARRAY_DEPTH_TILE = 0x1,
+} TileType;
+typedef enum NonDispTilingOrder {
+ ADDR_SURF_MICRO_TILING_DISPLAY = 0x0,
+ ADDR_SURF_MICRO_TILING_NON_DISPLAY = 0x1,
+} NonDispTilingOrder;
+typedef enum MicroTileMode {
+ ADDR_SURF_DISPLAY_MICRO_TILING = 0x0,
+ ADDR_SURF_THIN_MICRO_TILING = 0x1,
+ ADDR_SURF_DEPTH_MICRO_TILING = 0x2,
+ ADDR_SURF_ROTATED_MICRO_TILING = 0x3,
+ ADDR_SURF_THICK_MICRO_TILING = 0x4,
+} MicroTileMode;
+typedef enum TileSplit {
+ ADDR_SURF_TILE_SPLIT_64B = 0x0,
+ ADDR_SURF_TILE_SPLIT_128B = 0x1,
+ ADDR_SURF_TILE_SPLIT_256B = 0x2,
+ ADDR_SURF_TILE_SPLIT_512B = 0x3,
+ ADDR_SURF_TILE_SPLIT_1KB = 0x4,
+ ADDR_SURF_TILE_SPLIT_2KB = 0x5,
+ ADDR_SURF_TILE_SPLIT_4KB = 0x6,
+} TileSplit;
+typedef enum SampleSplit {
+ ADDR_SURF_SAMPLE_SPLIT_1 = 0x0,
+ ADDR_SURF_SAMPLE_SPLIT_2 = 0x1,
+ ADDR_SURF_SAMPLE_SPLIT_4 = 0x2,
+ ADDR_SURF_SAMPLE_SPLIT_8 = 0x3,
+} SampleSplit;
+typedef enum PipeConfig {
+ ADDR_SURF_P2 = 0x0,
+ ADDR_SURF_P2_RESERVED0 = 0x1,
+ ADDR_SURF_P2_RESERVED1 = 0x2,
+ ADDR_SURF_P2_RESERVED2 = 0x3,
+ ADDR_SURF_P4_8x16 = 0x4,
+ ADDR_SURF_P4_16x16 = 0x5,
+ ADDR_SURF_P4_16x32 = 0x6,
+ ADDR_SURF_P4_32x32 = 0x7,
+ ADDR_SURF_P8_16x16_8x16 = 0x8,
+ ADDR_SURF_P8_16x32_8x16 = 0x9,
+ ADDR_SURF_P8_32x32_8x16 = 0xa,
+ ADDR_SURF_P8_16x32_16x16 = 0xb,
+ ADDR_SURF_P8_32x32_16x16 = 0xc,
+ ADDR_SURF_P8_32x32_16x32 = 0xd,
+ ADDR_SURF_P8_32x64_32x32 = 0xe,
+ ADDR_SURF_P8_RESERVED0 = 0xf,
+ ADDR_SURF_P16_32x32_8x16 = 0x10,
+ ADDR_SURF_P16_32x32_16x16 = 0x11,
+} PipeConfig;
+typedef enum NumBanks {
+ ADDR_SURF_2_BANK = 0x0,
+ ADDR_SURF_4_BANK = 0x1,
+ ADDR_SURF_8_BANK = 0x2,
+ ADDR_SURF_16_BANK = 0x3,
+} NumBanks;
+typedef enum BankWidth {
+ ADDR_SURF_BANK_WIDTH_1 = 0x0,
+ ADDR_SURF_BANK_WIDTH_2 = 0x1,
+ ADDR_SURF_BANK_WIDTH_4 = 0x2,
+ ADDR_SURF_BANK_WIDTH_8 = 0x3,
+} BankWidth;
+typedef enum BankHeight {
+ ADDR_SURF_BANK_HEIGHT_1 = 0x0,
+ ADDR_SURF_BANK_HEIGHT_2 = 0x1,
+ ADDR_SURF_BANK_HEIGHT_4 = 0x2,
+ ADDR_SURF_BANK_HEIGHT_8 = 0x3,
+} BankHeight;
+typedef enum BankWidthHeight {
+ ADDR_SURF_BANK_WH_1 = 0x0,
+ ADDR_SURF_BANK_WH_2 = 0x1,
+ ADDR_SURF_BANK_WH_4 = 0x2,
+ ADDR_SURF_BANK_WH_8 = 0x3,
+} BankWidthHeight;
+typedef enum MacroTileAspect {
+ ADDR_SURF_MACRO_ASPECT_1 = 0x0,
+ ADDR_SURF_MACRO_ASPECT_2 = 0x1,
+ ADDR_SURF_MACRO_ASPECT_4 = 0x2,
+ ADDR_SURF_MACRO_ASPECT_8 = 0x3,
+} MacroTileAspect;
+typedef enum GATCL1RequestType {
+ GATCL1_TYPE_NORMAL = 0x0,
+ GATCL1_TYPE_SHOOTDOWN = 0x1,
+ GATCL1_TYPE_BYPASS = 0x2,
+} GATCL1RequestType;
+typedef enum TCC_CACHE_POLICIES {
+ TCC_CACHE_POLICY_LRU = 0x0,
+ TCC_CACHE_POLICY_STREAM = 0x1,
+} TCC_CACHE_POLICIES;
+typedef enum MTYPE {
+ MTYPE_NC_NV = 0x0,
+ MTYPE_NC = 0x1,
+ MTYPE_CC = 0x2,
+ MTYPE_UC = 0x3,
+} MTYPE;
+typedef enum PERFMON_COUNTER_MODE {
+ PERFMON_COUNTER_MODE_ACCUM = 0x0,
+ PERFMON_COUNTER_MODE_ACTIVE_CYCLES = 0x1,
+ PERFMON_COUNTER_MODE_MAX = 0x2,
+ PERFMON_COUNTER_MODE_DIRTY = 0x3,
+ PERFMON_COUNTER_MODE_SAMPLE = 0x4,
+ PERFMON_COUNTER_MODE_CYCLES_SINCE_FIRST_EVENT = 0x5,
+ PERFMON_COUNTER_MODE_CYCLES_SINCE_LAST_EVENT = 0x6,
+ PERFMON_COUNTER_MODE_CYCLES_GE_HI = 0x7,
+ PERFMON_COUNTER_MODE_CYCLES_EQ_HI = 0x8,
+ PERFMON_COUNTER_MODE_INACTIVE_CYCLES = 0x9,
+ PERFMON_COUNTER_MODE_RESERVED = 0xf,
+} PERFMON_COUNTER_MODE;
+typedef enum PERFMON_SPM_MODE {
+ PERFMON_SPM_MODE_OFF = 0x0,
+ PERFMON_SPM_MODE_16BIT_CLAMP = 0x1,
+ PERFMON_SPM_MODE_16BIT_NO_CLAMP = 0x2,
+ PERFMON_SPM_MODE_32BIT_CLAMP = 0x3,
+ PERFMON_SPM_MODE_32BIT_NO_CLAMP = 0x4,
+ PERFMON_SPM_MODE_RESERVED_5 = 0x5,
+ PERFMON_SPM_MODE_RESERVED_6 = 0x6,
+ PERFMON_SPM_MODE_RESERVED_7 = 0x7,
+ PERFMON_SPM_MODE_TEST_MODE_0 = 0x8,
+ PERFMON_SPM_MODE_TEST_MODE_1 = 0x9,
+ PERFMON_SPM_MODE_TEST_MODE_2 = 0xa,
+} PERFMON_SPM_MODE;
+typedef enum SurfaceTiling {
+ ARRAY_LINEAR = 0x0,
+ ARRAY_TILED = 0x1,
+} SurfaceTiling;
+typedef enum SurfaceArray {
+ ARRAY_1D = 0x0,
+ ARRAY_2D = 0x1,
+ ARRAY_3D = 0x2,
+ ARRAY_3D_SLICE = 0x3,
+} SurfaceArray;
+typedef enum ColorArray {
+ ARRAY_2D_ALT_COLOR = 0x0,
+ ARRAY_2D_COLOR = 0x1,
+ ARRAY_3D_SLICE_COLOR = 0x3,
+} ColorArray;
+typedef enum DepthArray {
+ ARRAY_2D_ALT_DEPTH = 0x0,
+ ARRAY_2D_DEPTH = 0x1,
+} DepthArray;
+typedef enum ENUM_NUM_SIMD_PER_CU {
+ NUM_SIMD_PER_CU = 0x4,
+} ENUM_NUM_SIMD_PER_CU;
+typedef enum MEM_PWR_FORCE_CTRL {
+ NO_FORCE_REQUEST = 0x0,
+ FORCE_LIGHT_SLEEP_REQUEST = 0x1,
+ FORCE_DEEP_SLEEP_REQUEST = 0x2,
+ FORCE_SHUT_DOWN_REQUEST = 0x3,
+} MEM_PWR_FORCE_CTRL;
+typedef enum MEM_PWR_FORCE_CTRL2 {
+ NO_FORCE_REQ = 0x0,
+ FORCE_LIGHT_SLEEP_REQ = 0x1,
+} MEM_PWR_FORCE_CTRL2;
+typedef enum MEM_PWR_DIS_CTRL {
+ ENABLE_MEM_PWR_CTRL = 0x0,
+ DISABLE_MEM_PWR_CTRL = 0x1,
+} MEM_PWR_DIS_CTRL;
+typedef enum MEM_PWR_SEL_CTRL {
+ DYNAMIC_SHUT_DOWN_ENABLE = 0x0,
+ DYNAMIC_DEEP_SLEEP_ENABLE = 0x1,
+ DYNAMIC_LIGHT_SLEEP_ENABLE = 0x2,
+} MEM_PWR_SEL_CTRL;
+typedef enum MEM_PWR_SEL_CTRL2 {
+ DYNAMIC_DEEP_SLEEP_EN = 0x0,
+ DYNAMIC_LIGHT_SLEEP_EN = 0x1,
+} MEM_PWR_SEL_CTRL2;
+
+#endif /* SMU_7_1_3_ENUM_H */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h
new file mode 100644
index 000000000000..1ede9e274714
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h
@@ -0,0 +1,6080 @@
+/*
+ * SMU_7_1_3 Register documentation
+ *
+ * Copyright (C) 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SMU_7_1_3_SH_MASK_H
+#define SMU_7_1_3_SH_MASK_H
+
+#define GCK_SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff
+#define GCK_SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0
+#define GCK_SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff
+#define GCK_SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0
+#define GCK_MCLK_FUSES__StartupMClkDid_MASK 0x7f
+#define GCK_MCLK_FUSES__StartupMClkDid__SHIFT 0x0
+#define GCK_MCLK_FUSES__MClkADCA_MASK 0x780
+#define GCK_MCLK_FUSES__MClkADCA__SHIFT 0x7
+#define GCK_MCLK_FUSES__MClkDDCA_MASK 0x1800
+#define GCK_MCLK_FUSES__MClkDDCA__SHIFT 0xb
+#define GCK_MCLK_FUSES__MClkDiDtWait_MASK 0xe000
+#define GCK_MCLK_FUSES__MClkDiDtWait__SHIFT 0xd
+#define GCK_MCLK_FUSES__MClkDiDtFloor_MASK 0x30000
+#define GCK_MCLK_FUSES__MClkDiDtFloor__SHIFT 0x10
+#define CG_DCLK_CNTL__DCLK_DIVIDER_MASK 0x7f
+#define CG_DCLK_CNTL__DCLK_DIVIDER__SHIFT 0x0
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK 0x100
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_EN__SHIFT 0x8
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_TOG_MASK 0x200
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_TOG__SHIFT 0x9
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00
+#define CG_DCLK_CNTL__DCLK_DIR_CNTL_DIVIDER__SHIFT 0xa
+#define CG_DCLK_STATUS__DCLK_STATUS_MASK 0x1
+#define CG_DCLK_STATUS__DCLK_STATUS__SHIFT 0x0
+#define CG_DCLK_STATUS__DCLK_DIR_CNTL_DONETOG_MASK 0x2
+#define CG_DCLK_STATUS__DCLK_DIR_CNTL_DONETOG__SHIFT 0x1
+#define CG_VCLK_CNTL__VCLK_DIVIDER_MASK 0x7f
+#define CG_VCLK_CNTL__VCLK_DIVIDER__SHIFT 0x0
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_EN_MASK 0x100
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_EN__SHIFT 0x8
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_TOG_MASK 0x200
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_TOG__SHIFT 0x9
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00
+#define CG_VCLK_CNTL__VCLK_DIR_CNTL_DIVIDER__SHIFT 0xa
+#define CG_VCLK_STATUS__VCLK_STATUS_MASK 0x1
+#define CG_VCLK_STATUS__VCLK_STATUS__SHIFT 0x0
+#define CG_VCLK_STATUS__VCLK_DIR_CNTL_DONETOG_MASK 0x2
+#define CG_VCLK_STATUS__VCLK_DIR_CNTL_DONETOG__SHIFT 0x1
+#define CG_ECLK_CNTL__ECLK_DIVIDER_MASK 0x7f
+#define CG_ECLK_CNTL__ECLK_DIVIDER__SHIFT 0x0
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK 0x100
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_EN__SHIFT 0x8
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_TOG_MASK 0x200
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_TOG__SHIFT 0x9
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_DIVIDER_MASK 0x1fc00
+#define CG_ECLK_CNTL__ECLK_DIR_CNTL_DIVIDER__SHIFT 0xa
+#define CG_ECLK_STATUS__ECLK_STATUS_MASK 0x1
+#define CG_ECLK_STATUS__ECLK_STATUS__SHIFT 0x0
+#define CG_ECLK_STATUS__ECLK_DIR_CNTL_DONETOG_MASK 0x2
+#define CG_ECLK_STATUS__ECLK_DIR_CNTL_DONETOG__SHIFT 0x1
+#define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x7f
+#define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x0
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_EN_MASK 0x100
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_EN__SHIFT 0x8
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_TOG_MASK 0x200
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_TOG__SHIFT 0x9
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_DIVIDER_MASK 0x1fc00
+#define CG_ACLK_CNTL__ACLK_DIR_CNTL_DIVIDER__SHIFT 0xa
+#define CG_MCLK_CNTL__MCLK_DIVIDER_MASK 0x7f
+#define CG_MCLK_CNTL__MCLK_DIVIDER__SHIFT 0x0
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_EN_MASK 0x100
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_EN__SHIFT 0x8
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_TOG_MASK 0x200
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_TOG__SHIFT 0x9
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00
+#define CG_MCLK_CNTL__MCLK_DIR_CNTL_DIVIDER__SHIFT 0xa
+#define CG_MCLK_STATUS__MCLK_STATUS_MASK 0x1
+#define CG_MCLK_STATUS__MCLK_STATUS__SHIFT 0x0
+#define CG_MCLK_STATUS__MCLK_DIR_CNTL_DONETOG_MASK 0x2
+#define CG_MCLK_STATUS__MCLK_DIR_CNTL_DONETOG__SHIFT 0x1
+#define GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK 0x1
+#define GCK_DFS_BYPASS_CNTL__BYPASSECLK__SHIFT 0x0
+#define GCK_DFS_BYPASS_CNTL__BYPASSLCLK_MASK 0x2
+#define GCK_DFS_BYPASS_CNTL__BYPASSLCLK__SHIFT 0x1
+#define GCK_DFS_BYPASS_CNTL__BYPASSEVCLK_MASK 0x4
+#define GCK_DFS_BYPASS_CNTL__BYPASSEVCLK__SHIFT 0x2
+#define GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK 0x8
+#define GCK_DFS_BYPASS_CNTL__BYPASSDCLK__SHIFT 0x3
+#define GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK 0x10
+#define GCK_DFS_BYPASS_CNTL__BYPASSVCLK__SHIFT 0x4
+#define GCK_DFS_BYPASS_CNTL__BYPASSDISPCLK_MASK 0x20
+#define GCK_DFS_BYPASS_CNTL__BYPASSDISPCLK__SHIFT 0x5
+#define GCK_DFS_BYPASS_CNTL__BYPASSDPREFCLK_MASK 0x40
+#define GCK_DFS_BYPASS_CNTL__BYPASSDPREFCLK__SHIFT 0x6
+#define GCK_DFS_BYPASS_CNTL__BYPASSACLK_MASK 0x80
+#define GCK_DFS_BYPASS_CNTL__BYPASSACLK__SHIFT 0x7
+#define GCK_DFS_BYPASS_CNTL__BYPASSADIVCLK_MASK 0x100
+#define GCK_DFS_BYPASS_CNTL__BYPASSADIVCLK__SHIFT 0x8
+#define GCK_DFS_BYPASS_CNTL__BYPASSPSPCLK_MASK 0x200
+#define GCK_DFS_BYPASS_CNTL__BYPASSPSPCLK__SHIFT 0x9
+#define GCK_DFS_BYPASS_CNTL__BYPASSSAMCLK_MASK 0x400
+#define GCK_DFS_BYPASS_CNTL__BYPASSSAMCLK__SHIFT 0xa
+#define GCK_DFS_BYPASS_CNTL__BYPASSSCLK_MASK 0x800
+#define GCK_DFS_BYPASS_CNTL__BYPASSSCLK__SHIFT 0xb
+#define GCK_DFS_BYPASS_CNTL__USE_SPLL_BYPASS_EN_MASK 0x1000
+#define GCK_DFS_BYPASS_CNTL__USE_SPLL_BYPASS_EN__SHIFT 0xc
+#define GCK_DFS_BYPASS_CNTL__BYPASSMCLK_MASK 0x2000
+#define GCK_DFS_BYPASS_CNTL__BYPASSMCLK__SHIFT 0xd
+#define CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK 0x1
+#define CG_SPLL_FUNC_CNTL__SPLL_RESET__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL__SPLL_PWRON_MASK 0x2
+#define CG_SPLL_FUNC_CNTL__SPLL_PWRON__SHIFT 0x1
+#define CG_SPLL_FUNC_CNTL__SPLL_DIVEN_MASK 0x4
+#define CG_SPLL_FUNC_CNTL__SPLL_DIVEN__SHIFT 0x2
+#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK 0x8
+#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN__SHIFT 0x3
+#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_THRU_DFS_MASK 0x10
+#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_THRU_DFS__SHIFT 0x4
+#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV_MASK 0x7e0
+#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV__SHIFT 0x5
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_UPDATE_MASK 0x800
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_UPDATE__SHIFT 0xb
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_EN_MASK 0x1000
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_EN__SHIFT 0xc
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_MASK 0x7f00000
+#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A__SHIFT 0x14
+#define CG_SPLL_FUNC_CNTL__SPLL_DIVA_ACK_MASK 0x8000000
+#define CG_SPLL_FUNC_CNTL__SPLL_DIVA_ACK__SHIFT 0x1b
+#define CG_SPLL_FUNC_CNTL__SPLL_OTEST_LOCK_EN_MASK 0x10000000
+#define CG_SPLL_FUNC_CNTL__SPLL_OTEST_LOCK_EN__SHIFT 0x1c
+#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL_MASK 0x1ff
+#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_MASK 0x800
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ__SHIFT 0xb
+#define CG_SPLL_FUNC_CNTL_2__SPLL_BYPASS_CHG_MASK 0x400000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_BYPASS_CHG__SHIFT 0x16
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK 0x800000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG__SHIFT 0x17
+#define CG_SPLL_FUNC_CNTL_2__SPLL_RESET_CHG_MASK 0x1000000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_RESET_CHG__SHIFT 0x18
+#define CG_SPLL_FUNC_CNTL_2__SPLL_BABY_STEP_CHG_MASK 0x2000000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_BABY_STEP_CHG__SHIFT 0x19
+#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE_MASK 0x4000000
+#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE__SHIFT 0x1a
+#define CG_SPLL_FUNC_CNTL_2__SPLL_UNLOCK_CLEAR_MASK 0x8000000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_UNLOCK_CLEAR__SHIFT 0x1b
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CLKF_UPDATE_MASK 0x10000000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_CLKF_UPDATE__SHIFT 0x1c
+#define CG_SPLL_FUNC_CNTL_2__SPLL_TEST_UNLOCK_CLR_MASK 0x40000000
+#define CG_SPLL_FUNC_CNTL_2__SPLL_TEST_UNLOCK_CLR__SHIFT 0x1e
+#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV_MASK 0x3ffffff
+#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN_MASK 0x10000000
+#define CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN__SHIFT 0x1c
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_TEST_SEL_MASK 0xf
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_TEST_SEL__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_SEL_MASK 0x60
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_SEL__SHIFT 0x5
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EN_MASK 0x180
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EN__SHIFT 0x7
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_MASK 0xe00
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE__SHIFT 0x9
+#define CG_SPLL_FUNC_CNTL_4__PCC_INC_DIV_MASK 0x7f000
+#define CG_SPLL_FUNC_CNTL_4__PCC_INC_DIV__SHIFT 0xc
+#define CG_SPLL_FUNC_CNTL_4__TEST_FRAC_BYPASS_MASK 0x200000
+#define CG_SPLL_FUNC_CNTL_4__TEST_FRAC_BYPASS__SHIFT 0x15
+#define CG_SPLL_FUNC_CNTL_4__SPLL_ILOCK_MASK 0x800000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_ILOCK__SHIFT 0x17
+#define CG_SPLL_FUNC_CNTL_4__SPLL_FBCLK_SEL_MASK 0x1000000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_FBCLK_SEL__SHIFT 0x18
+#define CG_SPLL_FUNC_CNTL_4__SPLL_VCTRLADC_EN_MASK 0x2000000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_VCTRLADC_EN__SHIFT 0x19
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_MASK 0xc000000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT__SHIFT 0x1a
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_EXT_MASK 0x70000000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_EXT__SHIFT 0x1c
+#define CG_SPLL_FUNC_CNTL_4__SPLL_VTOI_BIAS_CNTL_MASK 0x80000000
+#define CG_SPLL_FUNC_CNTL_4__SPLL_VTOI_BIAS_CNTL__SHIFT 0x1f
+#define CG_SPLL_FUNC_CNTL_5__FBDIV_SSC_BYPASS_MASK 0x1
+#define CG_SPLL_FUNC_CNTL_5__FBDIV_SSC_BYPASS__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL_5__RISEFBVCO_EN_MASK 0x2
+#define CG_SPLL_FUNC_CNTL_5__RISEFBVCO_EN__SHIFT 0x1
+#define CG_SPLL_FUNC_CNTL_5__PFD_RESET_CNTRL_MASK 0xc
+#define CG_SPLL_FUNC_CNTL_5__PFD_RESET_CNTRL__SHIFT 0x2
+#define CG_SPLL_FUNC_CNTL_5__RESET_TIMER_MASK 0x30
+#define CG_SPLL_FUNC_CNTL_5__RESET_TIMER__SHIFT 0x4
+#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_CNTRL_MASK 0xc0
+#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_CNTRL__SHIFT 0x6
+#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_EN_MASK 0x100
+#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_EN__SHIFT 0x8
+#define CG_SPLL_FUNC_CNTL_5__RESET_ANTI_MUX_MASK 0x200
+#define CG_SPLL_FUNC_CNTL_5__RESET_ANTI_MUX__SHIFT 0x9
+#define CG_SPLL_FUNC_CNTL_6__SCLKMUX0_CLKOFF_CNT_MASK 0xff
+#define CG_SPLL_FUNC_CNTL_6__SCLKMUX0_CLKOFF_CNT__SHIFT 0x0
+#define CG_SPLL_FUNC_CNTL_6__SCLKMUX1_CLKOFF_CNT_MASK 0xff00
+#define CG_SPLL_FUNC_CNTL_6__SCLKMUX1_CLKOFF_CNT__SHIFT 0x8
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_EN_MASK 0x10000
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_EN__SHIFT 0x10
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_IN_MASK 0x1e0000
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_IN__SHIFT 0x11
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_OUT_MASK 0x1e00000
+#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_OUT__SHIFT 0x15
+#define CG_SPLL_FUNC_CNTL_6__SPLL_LF_CNTR_MASK 0xfe000000
+#define CG_SPLL_FUNC_CNTL_6__SPLL_LF_CNTR__SHIFT 0x19
+#define CG_SPLL_FUNC_CNTL_7__SPLL_BW_CNTRL_MASK 0xfff
+#define CG_SPLL_FUNC_CNTL_7__SPLL_BW_CNTRL__SHIFT 0x0
+#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK 0x1
+#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL__SHIFT 0x0
+#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV_MASK 0x2
+#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV__SHIFT 0x1
+#define SPLL_CNTL_MODE__SPLL_TEST_MASK 0x4
+#define SPLL_CNTL_MODE__SPLL_TEST__SHIFT 0x2
+#define SPLL_CNTL_MODE__SPLL_FASTEN_MASK 0x8
+#define SPLL_CNTL_MODE__SPLL_FASTEN__SHIFT 0x3
+#define SPLL_CNTL_MODE__SPLL_ENSAT_MASK 0x10
+#define SPLL_CNTL_MODE__SPLL_ENSAT__SHIFT 0x4
+#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV_MASK 0xc00
+#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV__SHIFT 0xa
+#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT_MASK 0xff000
+#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT__SHIFT 0xc
+#define SPLL_CNTL_MODE__SPLL_RESET_EN_MASK 0x10000000
+#define SPLL_CNTL_MODE__SPLL_RESET_EN__SHIFT 0x1c
+#define SPLL_CNTL_MODE__SPLL_VCO_MODE_MASK 0x60000000
+#define SPLL_CNTL_MODE__SPLL_VCO_MODE__SHIFT 0x1d
+#define CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK 0x1
+#define CG_SPLL_SPREAD_SPECTRUM__SSEN__SHIFT 0x0
+#define CG_SPLL_SPREAD_SPECTRUM__CLKS_MASK 0xfff0
+#define CG_SPLL_SPREAD_SPECTRUM__CLKS__SHIFT 0x4
+#define CG_SPLL_SPREAD_SPECTRUM_2__CLKV_MASK 0x3ffffff
+#define CG_SPLL_SPREAD_SPECTRUM_2__CLKV__SHIFT 0x0
+#define MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK 0xff00
+#define MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT 0x8
+#define CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK 0x2
+#define CG_CLKPIN_CNTL__XTALIN_DIVIDE__SHIFT 0x1
+#define CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK 0x4
+#define CG_CLKPIN_CNTL__BCLK_AS_XCLK__SHIFT 0x2
+#define CG_CLKPIN_CNTL_2__ENABLE_XCLK_MASK 0x1
+#define CG_CLKPIN_CNTL_2__ENABLE_XCLK__SHIFT 0x0
+#define CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK 0x8
+#define CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN__SHIFT 0x3
+#define CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK_MASK 0x100
+#define CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK__SHIFT 0x8
+#define CG_CLKPIN_CNTL_2__XO_IN_OSCIN_EN_MASK 0x4000
+#define CG_CLKPIN_CNTL_2__XO_IN_OSCIN_EN__SHIFT 0xe
+#define CG_CLKPIN_CNTL_2__XO_IN_ICORE_CLK_OE_MASK 0x8000
+#define CG_CLKPIN_CNTL_2__XO_IN_ICORE_CLK_OE__SHIFT 0xf
+#define CG_CLKPIN_CNTL_2__XO_IN_CML_RXEN_MASK 0x10000
+#define CG_CLKPIN_CNTL_2__XO_IN_CML_RXEN__SHIFT 0x10
+#define CG_CLKPIN_CNTL_2__XO_IN_BIDIR_CML_OE_MASK 0x20000
+#define CG_CLKPIN_CNTL_2__XO_IN_BIDIR_CML_OE__SHIFT 0x11
+#define CG_CLKPIN_CNTL_2__XO_IN2_OSCIN_EN_MASK 0x40000
+#define CG_CLKPIN_CNTL_2__XO_IN2_OSCIN_EN__SHIFT 0x12
+#define CG_CLKPIN_CNTL_2__XO_IN2_ICORE_CLK_OE_MASK 0x80000
+#define CG_CLKPIN_CNTL_2__XO_IN2_ICORE_CLK_OE__SHIFT 0x13
+#define CG_CLKPIN_CNTL_2__XO_IN2_CML_RXEN_MASK 0x100000
+#define CG_CLKPIN_CNTL_2__XO_IN2_CML_RXEN__SHIFT 0x14
+#define CG_CLKPIN_CNTL_2__XO_IN2_BIDIR_CML_OE_MASK 0x200000
+#define CG_CLKPIN_CNTL_2__XO_IN2_BIDIR_CML_OE__SHIFT 0x15
+#define CG_CLKPIN_CNTL_2__CML_CTRL_MASK 0xc00000
+#define CG_CLKPIN_CNTL_2__CML_CTRL__SHIFT 0x16
+#define CG_CLKPIN_CNTL_2__CLK_SPARE_MASK 0xff000000
+#define CG_CLKPIN_CNTL_2__CLK_SPARE__SHIFT 0x18
+#define CG_CLKPIN_CNTL_DC__OSC_EN_MASK 0x1
+#define CG_CLKPIN_CNTL_DC__OSC_EN__SHIFT 0x0
+#define CG_CLKPIN_CNTL_DC__XTL_LOW_GAIN_MASK 0x6
+#define CG_CLKPIN_CNTL_DC__XTL_LOW_GAIN__SHIFT 0x1
+#define CG_CLKPIN_CNTL_DC__XTL_XOCLK_DRV_R_EN_MASK 0x200
+#define CG_CLKPIN_CNTL_DC__XTL_XOCLK_DRV_R_EN__SHIFT 0x9
+#define CG_CLKPIN_CNTL_DC__XTALIN_SEL_MASK 0x1c00
+#define CG_CLKPIN_CNTL_DC__XTALIN_SEL__SHIFT 0xa
+#define THM_CLK_CNTL__CMON_CLK_SEL_MASK 0xff
+#define THM_CLK_CNTL__CMON_CLK_SEL__SHIFT 0x0
+#define THM_CLK_CNTL__TMON_CLK_SEL_MASK 0xff00
+#define THM_CLK_CNTL__TMON_CLK_SEL__SHIFT 0x8
+#define THM_CLK_CNTL__CTF_CLK_SHUTOFF_EN_MASK 0x10000
+#define THM_CLK_CNTL__CTF_CLK_SHUTOFF_EN__SHIFT 0x10
+#define MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK 0xff
+#define MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT 0x0
+#define MISC_CLK_CTRL__ZCLK_SEL_MASK 0xff00
+#define MISC_CLK_CTRL__ZCLK_SEL__SHIFT 0x8
+#define MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL_MASK 0xff0000
+#define MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL__SHIFT 0x10
+#define GCK_PLL_TEST_CNTL__TST_SRC_SEL_MASK 0x1f
+#define GCK_PLL_TEST_CNTL__TST_SRC_SEL__SHIFT 0x0
+#define GCK_PLL_TEST_CNTL__TST_REF_SEL_MASK 0x3e0
+#define GCK_PLL_TEST_CNTL__TST_REF_SEL__SHIFT 0x5
+#define GCK_PLL_TEST_CNTL__REF_TEST_COUNT_MASK 0x1fc00
+#define GCK_PLL_TEST_CNTL__REF_TEST_COUNT__SHIFT 0xa
+#define GCK_PLL_TEST_CNTL__TST_RESET_MASK 0x20000
+#define GCK_PLL_TEST_CNTL__TST_RESET__SHIFT 0x11
+#define GCK_PLL_TEST_CNTL__TST_CLK_SEL_MODE_MASK 0x40000
+#define GCK_PLL_TEST_CNTL__TST_CLK_SEL_MODE__SHIFT 0x12
+#define GCK_PLL_TEST_CNTL_2__TEST_COUNT_MASK 0xfffe0000
+#define GCK_PLL_TEST_CNTL_2__TEST_COUNT__SHIFT 0x11
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ECLK_BYPASS_CNTL_MASK 0x7
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ECLK_BYPASS_CNTL__SHIFT 0x0
+#define GCK_ADFS_CLK_BYPASS_CNTL1__SCLK_BYPASS_CNTL_MASK 0x38
+#define GCK_ADFS_CLK_BYPASS_CNTL1__SCLK_BYPASS_CNTL__SHIFT 0x3
+#define GCK_ADFS_CLK_BYPASS_CNTL1__LCLK_BYPASS_CNTL_MASK 0x1c0
+#define GCK_ADFS_CLK_BYPASS_CNTL1__LCLK_BYPASS_CNTL__SHIFT 0x6
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DCLK_BYPASS_CNTL_MASK 0xe00
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DCLK_BYPASS_CNTL__SHIFT 0x9
+#define GCK_ADFS_CLK_BYPASS_CNTL1__VCLK_BYPASS_CNTL_MASK 0x7000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__VCLK_BYPASS_CNTL__SHIFT 0xc
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DISPCLK_BYPASS_CNTL_MASK 0x38000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DISPCLK_BYPASS_CNTL__SHIFT 0xf
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DRREFCLK_BYPASS_CNTL_MASK 0x1c0000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__DRREFCLK_BYPASS_CNTL__SHIFT 0x12
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_BYPASS_CNTL_MASK 0xe00000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_BYPASS_CNTL__SHIFT 0x15
+#define GCK_ADFS_CLK_BYPASS_CNTL1__SAMCLK_BYPASS_CNTL_MASK 0x7000000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__SAMCLK_BYPASS_CNTL__SHIFT 0x18
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_DIV_BYPASS_CNTL_MASK 0x38000000
+#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_DIV_BYPASS_CNTL__SHIFT 0x1b
+#define SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_0__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_0__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_0__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_0__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_1__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_1__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_1__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_1__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_2__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_2__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_2__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_2__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_3__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_3__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_3__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_3__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_4__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_4__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_4__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_4__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_5__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_5__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_5__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_5__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_6__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_6__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_6__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_6__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_INDEX_7__SMC_IND_ADDR_MASK 0xffffffff
+#define SMC_IND_INDEX_7__SMC_IND_ADDR__SHIFT 0x0
+#define SMC_IND_DATA_7__SMC_IND_DATA_MASK 0xffffffff
+#define SMC_IND_DATA_7__SMC_IND_DATA__SHIFT 0x0
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK 0x1
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0__SHIFT 0x0
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1_MASK 0x2
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1__SHIFT 0x1
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2_MASK 0x4
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2__SHIFT 0x2
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3_MASK 0x8
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3__SHIFT 0x3
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_4_MASK 0x10
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_4__SHIFT 0x4
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_5_MASK 0x20
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_5__SHIFT 0x5
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_6_MASK 0x40
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_6__SHIFT 0x6
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_7_MASK 0x80
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_7__SHIFT 0x7
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_8_MASK 0x100
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_8__SHIFT 0x8
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_9_MASK 0x200
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_9__SHIFT 0x9
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_10_MASK 0x400
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_10__SHIFT 0xa
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_11_MASK 0x800
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_11__SHIFT 0xb
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_12_MASK 0x1000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_12__SHIFT 0xc
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_13_MASK 0x2000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_13__SHIFT 0xd
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_14_MASK 0x4000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_14__SHIFT 0xe
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_15_MASK 0x8000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_15__SHIFT 0xf
+#define SMC_MESSAGE_0__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_0__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_0__SMC_RESP_MASK 0xffff
+#define SMC_RESP_0__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_1__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_1__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_1__SMC_RESP_MASK 0xffff
+#define SMC_RESP_1__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_2__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_2__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_2__SMC_RESP_MASK 0xffff
+#define SMC_RESP_2__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_3__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_3__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_3__SMC_RESP_MASK 0xffff
+#define SMC_RESP_3__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_4__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_4__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_4__SMC_RESP_MASK 0xffff
+#define SMC_RESP_4__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_5__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_5__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_5__SMC_RESP_MASK 0xffff
+#define SMC_RESP_5__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_6__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_6__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_6__SMC_RESP_MASK 0xffff
+#define SMC_RESP_6__SMC_RESP__SHIFT 0x0
+#define SMC_MESSAGE_7__SMC_MSG_MASK 0xffff
+#define SMC_MESSAGE_7__SMC_MSG__SHIFT 0x0
+#define SMC_RESP_7__SMC_RESP_MASK 0xffff
+#define SMC_RESP_7__SMC_RESP__SHIFT 0x0
+#define SMC_MSG_ARG_0__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_0__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_1__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_1__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_2__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_2__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_3__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_3__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_4__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_4__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_5__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_5__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_6__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_6__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MSG_ARG_7__SMC_MSG_ARG_MASK 0xffffffff
+#define SMC_MSG_ARG_7__SMC_MSG_ARG__SHIFT 0x0
+#define SMC_MESSAGE_8__SMC_MSG_MASK 0xffff