aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2011-12-20 14:53:44 -0500
committerDave Airlie <airlied@redhat.com>2011-12-20 14:53:44 -0500
commit4bc22a1aa02a0aae97a905091727345085281e61 (patch)
treeb34a37395efa5be0a9af4ba5b627e43813713ed3
parente2e022ec51a7e85ce4352a53fde1cd11d58e151a (diff)
parentb15ba51207e54245409d6f46e20dab36f906eed1 (diff)
Merge branch 'drm-radeon-next' of ../drm-radeon-next into drm-core-next
* 'drm-radeon-next' of ../drm-radeon-next: drm/radeon: introduce a sub allocator and convert ib pool to it v4 drm/radeon/kms: add support for per-ring fence interrupts drm/radeon/kms: add cayman specific fence_ring_emit drm/radeon/kms: add some new ring params to better handle other ring types drm/radeon: improve radeon_test_syncing function drm/radeon: precompute fence cpu/gpu addr once v3 drm/radeon: move ring debugfs into radeon_ring.c drm/radeon: rename struct radeon_cp to radeon_ring drm/radeon: disable compute rings on cayman for now drm/radeon: add radeon_fence_count_emited function drm/radeon: make some asic pointers per ring drm/radeon: Add radeon_test_syncing function v2 drm/radeon: make cp variable an array drm/radeon: make ring rptr and wptr register offsets variable drm/radeon: make all functions work with multiple rings. drm/radeon/kms: add support for semaphores v3 drm/radeon/kms: add support for multiple fence queues v2 drm/radeon: fix a spelling mistake drm/radeon: no need to check all relocs for duplicates drm/radeon: fix debugfs handling v3
-rw-r--r--drivers/gpu/drm/radeon/Makefile5
-rw-r--r--drivers/gpu/drm/radeon/evergreen.c234
-rw-r--r--drivers/gpu/drm/radeon/evergreen_blit_kms.c242
-rw-r--r--drivers/gpu/drm/radeon/ni.c237
-rw-r--r--drivers/gpu/drm/radeon/nid.h33
-rw-r--r--drivers/gpu/drm/radeon/r100.c227
-rw-r--r--drivers/gpu/drm/radeon/r200.c21
-rw-r--r--drivers/gpu/drm/radeon/r300.c154
-rw-r--r--drivers/gpu/drm/radeon/r420.c49
-rw-r--r--drivers/gpu/drm/radeon/r520.c25
-rw-r--r--drivers/gpu/drm/radeon/r600.c273
-rw-r--r--drivers/gpu/drm/radeon/r600_blit_kms.c221
-rw-r--r--drivers/gpu/drm/radeon/r600_cp.c2
-rw-r--r--drivers/gpu/drm/radeon/r600d.h2
-rw-r--r--drivers/gpu/drm/radeon/radeon.h221
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.c180
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.h36
-rw-r--r--drivers/gpu/drm/radeon/radeon_benchmark.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_cs.c7
-rw-r--r--drivers/gpu/drm/radeon/radeon_device.c61
-rw-r--r--drivers/gpu/drm/radeon/radeon_fence.c304
-rw-r--r--drivers/gpu/drm/radeon/radeon_gem.c6
-rw-r--r--drivers/gpu/drm/radeon/radeon_irq_kms.c24
-rw-r--r--drivers/gpu/drm/radeon/radeon_object.h20
-rw-r--r--drivers/gpu/drm/radeon/radeon_pm.c34
-rw-r--r--drivers/gpu/drm/radeon/radeon_ring.c462
-rw-r--r--drivers/gpu/drm/radeon/radeon_sa.c189
-rw-r--r--drivers/gpu/drm/radeon/radeon_semaphore.c161
-rw-r--r--drivers/gpu/drm/radeon/radeon_test.c267
-rw-r--r--drivers/gpu/drm/radeon/radeon_ttm.c8
-rw-r--r--drivers/gpu/drm/radeon/rs400.c27
-rw-r--r--drivers/gpu/drm/radeon/rs600.c34
-rw-r--r--drivers/gpu/drm/radeon/rs690.c30
-rw-r--r--drivers/gpu/drm/radeon/rv515.c106
-rw-r--r--drivers/gpu/drm/radeon/rv770.c63
35 files changed, 2629 insertions, 1338 deletions
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index cf8b4bc3e73..2139fe893ec 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -70,7 +70,8 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
70 r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \ 70 r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \
71 r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \ 71 r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \
72 evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \ 72 evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
73 radeon_trace_points.o ni.o cayman_blit_shaders.o atombios_encoders.o 73 radeon_trace_points.o ni.o cayman_blit_shaders.o atombios_encoders.o \
74 radeon_semaphore.o radeon_sa.o
74 75
75radeon-$(CONFIG_COMPAT) += radeon_ioc32.o 76radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
76radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o 77radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
@@ -78,4 +79,4 @@ radeon-$(CONFIG_ACPI) += radeon_acpi.o
78 79
79obj-$(CONFIG_DRM_RADEON)+= radeon.o 80obj-$(CONFIG_DRM_RADEON)+= radeon.o
80 81
81CFLAGS_radeon_trace_points.o := -I$(src) \ No newline at end of file 82CFLAGS_radeon_trace_points.o := -I$(src)
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 5e00d1670aa..1934728e246 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -40,6 +40,8 @@
40static void evergreen_gpu_init(struct radeon_device *rdev); 40static void evergreen_gpu_init(struct radeon_device *rdev);
41void evergreen_fini(struct radeon_device *rdev); 41void evergreen_fini(struct radeon_device *rdev);
42void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 42void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
43extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
44 int ring, u32 cp_int_cntl);
43 45
44void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 46void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
45{ 47{
@@ -1311,18 +1313,20 @@ void evergreen_mc_program(struct radeon_device *rdev)
1311 */ 1313 */
1312void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 1314void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1313{ 1315{
1316 struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
1317
1314 /* set to DX10/11 mode */ 1318 /* set to DX10/11 mode */
1315 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0)); 1319 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1316 radeon_ring_write(rdev, 1); 1320 radeon_ring_write(ring, 1);
1317 /* FIXME: implement */ 1321 /* FIXME: implement */
1318 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 1322 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1319 radeon_ring_write(rdev, 1323 radeon_ring_write(ring,
1320#ifdef __BIG_ENDIAN 1324#ifdef __BIG_ENDIAN
1321 (2 << 0) | 1325 (2 << 0) |
1322#endif 1326#endif
1323 (ib->gpu_addr & 0xFFFFFFFC)); 1327 (ib->gpu_addr & 0xFFFFFFFC));
1324 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF); 1328 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1325 radeon_ring_write(rdev, ib->length_dw); 1329 radeon_ring_write(ring, ib->length_dw);
1326} 1330}
1327 1331
1328 1332
@@ -1360,71 +1364,73 @@ static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1360 1364
1361static int evergreen_cp_start(struct radeon_device *rdev) 1365static int evergreen_cp_start(struct radeon_device *rdev)
1362{ 1366{
1367 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1363 int r, i; 1368 int r, i;
1364 uint32_t cp_me; 1369 uint32_t cp_me;
1365 1370
1366 r = radeon_ring_lock(rdev, 7); 1371 r = radeon_ring_lock(rdev, ring, 7);
1367 if (r) { 1372 if (r) {
1368 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1373 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1369 return r; 1374 return r;
1370 } 1375 }
1371 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1376 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1372 radeon_ring_write(rdev, 0x1); 1377 radeon_ring_write(ring, 0x1);
1373 radeon_ring_write(rdev, 0x0); 1378 radeon_ring_write(ring, 0x0);
1374 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1); 1379 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
1375 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1380 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1376 radeon_ring_write(rdev, 0); 1381 radeon_ring_write(ring, 0);
1377 radeon_ring_write(rdev, 0); 1382 radeon_ring_write(ring, 0);
1378 radeon_ring_unlock_commit(rdev); 1383 radeon_ring_unlock_commit(rdev, ring);
1379 1384
1380 cp_me = 0xff; 1385 cp_me = 0xff;
1381 WREG32(CP_ME_CNTL, cp_me); 1386 WREG32(CP_ME_CNTL, cp_me);
1382 1387
1383 r = radeon_ring_lock(rdev, evergreen_default_size + 19); 1388 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1384 if (r) { 1389 if (r) {
1385 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1390 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1386 return r; 1391 return r;
1387 } 1392 }
1388 1393
1389 /* setup clear context state */ 1394 /* setup clear context state */
1390 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1395 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1391 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1396 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1392 1397
1393 for (i = 0; i < evergreen_default_size; i++) 1398 for (i = 0; i < evergreen_default_size; i++)
1394 radeon_ring_write(rdev, evergreen_default_state[i]); 1399 radeon_ring_write(ring, evergreen_default_state[i]);
1395 1400
1396 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1401 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1397 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE); 1402 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1398 1403
1399 /* set clear context state */ 1404 /* set clear context state */
1400 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0)); 1405 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1401 radeon_ring_write(rdev, 0); 1406 radeon_ring_write(ring, 0);
1402 1407
1403 /* SQ_VTX_BASE_VTX_LOC */ 1408 /* SQ_VTX_BASE_VTX_LOC */
1404 radeon_ring_write(rdev, 0xc0026f00); 1409 radeon_ring_write(ring, 0xc0026f00);
1405 radeon_ring_write(rdev, 0x00000000); 1410 radeon_ring_write(ring, 0x00000000);
1406 radeon_ring_write(rdev, 0x00000000); 1411 radeon_ring_write(ring, 0x00000000);
1407 radeon_ring_write(rdev, 0x00000000); 1412 radeon_ring_write(ring, 0x00000000);
1408 1413
1409 /* Clear consts */ 1414 /* Clear consts */
1410 radeon_ring_write(rdev, 0xc0036f00); 1415 radeon_ring_write(ring, 0xc0036f00);
1411 radeon_ring_write(rdev, 0x00000bc4); 1416 radeon_ring_write(ring, 0x00000bc4);
1412 radeon_ring_write(rdev, 0xffffffff); 1417 radeon_ring_write(ring, 0xffffffff);
1413 radeon_ring_write(rdev, 0xffffffff); 1418 radeon_ring_write(ring, 0xffffffff);
1414 radeon_ring_write(rdev, 0xffffffff); 1419 radeon_ring_write(ring, 0xffffffff);
1415 1420
1416 radeon_ring_write(rdev, 0xc0026900); 1421 radeon_ring_write(ring, 0xc0026900);
1417 radeon_ring_write(rdev, 0x00000316); 1422 radeon_ring_write(ring, 0x00000316);
1418 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1423 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1419 radeon_ring_write(rdev, 0x00000010); /* */ 1424 radeon_ring_write(ring, 0x00000010); /* */
1420 1425
1421 radeon_ring_unlock_commit(rdev); 1426 radeon_ring_unlock_commit(rdev, ring);
1422 1427
1423 return 0; 1428 return 0;
1424} 1429}
1425 1430
1426int evergreen_cp_resume(struct radeon_device *rdev) 1431int evergreen_cp_resume(struct radeon_device *rdev)
1427{ 1432{
1433 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1428 u32 tmp; 1434 u32 tmp;
1429 u32 rb_bufsz; 1435 u32 rb_bufsz;
1430 int r; 1436 int r;
@@ -1442,13 +1448,13 @@ int evergreen_cp_resume(struct radeon_device *rdev)
1442 RREG32(GRBM_SOFT_RESET); 1448 RREG32(GRBM_SOFT_RESET);
1443 1449
1444 /* Set ring buffer size */ 1450 /* Set ring buffer size */
1445 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 1451 rb_bufsz = drm_order(ring->ring_size / 8);
1446 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1452 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1447#ifdef __BIG_ENDIAN 1453#ifdef __BIG_ENDIAN
1448 tmp |= BUF_SWAP_32BIT; 1454 tmp |= BUF_SWAP_32BIT;
1449#endif 1455#endif
1450 WREG32(CP_RB_CNTL, tmp); 1456 WREG32(CP_RB_CNTL, tmp);
1451 WREG32(CP_SEM_WAIT_TIMER, 0x4); 1457 WREG32(CP_SEM_WAIT_TIMER, 0x0);
1452 1458
1453 /* Set the write pointer delay */ 1459 /* Set the write pointer delay */
1454 WREG32(CP_RB_WPTR_DELAY, 0); 1460 WREG32(CP_RB_WPTR_DELAY, 0);
@@ -1456,8 +1462,8 @@ int evergreen_cp_resume(struct radeon_device *rdev)
1456 /* Initialize the ring buffer's read and write pointers */ 1462 /* Initialize the ring buffer's read and write pointers */
1457 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 1463 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1458 WREG32(CP_RB_RPTR_WR, 0); 1464 WREG32(CP_RB_RPTR_WR, 0);
1459 rdev->cp.wptr = 0; 1465 ring->wptr = 0;
1460 WREG32(CP_RB_WPTR, rdev->cp.wptr); 1466 WREG32(CP_RB_WPTR, ring->wptr);
1461 1467
1462 /* set the wb address wether it's enabled or not */ 1468 /* set the wb address wether it's enabled or not */
1463 WREG32(CP_RB_RPTR_ADDR, 1469 WREG32(CP_RB_RPTR_ADDR,
@@ -1475,16 +1481,16 @@ int evergreen_cp_resume(struct radeon_device *rdev)
1475 mdelay(1); 1481 mdelay(1);
1476 WREG32(CP_RB_CNTL, tmp); 1482 WREG32(CP_RB_CNTL, tmp);
1477 1483
1478 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8); 1484 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1479 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 1485 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1480 1486
1481 rdev->cp.rptr = RREG32(CP_RB_RPTR); 1487 ring->rptr = RREG32(CP_RB_RPTR);
1482 1488
1483 evergreen_cp_start(rdev); 1489 evergreen_cp_start(rdev);
1484 rdev->cp.ready = true; 1490 ring->ready = true;
1485 r = radeon_ring_test(rdev); 1491 r = radeon_ring_test(rdev, ring);
1486 if (r) { 1492 if (r) {
1487 rdev->cp.ready = false; 1493 ring->ready = false;
1488 return r; 1494 return r;
1489 } 1495 }
1490 return 0; 1496 return 0;
@@ -2353,7 +2359,7 @@ int evergreen_mc_init(struct radeon_device *rdev)
2353 return 0; 2359 return 0;
2354} 2360}
2355 2361
2356bool evergreen_gpu_is_lockup(struct radeon_device *rdev) 2362bool evergreen_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2357{ 2363{
2358 u32 srbm_status; 2364 u32 srbm_status;
2359 u32 grbm_status; 2365 u32 grbm_status;
@@ -2366,19 +2372,19 @@ bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
2366 grbm_status_se0 = RREG32(GRBM_STATUS_SE0); 2372 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2367 grbm_status_se1 = RREG32(GRBM_STATUS_SE1); 2373 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2368 if (!(grbm_status & GUI_ACTIVE)) { 2374 if (!(grbm_status & GUI_ACTIVE)) {
2369 r100_gpu_lockup_update(lockup, &rdev->cp); 2375 r100_gpu_lockup_update(lockup, ring);
2370 return false; 2376 return false;
2371 } 2377 }
2372 /* force CP activities */ 2378 /* force CP activities */
2373 r = radeon_ring_lock(rdev, 2); 2379 r = radeon_ring_lock(rdev, ring, 2);
2374 if (!r) { 2380 if (!r) {
2375 /* PACKET2 NOP */ 2381 /* PACKET2 NOP */
2376 radeon_ring_write(rdev, 0x80000000); 2382 radeon_ring_write(ring, 0x80000000);
2377 radeon_ring_write(rdev, 0x80000000); 2383 radeon_ring_write(ring, 0x80000000);
2378 radeon_ring_unlock_commit(rdev); 2384 radeon_ring_unlock_commit(rdev, ring);
2379 } 2385 }
2380 rdev->cp.rptr = RREG32(CP_RB_RPTR); 2386 ring->rptr = RREG32(CP_RB_RPTR);
2381 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp); 2387 return r100_gpu_cp_is_lockup(rdev, lockup, ring);
2382} 2388}
2383 2389
2384static int evergreen_gpu_soft_reset(struct radeon_device *rdev) 2390static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
@@ -2470,7 +2476,13 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2470{ 2476{
2471 u32 tmp; 2477 u32 tmp;
2472 2478
2473 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 2479 if (rdev->family >= CHIP_CAYMAN) {
2480 cayman_cp_int_cntl_setup(rdev, 0,
2481 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2482 cayman_cp_int_cntl_setup(rdev, 1, 0);
2483 cayman_cp_int_cntl_setup(rdev, 2, 0);
2484 } else
2485 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2474 WREG32(GRBM_INT_CNTL, 0); 2486 WREG32(GRBM_INT_CNTL, 0);
2475 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2487 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2476 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2488 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
@@ -2515,6 +2527,7 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2515int evergreen_irq_set(struct radeon_device *rdev) 2527int evergreen_irq_set(struct radeon_device *rdev)
2516{ 2528{
2517 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 2529 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2530 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2518 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 2531 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2519 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 2532 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2520 u32 grbm_int_cntl = 0; 2533 u32 grbm_int_cntl = 0;
@@ -2539,11 +2552,28 @@ int evergreen_irq_set(struct radeon_device *rdev)
2539 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 2552 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2540 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 2553 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2541 2554
2542 if (rdev->irq.sw_int) { 2555 if (rdev->family >= CHIP_CAYMAN) {
2543 DRM_DEBUG("evergreen_irq_set: sw int\n"); 2556 /* enable CP interrupts on all rings */
2544 cp_int_cntl |= RB_INT_ENABLE; 2557 if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
2545 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 2558 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2559 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2560 }
2561 if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP1_INDEX]) {
2562 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
2563 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
2564 }
2565 if (rdev->irq.sw_int[CAYMAN_RING_TYPE_CP2_INDEX]) {
2566 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
2567 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
2568 }
2569 } else {
2570 if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
2571 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2572 cp_int_cntl |= RB_INT_ENABLE;
2573 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2574 }
2546 } 2575 }
2576
2547 if (rdev->irq.crtc_vblank_int[0] || 2577 if (rdev->irq.crtc_vblank_int[0] ||
2548 rdev->irq.pflip[0]) { 2578 rdev->irq.pflip[0]) {
2549 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 2579 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
@@ -2603,7 +2633,12 @@ int evergreen_irq_set(struct radeon_device *rdev)
2603 grbm_int_cntl |= GUI_IDLE_INT_ENABLE; 2633 grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2604 } 2634 }
2605 2635
2606 WREG32(CP_INT_CNTL, cp_int_cntl); 2636 if (rdev->family >= CHIP_CAYMAN) {
2637 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
2638 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
2639 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
2640 } else
2641 WREG32(CP_INT_CNTL, cp_int_cntl);
2607 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 2642 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2608 2643
2609 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 2644 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
@@ -3018,11 +3053,24 @@ restart_ih:
3018 case 177: /* CP_INT in IB1 */ 3053 case 177: /* CP_INT in IB1 */
3019 case 178: /* CP_INT in IB2 */ 3054 case 178: /* CP_INT in IB2 */
3020 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 3055 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3021 radeon_fence_process(rdev); 3056 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3022 break; 3057 break;
3023 case 181: /* CP EOP event */ 3058 case 181: /* CP EOP event */
3024 DRM_DEBUG("IH: CP EOP\n"); 3059 DRM_DEBUG("IH: CP EOP\n");
3025 radeon_fence_process(rdev); 3060 if (rdev->family >= CHIP_CAYMAN) {
3061 switch (src_data) {
3062 case 0:
3063 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3064 break;
3065 case 1:
3066 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3067 break;
3068 case 2:
3069 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3070 break;
3071 }
3072 } else
3073 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3026 break; 3074 break;
3027 case 233: /* GUI IDLE */ 3075 case 233: /* GUI IDLE */
3028 DRM_DEBUG("IH: GUI idle\n"); 3076 DRM_DEBUG("IH: GUI idle\n");
@@ -3052,6 +3100,7 @@ restart_ih:
3052 3100
3053static int evergreen_startup(struct radeon_device *rdev) 3101static int evergreen_startup(struct radeon_device *rdev)
3054{ 3102{
3103 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3055 int r; 3104 int r;
3056 3105
3057 /* enable pcie gen2 link */ 3106 /* enable pcie gen2 link */
@@ -3106,6 +3155,12 @@ static int evergreen_startup(struct radeon_device *rdev)
3106 if (r) 3155 if (r)
3107 return r; 3156 return r;
3108 3157
3158 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3159 if (r) {
3160 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3161 return r;
3162 }
3163
3109 /* Enable IRQ */ 3164 /* Enable IRQ */
3110 r = r600_irq_init(rdev); 3165 r = r600_irq_init(rdev);
3111 if (r) { 3166 if (r) {
@@ -3115,7 +3170,9 @@ static int evergreen_startup(struct radeon_device *rdev)
3115 } 3170 }
3116 evergreen_irq_set(rdev); 3171 evergreen_irq_set(rdev);
3117 3172
3118 r = radeon_ring_init(rdev, rdev->cp.ring_size); 3173 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3174 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3175 0, 0xfffff, RADEON_CP_PACKET2);
3119 if (r) 3176 if (r)
3120 return r; 3177 return r;
3121 r = evergreen_cp_load_microcode(rdev); 3178 r = evergreen_cp_load_microcode(rdev);
@@ -3125,6 +3182,17 @@ static int evergreen_startup(struct radeon_device *rdev)
3125 if (r) 3182 if (r)
3126 return r; 3183 return r;
3127 3184
3185 r = radeon_ib_pool_start(rdev);
3186 if (r)
3187 return r;
3188
3189 r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
3190 if (r) {
3191 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3192 rdev->accel_working = false;
3193 return r;
3194 }
3195
3128 return 0; 3196 return 0;
3129} 3197}
3130 3198
@@ -3144,31 +3212,29 @@ int evergreen_resume(struct radeon_device *rdev)
3144 /* post card */ 3212 /* post card */
3145 atom_asic_init(rdev->mode_info.atom_context); 3213 atom_asic_init(rdev->mode_info.atom_context);
3146 3214
3215 rdev->accel_working = true;
3147 r = evergreen_startup(rdev); 3216 r = evergreen_startup(rdev);
3148 if (r) { 3217 if (r) {
3149 DRM_ERROR("evergreen startup failed on resume\n"); 3218 DRM_ERROR("evergreen startup failed on resume\n");
3150 return r; 3219 return r;
3151 } 3220 }
3152 3221
3153 r = r600_ib_test(rdev);
3154 if (r) {
3155 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3156 return r;
3157 }
3158
3159 return r; 3222 return r;
3160 3223
3161} 3224}
3162 3225
3163int evergreen_suspend(struct radeon_device *rdev) 3226int evergreen_suspend(struct radeon_device *rdev)
3164{ 3227{
3228 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3229
3165 /* FIXME: we should wait for ring to be empty */ 3230 /* FIXME: we should wait for ring to be empty */
3231 radeon_ib_pool_suspend(rdev);
3232 r600_blit_suspend(rdev);
3166 r700_cp_stop(rdev); 3233 r700_cp_stop(rdev);
3167 rdev->cp.ready = false; 3234 ring->ready = false;
3168 evergreen_irq_suspend(rdev); 3235 evergreen_irq_suspend(rdev);
3169 radeon_wb_disable(rdev); 3236 radeon_wb_disable(rdev);
3170 evergreen_pcie_gart_disable(rdev); 3237 evergreen_pcie_gart_disable(rdev);
3171 r600_blit_suspend(rdev);
3172 3238
3173 return 0; 3239 return 0;
3174} 3240}
@@ -3243,8 +3309,8 @@ int evergreen_init(struct radeon_device *rdev)
3243 if (r) 3309 if (r)
3244 return r; 3310 return r;
3245 3311
3246 rdev->cp.ring_obj = NULL; 3312 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3247 r600_ring_init(rdev, 1024 * 1024); 3313 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3248 3314
3249 rdev->ih.ring_obj = NULL; 3315 rdev->ih.ring_obj = NULL;
3250 r600_ih_ring_init(rdev, 64 * 1024); 3316 r600_ih_ring_init(rdev, 64 * 1024);
@@ -3253,29 +3319,24 @@ int evergreen_init(struct radeon_device *rdev)
3253 if (r) 3319 if (r)
3254 return r; 3320 return r;
3255 3321
3322 r = radeon_ib_pool_init(rdev);
3256 rdev->accel_working = true; 3323 rdev->accel_working = true;
3324 if (r) {
3325 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3326 rdev->accel_working = false;
3327 }
3328
3257 r = evergreen_startup(rdev); 3329 r = evergreen_startup(rdev);
3258 if (r) { 3330 if (r) {
3259 dev_err(rdev->dev, "disabling GPU acceleration\n"); 3331 dev_err(rdev->dev, "disabling GPU acceleration\n");
3260 r700_cp_fini(rdev); 3332 r700_cp_fini(rdev);
3261 r600_irq_fini(rdev); 3333 r600_irq_fini(rdev);
3262 radeon_wb_fini(rdev); 3334 radeon_wb_fini(rdev);
3335 r100_ib_fini(rdev);
3263 radeon_irq_kms_fini(rdev); 3336 radeon_irq_kms_fini(rdev);
3264 evergreen_pcie_gart_fini(rdev); 3337 evergreen_pcie_gart_fini(rdev);
3265 rdev->accel_working = false; 3338 rdev->accel_working = false;
3266 } 3339 }
3267 if (rdev->accel_working) {
3268 r = radeon_ib_pool_init(rdev);
3269 if (r) {
3270 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3271 rdev->accel_working = false;
3272 }
3273 r = r600_ib_test(rdev);
3274 if (r) {
3275 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3276 rdev->accel_working = false;
3277 }
3278 }
3279 return 0; 3340 return 0;
3280} 3341}
3281 3342
@@ -3285,11 +3346,12 @@ void evergreen_fini(struct radeon_device *rdev)
3285 r700_cp_fini(rdev); 3346 r700_cp_fini(rdev);
3286 r600_irq_fini(rdev); 3347 r600_irq_fini(rdev);
3287 radeon_wb_fini(rdev); 3348 radeon_wb_fini(rdev);
3288 radeon_ib_pool_fini(rdev); 3349 r100_ib_fini(rdev);
3289 radeon_irq_kms_fini(rdev); 3350 radeon_irq_kms_fini(rdev);
3290 evergreen_pcie_gart_fini(rdev); 3351 evergreen_pcie_gart_fini(rdev);
3291 r600_vram_scratch_fini(rdev); 3352 r600_vram_scratch_fini(rdev);
3292 radeon_gem_fini(rdev); 3353 radeon_gem_fini(rdev);
3354 radeon_semaphore_driver_fini(rdev);
3293 radeon_fence_driver_fini(rdev); 3355 radeon_fence_driver_fini(rdev);
3294 radeon_agp_fini(rdev); 3356 radeon_agp_fini(rdev);
3295 radeon_bo_fini(rdev); 3357 radeon_bo_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_kms.c b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
index 914e5af8416..2379849515c 100644
--- a/drivers/gpu/drm/radeon/evergreen_blit_kms.c
+++ b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
@@ -49,6 +49,7 @@ static void
49set_render_target(struct radeon_device *rdev, int format, 49set_render_target(struct radeon_device *rdev, int format,
50 int w, int h, u64 gpu_addr) 50 int w, int h, u64 gpu_addr)
51{ 51{
52 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
52 u32 cb_color_info; 53 u32 cb_color_info;
53 int pitch, slice; 54 int pitch, slice;
54 55
@@ -62,23 +63,23 @@ set_render_target(struct radeon_device *rdev, int format,
62 pitch = (w / 8) - 1; 63 pitch = (w / 8) - 1;
63 slice = ((w * h) / 64) - 1; 64 slice = ((w * h) / 64) - 1;
64 65
65 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 15)); 66 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 15));
66 radeon_ring_write(rdev, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_START) >> 2); 67 radeon_ring_write(ring, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_START) >> 2);
67 radeon_ring_write(rdev, gpu_addr >> 8); 68 radeon_ring_write(ring, gpu_addr >> 8);
68 radeon_ring_write(rdev, pitch); 69 radeon_ring_write(ring, pitch);
69 radeon_ring_write(rdev, slice); 70 radeon_ring_write(ring, slice);
70 radeon_ring_write(rdev, 0); 71 radeon_ring_write(ring, 0);
71 radeon_ring_write(rdev, cb_color_info); 72 radeon_ring_write(ring, cb_color_info);
72 radeon_ring_write(rdev, 0); 73 radeon_ring_write(ring, 0);
73 radeon_ring_write(rdev, (w - 1) | ((h - 1) << 16)); 74 radeon_ring_write(ring, (w - 1) | ((h - 1) << 16));
74 radeon_ring_write(rdev, 0); 75 radeon_ring_write(ring, 0);
75 radeon_ring_write(rdev, 0); 76 radeon_ring_write(ring, 0);
76 radeon_ring_write(rdev, 0); 77 radeon_ring_write(ring, 0);
77 radeon_ring_write(rdev, 0); 78 radeon_ring_write(ring, 0);
78 radeon_ring_write(rdev, 0); 79 radeon_ring_write(ring, 0);
79 radeon_ring_write(rdev, 0); 80 radeon_ring_write(ring, 0);
80 radeon_ring_write(rdev, 0); 81 radeon_ring_write(ring, 0);
81 radeon_ring_write(rdev, 0); 82 radeon_ring_write(ring, 0);
82} 83}
83 84
84/* emits 5dw */ 85/* emits 5dw */
@@ -87,6 +88,7 @@ cp_set_surface_sync(struct radeon_device *rdev,
87 u32 sync_type, u32 size, 88 u32 sync_type, u32 size,
88 u64 mc_addr) 89 u64 mc_addr)
89{ 90{
91 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
90 u32 cp_coher_size; 92 u32 cp_coher_size;
91 93
92 if (size == 0xffffffff) 94 if (size == 0xffffffff)
@@ -99,39 +101,40 @@ cp_set_surface_sync(struct radeon_device *rdev,
99 * to the RB directly. For IBs, the CP programs this as part of the 101 * to the RB directly. For IBs, the CP programs this as part of the
100 * surface_sync packet. 102 * surface_sync packet.
101 */ 103 */
102 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 104 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
103 radeon_ring_write(rdev, (0x85e8 - PACKET3_SET_CONFIG_REG_START) >> 2); 105 radeon_ring_write(ring, (0x85e8 - PACKET3_SET_CONFIG_REG_START) >> 2);
104 radeon_ring_write(rdev, 0); /* CP_COHER_CNTL2 */ 106 radeon_ring_write(ring, 0); /* CP_COHER_CNTL2 */
105 } 107 }
106 radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3)); 108 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
107 radeon_ring_write(rdev, sync_type); 109 radeon_ring_write(ring, sync_type);
108 radeon_ring_write(rdev, cp_coher_size); 110 radeon_ring_write(ring, cp_coher_size);
109 radeon_ring_write(rdev, mc_addr >> 8); 111 radeon_ring_write(ring, mc_addr >> 8);
110 radeon_ring_write(rdev, 10); /* poll interval */ 112 radeon_ring_write(ring, 10); /* poll interval */
111} 113}
112 114
113/* emits 11dw + 1 surface sync = 16dw */ 115/* emits 11dw + 1 surface sync = 16dw */
114static void 116static void
115set_shaders(struct radeon_device *rdev) 117set_shaders(struct radeon_device *rdev)
116{ 118{
119 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
117 u64 gpu_addr; 120 u64 gpu_addr;
118 121
119 /* VS */ 122 /* VS */
120 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; 123 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
121 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 3)); 124 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 3));
122 radeon_ring_write(rdev, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_START) >> 2); 125 radeon_ring_write(ring, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_START) >> 2);
123 radeon_ring_write(rdev, gpu_addr >> 8); 126 radeon_ring_write(ring, gpu_addr >> 8);
124 radeon_ring_write(rdev, 2); 127 radeon_ring_write(ring, 2);
125 radeon_ring_write(rdev, 0); 128 radeon_ring_write(ring, 0);
126 129
127 /* PS */ 130 /* PS */
128 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset; 131 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset;
129 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 4)); 132 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 4));
130 radeon_ring_write(rdev, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_START) >> 2); 133 radeon_ring_write(ring, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_START) >> 2);
131 radeon_ring_write(rdev, gpu_addr >> 8); 134 radeon_ring_write(ring, gpu_addr >> 8);
132 radeon_ring_write(rdev, 1); 135 radeon_ring_write(ring, 1);
133 radeon_ring_write(rdev, 0); 136 radeon_ring_write(ring, 0);
134 radeon_ring_write(rdev, 2); 137 radeon_ring_write(ring, 2);
135 138
136 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; 139 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
137 cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr); 140 cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr);
@@ -141,6 +144,7 @@ set_shaders(struct radeon_device *rdev)
141static void 144static void
142set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr) 145set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
143{ 146{
147 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
144 u32 sq_vtx_constant_word2, sq_vtx_constant_word3; 148 u32 sq_vtx_constant_word2, sq_vtx_constant_word3;
145 149
146 /* high addr, stride */ 150 /* high addr, stride */
@@ -155,16 +159,16 @@ set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
155 SQ_VTCX_SEL_Z(SQ_SEL_Z) | 159 SQ_VTCX_SEL_Z(SQ_SEL_Z) |
156 SQ_VTCX_SEL_W(SQ_SEL_W); 160 SQ_VTCX_SEL_W(SQ_SEL_W);
157 161
158 radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 8)); 162 radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 8));
159 radeon_ring_write(rdev, 0x580); 163 radeon_ring_write(ring, 0x580);
160 radeon_ring_write(rdev, gpu_addr & 0xffffffff); 164 radeon_ring_write(ring, gpu_addr & 0xffffffff);
161 radeon_ring_write(rdev, 48 - 1); /* size */ 165 radeon_ring_write(ring, 48 - 1); /* size */
162 radeon_ring_write(rdev, sq_vtx_constant_word2); 166 radeon_ring_write(ring, sq_vtx_constant_word2);
163 radeon_ring_write(rdev, sq_vtx_constant_word3); 167 radeon_ring_write(ring, sq_vtx_constant_word3);
164 radeon_ring_write(rdev, 0); 168 radeon_ring_write(ring, 0);
165 radeon_ring_write(rdev, 0); 169 radeon_ring_write(ring, 0);
166 radeon_ring_write(rdev, 0); 170 radeon_ring_write(ring, 0);
167 radeon_ring_write(rdev, S__SQ_CONSTANT_TYPE(SQ_TEX_VTX_VALID_BUFFER)); 171 radeon_ring_write(ring, S__SQ_CONSTANT_TYPE(SQ_TEX_VTX_VALID_BUFFER));
168 172
169 if ((rdev->family == CHIP_CEDAR) || 173 if ((rdev->family == CHIP_CEDAR) ||
170 (rdev->family == CHIP_PALM) || 174 (rdev->family == CHIP_PALM) ||
@@ -185,6 +189,7 @@ set_tex_resource(struct radeon_device *rdev,
185 int format, int w, int h, int pitch, 189 int format, int w, int h, int pitch,
186 u64 gpu_addr, u32 size) 190 u64 gpu_addr, u32 size)
187{ 191{
192 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
188 u32 sq_tex_resource_word0, sq_tex_resource_word1; 193 u32 sq_tex_resource_word0, sq_tex_resource_word1;
189 u32 sq_tex_resource_word4, sq_tex_resource_word7; 194 u32 sq_tex_resource_word4, sq_tex_resource_word7;
190 195
@@ -208,16 +213,16 @@ set_tex_resource(struct radeon_device *rdev,
208 cp_set_surface_sync(rdev, 213 cp_set_surface_sync(rdev,
209 PACKET3_TC_ACTION_ENA, size, gpu_addr); 214 PACKET3_TC_ACTION_ENA, size, gpu_addr);
210 215
211 radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 8)); 216 radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 8));
212 radeon_ring_write(rdev, 0); 217 radeon_ring_write(ring, 0);
213 radeon_ring_write(rdev, sq_tex_resource_word0); 218 radeon_ring_write(ring, sq_tex_resource_word0);
214 radeon_ring_write(rdev, sq_tex_resource_word1); 219 radeon_ring_write(ring, sq_tex_resource_word1);
215 radeon_ring_write(rdev, gpu_addr >> 8); 220 radeon_ring_write(ring, gpu_addr >> 8);
216 radeon_ring_write(rdev, gpu_addr >> 8); 221 radeon_ring_write(ring, gpu_addr >> 8);
217 radeon_ring_write(rdev, sq_tex_resource_word4); 222 radeon_ring_write(ring, sq_tex_resource_word4);
218 radeon_ring_write(rdev, 0); 223 radeon_ring_write(ring, 0);
219 radeon_ring_write(rdev, 0); 224 radeon_ring_write(ring, 0);
220 radeon_ring_write(rdev, sq_tex_resource_word7); 225 radeon_ring_write(ring, sq_tex_resource_word7);
221} 226}
222 227
223/* emits 12 */ 228/* emits 12 */
@@ -225,6 +230,7 @@ static void
225set_scissors(struct radeon_device *rdev, int x1, int y1, 230set_scissors(struct radeon_device *rdev, int x1, int y1,
226 int x2, int y2) 231 int x2, int y2)
227{ 232{
233 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
228 /* workaround some hw bugs */ 234 /* workaround some hw bugs */
229 if (x2 == 0) 235 if (x2 == 0)
230 x1 = 1; 236 x1 = 1;
@@ -235,43 +241,44 @@ set_scissors(struct radeon_device *rdev, int x1, int y1,
235 x2 = 2; 241 x2 = 2;
236 } 242 }
237 243
238 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 244 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
239 radeon_ring_write(rdev, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); 245 radeon_ring_write(ring, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
240 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16)); 246 radeon_ring_write(ring, (x1 << 0) | (y1 << 16));
241 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 247 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
242 248
243 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 249 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
244 radeon_ring_write(rdev, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); 250 radeon_ring_write(ring, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
245 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31)); 251 radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31));
246 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 252 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
247 253
248 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 254 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
249 radeon_ring_write(rdev, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2); 255 radeon_ring_write(ring, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_START) >> 2);
250 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31)); 256 radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31));
251 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 257 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
252} 258}
253 259
254/* emits 10 */ 260/* emits 10 */
255static void 261static void
256draw_auto(struct radeon_device *rdev) 262draw_auto(struct radeon_device *rdev)
257{ 263{
258 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 264 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
259 radeon_ring_write(rdev, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_START) >> 2); 265 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
260 radeon_ring_write(rdev, DI_PT_RECTLIST); 266 radeon_ring_write(ring, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_START) >> 2);
267 radeon_ring_write(ring, DI_PT_RECTLIST);
261 268
262 radeon_ring_write(rdev, PACKET3(PACKET3_INDEX_TYPE, 0)); 269 radeon_ring_write(ring, PACKET3(PACKET3_INDEX_TYPE, 0));
263 radeon_ring_write(rdev, 270 radeon_ring_write(ring,
264#ifdef __BIG_ENDIAN 271#ifdef __BIG_ENDIAN
265 (2 << 2) | 272 (2 << 2) |
266#endif 273#endif
267 DI_INDEX_SIZE_16_BIT); 274 DI_INDEX_SIZE_16_BIT);
268 275
269 radeon_ring_write(rdev, PACKET3(PACKET3_NUM_INSTANCES, 0)); 276 radeon_ring_write(ring, PACKET3(PACKET3_NUM_INSTANCES, 0));
270 radeon_ring_write(rdev, 1); 277 radeon_ring_write(ring, 1);
271 278
272 radeon_ring_write(rdev, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1)); 279 radeon_ring_write(ring, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1));
273 radeon_ring_write(rdev, 3); 280 radeon_ring_write(ring, 3);
274 radeon_ring_write(rdev, DI_SRC_SEL_AUTO_INDEX); 281 radeon_ring_write(ring, DI_SRC_SEL_AUTO_INDEX);
275 282
276} 283}
277 284
@@ -279,6 +286,7 @@ draw_auto(struct radeon_device *rdev)
279static void 286static void
280set_default_state(struct radeon_device *rdev) 287set_default_state(struct radeon_device *rdev)
281{ 288{
289 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
282 u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2, sq_gpr_resource_mgmt_3; 290 u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2, sq_gpr_resource_mgmt_3;
283 u32 sq_thread_resource_mgmt, sq_thread_resource_mgmt_2; 291 u32 sq_thread_resource_mgmt, sq_thread_resource_mgmt_2;
284 u32 sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2, sq_stack_resource_mgmt_3; 292 u32 sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2, sq_stack_resource_mgmt_3;
@@ -292,8 +300,8 @@ set_default_state(struct radeon_device *rdev)
292 int dwords; 300 int dwords;
293 301
294 /* set clear context state */ 302 /* set clear context state */
295 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0)); 303 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
296 radeon_ring_write(rdev, 0); 304 radeon_ring_write(ring, 0);
297 305
298 if (rdev->family < CHIP_CAYMAN) { 306 if (rdev->family < CHIP_CAYMAN) {
299 switch (rdev->family) { 307 switch (rdev->family) {
@@ -550,60 +558,60 @@ set_default_state(struct radeon_device *rdev)
550 NUM_LS_STACK_ENTRIES(num_ls_stack_entries)); 558 NUM_LS_STACK_ENTRIES(num_ls_stack_entries));
551 559
552 /* disable dyn gprs */ 560 /* disable dyn gprs */
553 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 561 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
554 radeon_ring_write(rdev, (SQ_DYN_GPR_CNTL_PS_FLUSH_REQ - PACKET3_SET_CONFIG_REG_START) >> 2); 562 radeon_ring_write(ring, (SQ_DYN_GPR_CNTL_PS_FLUSH_REQ - PACKET3_SET_CONFIG_REG_START) >> 2);
555 radeon_ring_write(rdev, 0); 563 radeon_ring_write(ring, 0);
556 564
557 /* setup LDS */ 565 /* setup LDS */
558 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 566 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
559 radeon_ring_write(rdev, (SQ_LDS_RESOURCE_MGMT - PACKET3_SET_CONFIG_REG_START) >> 2); 567 radeon_ring_write(ring, (SQ_LDS_RESOURCE_MGMT - PACKET3_SET_CONFIG_REG_START) >> 2);
560 radeon_ring_write(rdev, 0x10001000); 568 radeon_ring_write(ring, 0x10001000);
561 569
562 /* SQ config */ 570 /* SQ config */
563 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 11)); 571 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 11));
564 radeon_ring_write(rdev, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_START) >> 2); 572 radeon_ring_write(ring, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_START) >> 2);
565 radeon_ring_write(rdev, sq_config); 573 radeon_ring_write(ring, sq_config);
566 radeon_ring_write(rdev, sq_gpr_resource_mgmt_1); 574 radeon_ring_write(ring, sq_gpr_resource_mgmt_1);
567 radeon_ring_write(rdev, sq_gpr_resource_mgmt_2); 575 radeon_ring_write(ring, sq_gpr_resource_mgmt_2);
568 radeon_ring_write(rdev, sq_gpr_resource_mgmt_3); 576 radeon_ring_write(ring, sq_gpr_resource_mgmt_3);
569 radeon_ring_write(rdev, 0); 577 radeon_ring_write(ring, 0);
570 radeon_ring_write(rdev, 0); 578 radeon_ring_write(ring, 0);
571 radeon_ring_write(rdev, sq_thread_resource_mgmt); 579 radeon_ring_write(ring, sq_thread_resource_mgmt);
572 radeon_ring_write(rdev, sq_thread_resource_mgmt_2); 580 radeon_ring_write(ring, sq_thread_resource_mgmt_2);
573 radeon_ring_write(rdev, sq_stack_resource_mgmt_1); 581 radeon_ring_write(ring, sq_stack_resource_mgmt_1);
574 radeon_ring_write(rdev, sq_stack_resource_mgmt_2); 582 radeon_ring_write(ring, sq_stack_resource_mgmt_2);
575 radeon_ring_write(rdev, sq_stack_resource_mgmt_3); 583 radeon_ring_write(ring, sq_stack_resource_mgmt_3);
576 } 584 }
577 585
578 /* CONTEXT_CONTROL */ 586 /* CONTEXT_CONTROL */
579 radeon_ring_write(rdev, 0xc0012800); 587 radeon_ring_write(ring, 0xc0012800);
580 radeon_ring_write(rdev, 0x80000000); 588 radeon_ring_write(ring, 0x80000000);
581 radeon_ring_write(rdev, 0x80000000); 589 radeon_ring_write(ring, 0x80000000);
582 590
583 /* SQ_VTX_BASE_VTX_LOC */ 591 /* SQ_VTX_BASE_VTX_LOC */
584 radeon_ring_write(rdev, 0xc0026f00); 592 radeon_ring_write(ring, 0xc0026f00);
585 radeon_ring_write(rdev, 0x00000000); 593 radeon_ring_write(ring, 0x00000000);
586 radeon_ring_write(rdev, 0x00000000); 594 radeon_ring_write(ring, 0x00000000);
587 radeon_ring_write(rdev, 0x00000000); 595 radeon_ring_write(ring, 0x00000000);
588 596
589 /* SET_SAMPLER */ 597 /* SET_SAMPLER */
590 radeon_ring_write(rdev, 0xc0036e00); 598 radeon_ring_write(ring, 0xc0036e00);
591 radeon_ring_write(rdev, 0x00000000); 599 radeon_ring_write(ring, 0x00000000);
592 radeon_ring_write(rdev, 0x00000012); 600 radeon_ring_write(ring, 0x00000012);
593 radeon_ring_write(rdev, 0x00000000); 601 radeon_ring_write(ring, 0x00000000);
594 radeon_ring_write(rdev, 0x00000000); 602 radeon_ring_write(ring, 0x00000000);
595 603
596 /* set to DX10/11 mode */ 604 /* set to DX10/11 mode */
597 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0)); 605 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
598 radeon_ring_write(rdev, 1); 606 radeon_ring_write(ring, 1);
599 607
600 /* emit an IB pointing at default state */ 608 /* emit an IB pointing at default state */
601 dwords = ALIGN(rdev->r600_blit.state_len, 0x10); 609 dwords = ALIGN(rdev->r600_blit.state_len, 0x10);
602 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset; 610 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset;
603 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 611 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
604 radeon_ring_write(rdev, gpu_addr & 0xFFFFFFFC); 612 radeon_ring_write(ring, gpu_addr & 0xFFFFFFFC);
605 radeon_ring_write(rdev, upper_32_bits(gpu_addr) & 0xFF); 613 radeon_ring_write(ring, upper_32_bits(gpu_addr) & 0xFF);
606 radeon_ring_write(rdev, dwords); 614 radeon_ring_write(ring, dwords);
607 615
608} 616}
609 617
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 0e579985746..d89b2ebd5bb 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -1006,9 +1006,39 @@ void cayman_pcie_gart_fini(struct radeon_device *rdev)
1006 radeon_gart_fini(rdev); 1006 radeon_gart_fini(rdev);
1007} 1007}
1008 1008
1009void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1010 int ring, u32 cp_int_cntl)
1011{
1012 u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1013
1014 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1015 WREG32(CP_INT_CNTL, cp_int_cntl);
1016}
1017
1009/* 1018/*
1010 * CP. 1019 * CP.
1011 */ 1020 */
1021void cayman_fence_ring_emit(struct radeon_device *rdev,
1022 struct radeon_fence *fence)
1023{
1024 struct radeon_ring *ring = &rdev->ring[fence->ring];
1025 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1026
1027 /* flush read cache over gart */
1028 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1029 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
1030 radeon_ring_write(ring, 0xFFFFFFFF);
1031 radeon_ring_write(ring, 0);
1032 radeon_ring_write(ring, 10); /* poll interval */
1033 /* EVENT_WRITE_EOP - flush caches, send int */
1034 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1035 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1036 radeon_ring_write(ring, addr & 0xffffffff);
1037 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1038 radeon_ring_write(ring, fence->seq);
1039 radeon_ring_write(ring, 0);
1040}
1041
1012static void cayman_cp_enable(struct radeon_device *rdev, bool enable) 1042static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1013{ 1043{
1014 if (enable) 1044 if (enable)
@@ -1049,63 +1079,64 @@ static int cayman_cp_load_microcode(struct radeon_device *rdev)
1049 1079
1050static int cayman_cp_start(struct radeon_device *rdev) 1080static int cayman_cp_start(struct radeon_device *rdev)
1051{ 1081{
1082 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1052 int r, i; 1083 int r, i;
1053 1084
1054 r = radeon_ring_lock(rdev, 7); 1085 r = radeon_ring_lock(rdev, ring, 7);
1055 if (r) { 1086 if (r) {
1056 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1087 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1057 return r; 1088 return r;
1058 } 1089 }
1059 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1090 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1060 radeon_ring_write(rdev, 0x1); 1091 radeon_ring_write(ring, 0x1);
1061 radeon_ring_write(rdev, 0x0); 1092 radeon_ring_write(ring, 0x0);
1062 radeon_ring_write(rdev, rdev->config.cayman.max_hw_contexts - 1); 1093 radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1063 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1094 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1064 radeon_ring_write(rdev, 0); 1095 radeon_ring_write(ring, 0);
1065 radeon_ring_write(rdev, 0); 1096 radeon_ring_write(ring, 0);
1066 radeon_ring_unlock_commit(rdev); 1097 radeon_ring_unlock_commit(rdev, ring);
1067 1098
1068 cayman_cp_enable(rdev, true); 1099 cayman_cp_enable(rdev, true);
1069 1100
1070 r = radeon_ring_lock(rdev, cayman_default_size + 19); 1101 r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1071 if (r) { 1102 if (r) {
1072 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1103 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1073 return r; 1104 return r;
1074 } 1105 }
1075 1106
1076 /* setup clear context state */ 1107 /* setup clear context state */
1077 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1108 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1078 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1109 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1079 1110
1080 for (i = 0; i < cayman_default_size; i++) 1111 for (i = 0; i < cayman_default_size; i++)
1081 radeon_ring_write(rdev, cayman_default_state[i]); 1112 radeon_ring_write(ring, cayman_default_state[i]);
1082 1113
1083 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1114 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1084 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE); 1115 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1085 1116
1086 /* set clear context state */ 1117 /* set clear context state */
1087 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0)); 1118 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1088 radeon_ring_write(rdev, 0); 1119 radeon_ring_write(ring, 0);
1089 1120
1090 /* SQ_VTX_BASE_VTX_LOC */ 1121 /* SQ_VTX_BASE_VTX_LOC */
1091 radeon_ring_write(rdev, 0xc0026f00); 1122 radeon_ring_write(ring, 0xc0026f00);
1092 radeon_ring_write(rdev, 0x00000000); 1123 radeon_ring_write(ring, 0x00000000);
1093 radeon_ring_write(rdev, 0x00000000); 1124 radeon_ring_write(ring, 0x00000000);
1094 radeon_ring_write(rdev, 0x00000000); 1125 radeon_ring_write(ring, 0x00000000);
1095 1126
1096 /* Clear consts */ 1127 /* Clear consts */
1097 radeon_ring_write(rdev, 0xc0036f00); 1128 radeon_ring_write(ring, 0xc0036f00);
1098 radeon_ring_write(rdev, 0x00000bc4); 1129 radeon_ring_write(ring, 0x00000bc4);
1099 radeon_ring_write(rdev, 0xffffffff); 1130 radeon_ring_write(ring, 0xffffffff);
1100 radeon_ring_write(rdev, 0xffffffff); 1131 radeon_ring_write(ring, 0xffffffff);
1101 radeon_ring_write(rdev, 0xffffffff); 1132 radeon_ring_write(ring, 0xffffffff);
1102 1133
1103 radeon_ring_write(rdev, 0xc0026900); 1134 radeon_ring_write(ring, 0xc0026900);
1104 radeon_ring_write(rdev, 0x00000316); 1135 radeon_ring_write(ring, 0x00000316);
1105 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1136 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1106 radeon_ring_write(rdev, 0x00000010); /* */ 1137 radeon_ring_write(ring, 0x00000010); /* */
1107 1138
1108 radeon_ring_unlock_commit(rdev); 1139 radeon_ring_unlock_commit(rdev, ring);
1109 1140
1110 /* XXX init other rings */ 1141 /* XXX init other rings */
1111 1142
@@ -1115,11 +1146,12 @@ static int cayman_cp_start(struct radeon_device *rdev)
1115static void cayman_cp_fini(struct radeon_device *rdev) 1146static void cayman_cp_fini(struct radeon_device *rdev)
1116{ 1147{
1117 cayman_cp_enable(rdev, false); 1148 cayman_cp_enable(rdev, false);
1118 radeon_ring_fini(rdev); 1149 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1119} 1150}
1120 1151
1121int cayman_cp_resume(struct radeon_device *rdev) 1152int cayman_cp_resume(struct radeon_device *rdev)
1122{ 1153{
1154 struct radeon_ring *ring;
1123 u32 tmp; 1155 u32 tmp;
1124 u32 rb_bufsz; 1156 u32 rb_bufsz;
1125 int r; 1157 int r;
@@ -1136,7 +1168,7 @@ int cayman_cp_resume(struct radeon_device *rdev)
1136 WREG32(GRBM_SOFT_RESET, 0); 1168 WREG32(GRBM_SOFT_RESET, 0);
1137 RREG32(GRBM_SOFT_RESET); 1169 RREG32(GRBM_SOFT_RESET);
1138 1170
1139 WREG32(CP_SEM_WAIT_TIMER, 0x4); 1171 WREG32(CP_SEM_WAIT_TIMER, 0x0);
1140 1172
1141 /* Set the write pointer delay */ 1173 /* Set the write pointer delay */
1142 WREG32(CP_RB_WPTR_DELAY, 0); 1174 WREG32(CP_RB_WPTR_DELAY, 0);
@@ -1145,7 +1177,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
1145 1177
1146 /* ring 0 - compute and gfx */ 1178 /* ring 0 - compute and gfx */
1147 /* Set ring buffer size */ 1179 /* Set ring buffer size */
1148 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 1180 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1181 rb_bufsz = drm_order(ring->ring_size / 8);
1149 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1182 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1150#ifdef __BIG_ENDIAN 1183#ifdef __BIG_ENDIAN
1151 tmp |= BUF_SWAP_32BIT; 1184 tmp |= BUF_SWAP_32BIT;
@@ -1154,8 +1187,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
1154 1187
1155 /* Initialize the ring buffer's read and write pointers */ 1188 /* Initialize the ring buffer's read and write pointers */
1156 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 1189 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
1157 rdev->cp.wptr = 0; 1190 ring->wptr = 0;
1158 WREG32(CP_RB0_WPTR, rdev->cp.wptr); 1191 WREG32(CP_RB0_WPTR, ring->wptr);
1159 1192
1160 /* set the wb address wether it's enabled or not */ 1193 /* set the wb address wether it's enabled or not */
1161 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 1194 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1172,13 +1205,14 @@ int cayman_cp_resume(struct radeon_device *rdev)
1172 mdelay(1); 1205 mdelay(1);
1173 WREG32(CP_RB0_CNTL, tmp); 1206 WREG32(CP_RB0_CNTL, tmp);
1174 1207
1175 WREG32(CP_RB0_BASE, rdev->cp.gpu_addr >> 8); 1208 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
1176 1209
1177 rdev->cp.rptr = RREG32(CP_RB0_RPTR); 1210 ring->rptr = RREG32(CP_RB0_RPTR);
1178 1211
1179 /* ring1 - compute only */ 1212 /* ring1 - compute only */
1180 /* Set ring buffer size */ 1213 /* Set ring buffer size */
1181 rb_bufsz = drm_order(rdev->cp1.ring_size / 8); 1214 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
1215 rb_bufsz = drm_order(ring->ring_size / 8);
1182 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1216 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1183#ifdef __BIG_ENDIAN 1217#ifdef __BIG_ENDIAN
1184 tmp |= BUF_SWAP_32BIT; 1218 tmp |= BUF_SWAP_32BIT;
@@ -1187,8 +1221,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
1187 1221
1188 /* Initialize the ring buffer's read and write pointers */ 1222 /* Initialize the ring buffer's read and write pointers */
1189 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 1223 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
1190 rdev->cp1.wptr = 0; 1224 ring->wptr = 0;
1191 WREG32(CP_RB1_WPTR, rdev->cp1.wptr); 1225 WREG32(CP_RB1_WPTR, ring->wptr);
1192 1226
1193 /* set the wb address wether it's enabled or not */ 1227 /* set the wb address wether it's enabled or not */
1194 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 1228 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1197,13 +1231,14 @@ int cayman_cp_resume(struct radeon_device *rdev)
1197 mdelay(1); 1231 mdelay(1);
1198 WREG32(CP_RB1_CNTL, tmp); 1232 WREG32(CP_RB1_CNTL, tmp);
1199 1233
1200 WREG32(CP_RB1_BASE, rdev->cp1.gpu_addr >> 8); 1234 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
1201 1235
1202 rdev->cp1.rptr = RREG32(CP_RB1_RPTR); 1236 ring->rptr = RREG32(CP_RB1_RPTR);
1203 1237
1204 /* ring2 - compute only */ 1238 /* ring2 - compute only */
1205 /* Set ring buffer size */ 1239 /* Set ring buffer size */
1206 rb_bufsz = drm_order(rdev->cp2.ring_size / 8); 1240 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
1241 rb_bufsz = drm_order(ring->ring_size / 8);
1207 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1242 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1208#ifdef __BIG_ENDIAN 1243#ifdef __BIG_ENDIAN
1209 tmp |= BUF_SWAP_32BIT; 1244 tmp |= BUF_SWAP_32BIT;
@@ -1212,8 +1247,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
1212 1247
1213 /* Initialize the ring buffer's read and write pointers */ 1248 /* Initialize the ring buffer's read and write pointers */
1214 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 1249 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
1215 rdev->cp2.wptr = 0; 1250 ring->wptr = 0;
1216 WREG32(CP_RB2_WPTR, rdev->cp2.wptr); 1251 WREG32(CP_RB2_WPTR, ring->wptr);
1217 1252
1218 /* set the wb address wether it's enabled or not */ 1253 /* set the wb address wether it's enabled or not */
1219 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 1254 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1222,28 +1257,28 @@ int cayman_cp_resume(struct radeon_device *rdev)
1222 mdelay(1); 1257 mdelay(1);
1223 WREG32(CP_RB2_CNTL, tmp); 1258 WREG32(CP_RB2_CNTL, tmp);
1224 1259
1225 WREG32(CP_RB2_BASE, rdev->cp2.gpu_addr >> 8); 1260 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
1226 1261
1227 rdev->cp2.rptr = RREG32(CP_RB2_RPTR); 1262 ring->rptr = RREG32(CP_RB2_RPTR);
1228 1263
1229 /* start the rings */ 1264 /* start the rings */
1230 cayman_cp_start(rdev); 1265 cayman_cp_start(rdev);
1231 rdev->cp.ready = true; 1266 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1232 rdev->cp1.ready = true; 1267 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1233 rdev->cp2.ready = true; 1268 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1234 /* this only test cp0 */ 1269 /* this only test cp0 */
1235 r = radeon_ring_test(rdev); 1270 r = radeon_ring_test(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1236 if (r) { 1271 if (r) {
1237 rdev->cp.ready = false; 1272 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1238 rdev->cp1.ready = false; 1273 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1239 rdev->cp2.ready = false; 1274 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1240 return r; 1275 return r;
1241 } 1276 }
1242 1277
1243 return 0; 1278 return 0;
1244} 1279}
1245 1280
1246bool cayman_gpu_is_lockup(struct radeon_device *rdev) 1281bool cayman_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1247{ 1282{
1248 u32 srbm_status; 1283 u32 srbm_status;
1249 u32 grbm_status; 1284 u32 grbm_status;
@@ -1256,20 +1291,20 @@ bool cayman_gpu_is_lockup(struct radeon_device *rdev)
1256 grbm_status_se0 = RREG32(GRBM_STATUS_SE0); 1291 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
1257 grbm_status_se1 = RREG32(GRBM_STATUS_SE1); 1292 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
1258 if (!(grbm_status & GUI_ACTIVE)) { 1293 if (!(grbm_status & GUI_ACTIVE)) {
1259 r100_gpu_lockup_update(lockup, &rdev->cp); 1294 r100_gpu_lockup_update(lockup, ring);
1260 return false; 1295 return false;
1261 } 1296 }
1262 /* force CP activities */ 1297 /* force CP activities */
1263 r = radeon_ring_lock(rdev, 2); 1298 r = radeon_ring_lock(rdev, ring, 2);
1264 if (!r) { 1299 if (!r) {
1265 /* PACKET2 NOP */ 1300 /* PACKET2 NOP */
1266 radeon_ring_write(rdev, 0x80000000); 1301 radeon_ring_write(ring, 0x80000000);
1267 radeon_ring_write(rdev, 0x80000000); 1302 radeon_ring_write(ring, 0x80000000);
1268 radeon_ring_unlock_commit(rdev); 1303 radeon_ring_unlock_commit(rdev, ring);
1269 } 1304 }
1270 /* XXX deal with CP0,1,2 */ 1305 /* XXX deal with CP0,1,2 */
1271 rdev->cp.rptr = RREG32(CP_RB0_RPTR); 1306 ring->rptr = RREG32(ring->rptr_reg);
1272 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp); 1307 return r100_gpu_cp_is_lockup(rdev, lockup, ring);
1273} 1308}
1274 1309
1275static int cayman_gpu_soft_reset(struct radeon_device *rdev) 1310static int cayman_gpu_soft_reset(struct radeon_device *rdev)
@@ -1338,6 +1373,7 @@ int cayman_asic_reset(struct radeon_device *rdev)
1338 1373
1339static int cayman_startup(struct radeon_device *rdev) 1374static int cayman_startup(struct radeon_device *rdev)
1340{ 1375{
1376 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1341 int r; 1377 int r;
1342 1378
1343 /* enable pcie gen2 link */ 1379 /* enable pcie gen2 link */
@@ -1378,6 +1414,24 @@ static int cayman_startup(struct radeon_device *rdev)
1378 if (r) 1414 if (r)
1379 return r; 1415 return r;
1380 1416
1417 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1418 if (r) {
1419 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1420 return r;
1421 }
1422
1423 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
1424 if (r) {
1425 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1426 return r;
1427 }
1428
1429 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
1430 if (r) {
1431 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1432 return r;
1433 }
1434
1381 /* Enable IRQ */ 1435 /* Enable IRQ */
1382 r = r600_irq_init(rdev); 1436 r = r600_irq_init(rdev);
1383 if (r) { 1437 if (r) {
@@ -1387,7 +1441,9 @@ static int cayman_startup(struct radeon_device *rdev)
1387 } 1441 }
1388 evergreen_irq_set(rdev); 1442 evergreen_irq_set(rdev);
1389 1443
1390 r = radeon_ring_init(rdev, rdev->cp.ring_size); 1444 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1445 CP_RB0_RPTR, CP_RB0_WPTR,
1446 0, 0xfffff, RADEON_CP_PACKET2);
1391 if (r) 1447 if (r)
1392 return r; 1448 return r;
1393 r = cayman_cp_load_microcode(rdev); 1449 r = cayman_cp_load_microcode(rdev);
@@ -1397,6 +1453,17 @@ static int cayman_startup(struct radeon_device *rdev)
1397 if (r) 1453 if (r)
1398 return r; 1454 return r;
1399 1455
1456 r = radeon_ib_pool_start(rdev);
1457 if (r)
1458 return r;
1459
1460 r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
1461 if (r) {
1462 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
1463 rdev->accel_working = false;
1464 return r;
1465 }
1466
1400 return 0; 1467 return 0;
1401} 1468}
1402 1469
@@ -1411,32 +1478,25 @@ int cayman_resume(struct radeon_device *rdev)
1411 /* post card */ 1478 /* post card */
1412 atom_asic_init(rdev->mode_info.atom_context); 1479 atom_asic_init(rdev->mode_info.atom_context);
1413 1480
1481 rdev->accel_working = true;
1414 r = cayman_startup(rdev); 1482 r = cayman_startup(rdev);
1415 if (r) { 1483 if (r) {
1416 DRM_ERROR("cayman startup failed on resume\n"); 1484 DRM_ERROR("cayman startup failed on resume\n");
1417 return r; 1485 return r;
1418 } 1486 }
1419
1420 r = r600_ib_test(rdev);
1421 if (r) {
1422 DRM_ERROR("radeon: failled testing IB (%d).\n", r);
1423 return r;
1424 }
1425
1426 return r; 1487 return r;
1427
1428} 1488}
1429 1489
1430int cayman_suspend(struct radeon_device *rdev) 1490int cayman_suspend(struct radeon_device *rdev)
1431{ 1491{
1432 /* FIXME: we should wait for ring to be empty */ 1492 /* FIXME: we should wait for ring to be empty */
1493 radeon_ib_pool_suspend(rdev);
1494 r600_blit_suspend(rdev);
1433 cayman_cp_enable(rdev, false); 1495 cayman_cp_enable(rdev, false);
1434 rdev->cp.ready = false; 1496 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1435 evergreen_irq_suspend(rdev); 1497 evergreen_irq_suspend(rdev);
1436 radeon_wb_disable(rdev); 1498 radeon_wb_disable(rdev);
1437 cayman_pcie_gart_disable(rdev); 1499 cayman_pcie_gart_disable(rdev);
1438 r600_blit_suspend(rdev);
1439
1440 return 0; 1500 return 0;
1441} 1501}
1442 1502
@@ -1448,6 +1508,7 @@ int cayman_suspend(struct radeon_device *rdev)
1448 */ 1508 */
1449int cayman_init(struct radeon_device *rdev) 1509int cayman_init(struct radeon_device *rdev)
1450{ 1510{
1511 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1451 int r; 1512 int r;
1452 1513
1453 /* This don't do much */ 1514 /* This don't do much */
@@ -1500,8 +1561,8 @@ int cayman_init(struct radeon_device *rdev)
1500 if (r) 1561 if (r)
1501 return r; 1562 return r;
1502 1563
1503 rdev->cp.ring_obj = NULL; 1564 ring->ring_obj = NULL;
1504 r600_ring_init(rdev, 1024 * 1024); 1565 r600_ring_init(rdev, ring, 1024 * 1024);
1505 1566
1506 rdev->ih.ring_obj = NULL; 1567 rdev->ih.ring_obj = NULL;
1507 r600_ih_ring_init(rdev, 64 * 1024); 1568 r600_ih_ring_init(rdev, 64 * 1024);
@@ -1510,29 +1571,24 @@ int cayman_init(struct radeon_device *rdev)
1510 if (r) 1571 if (r)
1511 return r; 1572 return r;
1512 1573
1574 r = radeon_ib_pool_init(rdev);
1513 rdev->accel_working = true; 1575 rdev->accel_working = true;
1576 if (r) {
1577 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1578 rdev->accel_working = false;
1579 }
1580
1514 r = cayman_startup(rdev); 1581 r = cayman_startup(rdev);
1515 if (r) { 1582 if (r) {
1516 dev_err(rdev->dev, "disabling GPU acceleration\n"); 1583 dev_err(rdev->dev, "disabling GPU acceleration\n");
1517 cayman_cp_fini(rdev); 1584 cayman_cp_fini(rdev);
1518 r600_irq_fini(rdev); 1585 r600_irq_fini(rdev);
1519 radeon_wb_fini(rdev); 1586 radeon_wb_fini(rdev);
1587 r100_ib_fini(rdev);
1520 radeon_irq_kms_fini(rdev); 1588 radeon_irq_kms_fini(rdev);
1521 cayman_pcie_gart_fini(rdev); 1589 cayman_pcie_gart_fini(rdev);
1522 rdev->accel_working = false; 1590 rdev->accel_working = false;
1523 } 1591 }
1524 if (rdev->accel_working) {
1525 r = radeon_ib_pool_init(rdev);
1526 if (r) {
1527 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
1528 rdev->accel_working = false;
1529 }
1530 r = r600_ib_test(rdev);
1531 if (r) {
1532 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
1533 rdev->accel_working = false;
1534 }
1535 }
1536 1592
1537 /* Don't start up if the MC ucode is missing. 1593 /* Don't start up if the MC ucode is missing.
1538 * The default clocks and voltages before the MC ucode 1594 * The default clocks and voltages before the MC ucode
@@ -1552,11 +1608,12 @@ void cayman_fini(struct radeon_device *rdev)
1552 cayman_cp_fini(rdev); 1608 cayman_cp_fini(rdev);
1553 r600_irq_fini(rdev); 1609 r600_irq_fini(rdev);
1554 radeon_wb_fini(rdev); 1610 radeon_wb_fini(rdev);
1555 radeon_ib_pool_fini(rdev); 1611 r100_ib_fini(rdev);
1556 radeon_irq_kms_fini(rdev); 1612 radeon_irq_kms_fini(rdev);
1557 cayman_pcie_gart_fini(rdev); 1613 cayman_pcie_gart_fini(rdev);
1558 r600_vram_scratch_fini(rdev); 1614 r600_vram_scratch_fini(rdev);
1559 radeon_gem_fini(rdev); 1615 radeon_gem_fini(rdev);
1616 radeon_semaphore_driver_fini(rdev);
1560 radeon_fence_driver_fini(rdev); 1617 radeon_fence_driver_fini(rdev);
1561 radeon_bo_fini(rdev); 1618 radeon_bo_fini(rdev);
1562 radeon_atombios_fini(rdev); 1619 radeon_atombios_fini(rdev);
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index 4672869cdb2..0d3f52cff2f 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -42,6 +42,9 @@
42#define CAYMAN_MAX_TCC_MASK 0xFF 42#define CAYMAN_MAX_TCC_MASK 0xFF
43 43
44#define DMIF_ADDR_CONFIG 0xBD4 44#define DMIF_ADDR_CONFIG 0xBD4
45#define SRBM_GFX_CNTL 0x0E44
46#define RINGID(x) (((x) & 0x3) << 0)
47#define VMID(x) (((x) & 0x7) << 0)
45#define SRBM_STATUS 0x0E50 48#define SRBM_STATUS 0x0E50
46 49
47#define VM_CONTEXT0_REQUEST_RESPONSE 0x1470 50#define VM_CONTEXT0_REQUEST_RESPONSE 0x1470
@@ -394,6 +397,12 @@
394#define CP_RB0_RPTR_ADDR 0xC10C 397#define CP_RB0_RPTR_ADDR 0xC10C
395#define CP_RB0_RPTR_ADDR_HI 0xC110 398#define CP_RB0_RPTR_ADDR_HI 0xC110
396#define CP_RB0_WPTR 0xC114 399#define CP_RB0_WPTR 0xC114
400
401#define CP_INT_CNTL 0xC124
402# define CNTX_BUSY_INT_ENABLE (1 << 19)
403# define CNTX_EMPTY_INT_ENABLE (1 << 20)
404# define TIME_STAMP_INT_ENABLE (1 << 26)
405
397#define CP_RB1_BASE 0xC180 406#define CP_RB1_BASE 0xC180
398#define CP_RB1_CNTL 0xC184 407#define CP_RB1_CNTL 0xC184
399#define CP_RB1_RPTR_ADDR 0xC188 408#define CP_RB1_RPTR_ADDR 0xC188
@@ -411,6 +420,10 @@
411#define CP_ME_RAM_DATA 0xC160 420#define CP_ME_RAM_DATA 0xC160
412#define CP_DEBUG 0xC1FC 421#define CP_DEBUG 0xC1FC
413 422
423#define VGT_EVENT_INITIATOR 0x28a90
424# define CACHE_FLUSH_AND_INV_EVENT_TS (0x14 << 0)
425# define CACHE_FLUSH_AND_INV_EVENT (0x16 << 0)
426
414/* 427/*
415 * PM4 428 * PM4
416 */ 429 */
@@ -494,7 +507,27 @@
494#define PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16) 507#define PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
495#define PACKET3_COND_WRITE 0x45 508#define PACKET3_COND_WRITE 0x45
496#define PACKET3_EVENT_WRITE 0x46 509#define PACKET3_EVENT_WRITE 0x46
510#define EVENT_TYPE(x) ((x) << 0)
511#define EVENT_INDEX(x) ((x) << 8)
512 /* 0 - any non-TS event
513 * 1 - ZPASS_DONE
514 * 2 - SAMPLE_PIPELINESTAT
515 * 3 - SAMPLE_STREAMOUTSTAT*
516 * 4 - *S_PARTIAL_FLUSH
517 * 5 - TS events
518 */
497#define PACKET3_EVENT_WRITE_EOP 0x47 519#define PACKET3_EVENT_WRITE_EOP 0x47
520#define DATA_SEL(x) ((x) << 29)
521 /* 0 - discard
522 * 1 - send low 32bit data
523 * 2 - send 64bit data
524 * 3 - send 64bit counter value
525 */
526#define INT_SEL(x) ((x) << 24)
527 /* 0 - none
528 * 1 - interrupt only (DATA_SEL = 0)
529 * 2 - interrupt when data write is confirmed
530 */
498#define PACKET3_EVENT_WRITE_EOS 0x48 531#define PACKET3_EVENT_WRITE_EOS 0x48
499#define PACKET3_PREAMBLE_CNTL 0x4A 532#define PACKET3_PREAMBLE_CNTL 0x4A
500# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28) 533# define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE (2 << 28)
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index bfc08f6320f..657040b15b0 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -667,7 +667,7 @@ int r100_irq_set(struct radeon_device *rdev)
667 WREG32(R_000040_GEN_INT_CNTL, 0); 667 WREG32(R_000040_GEN_INT_CNTL, 0);
668 return -EINVAL; 668 return -EINVAL;
669 } 669 }
670 if (rdev->irq.sw_int) { 670 if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
671 tmp |= RADEON_SW_INT_ENABLE; 671 tmp |= RADEON_SW_INT_ENABLE;
672 } 672 }
673 if (rdev->irq.gui_idle) { 673 if (rdev->irq.gui_idle) {
@@ -739,7 +739,7 @@ int r100_irq_process(struct radeon_device *rdev)
739 while (status) { 739 while (status) {
740 /* SW interrupt */ 740 /* SW interrupt */
741 if (status & RADEON_SW_INT_TEST) { 741 if (status & RADEON_SW_INT_TEST) {
742 radeon_fence_process(rdev); 742 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
743 } 743 }
744 /* gui idle interrupt */ 744 /* gui idle interrupt */
745 if (status & RADEON_GUI_IDLE_STAT) { 745 if (status & RADEON_GUI_IDLE_STAT) {
@@ -811,25 +811,36 @@ u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc)
811void r100_fence_ring_emit(struct radeon_device *rdev, 811void r100_fence_ring_emit(struct radeon_device *rdev,
812 struct radeon_fence *fence) 812 struct radeon_fence *fence)
813{ 813{
814 struct radeon_ring *ring = &rdev->ring[fence->ring];
815
814 /* We have to make sure that caches are flushed before 816 /* We have to make sure that caches are flushed before
815 * CPU might read something from VRAM. */ 817 * CPU might read something from VRAM. */
816 radeon_ring_write(rdev, PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0)); 818 radeon_ring_write(ring, PACKET0(RADEON_RB3D_DSTCACHE_CTLSTAT, 0));
817 radeon_ring_write(rdev, RADEON_RB3D_DC_FLUSH_ALL); 819 radeon_ring_write(ring, RADEON_RB3D_DC_FLUSH_ALL);
818 radeon_ring_write(rdev, PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0)); 820 radeon_ring_write(ring, PACKET0(RADEON_RB3D_ZCACHE_CTLSTAT, 0));
819 radeon_ring_write(rdev, RADEON_RB3D_ZC_FLUSH_ALL); 821 radeon_ring_write(ring, RADEON_RB3D_ZC_FLUSH_ALL);
820 /* Wait until IDLE & CLEAN */ 822 /* Wait until IDLE & CLEAN */
821 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 823 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
822 radeon_ring_write(rdev, RADEON_WAIT_2D_IDLECLEAN | RADEON_WAIT_3D_IDLECLEAN); 824 radeon_ring_write(ring, RADEON_WAIT_2D_IDLECLEAN | RADEON_WAIT_3D_IDLECLEAN);
823 radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); 825 radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0));
824 radeon_ring_write(rdev, rdev->config.r100.hdp_cntl | 826 radeon_ring_write(ring, rdev->config.r100.hdp_cntl |
825 RADEON_HDP_READ_BUFFER_INVALIDATE); 827 RADEON_HDP_READ_BUFFER_INVALIDATE);
826 radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); 828 radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0));
827 radeon_ring_write(rdev, rdev->config.r100.hdp_cntl); 829 radeon_ring_write(ring, rdev->config.r100.hdp_cntl);
828 /* Emit fence sequence & fire IRQ */ 830 /* Emit fence sequence & fire IRQ */
829 radeon_ring_write(rdev, PACKET0(rdev->fence_drv.scratch_reg, 0)); 831 radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0));
830 radeon_ring_write(rdev, fence->seq); 832 radeon_ring_write(ring, fence->seq);
831 radeon_ring_write(rdev, PACKET0(RADEON_GEN_INT_STATUS, 0)); 833 radeon_ring_write(ring, PACKET0(RADEON_GEN_INT_STATUS, 0));
832 radeon_ring_write(rdev, RADEON_SW_INT_FIRE); 834 radeon_ring_write(ring, RADEON_SW_INT_FIRE);
835}
836
837void r100_semaphore_ring_emit(struct radeon_device *rdev,
838 struct radeon_ring *ring,
839 struct radeon_semaphore *semaphore,
840 bool emit_wait)
841{
842 /* Unused on older asics, since we don't have semaphores or multiple rings */
843 BUG();
833} 844}
834 845
835int r100_copy_blit(struct radeon_device *rdev, 846int r100_copy_blit(struct radeon_device *rdev,
@@ -838,6 +849,7 @@ int r100_copy_blit(struct radeon_device *rdev,
838 unsigned num_gpu_pages, 849 unsigned num_gpu_pages,
839 struct radeon_fence *fence) 850 struct radeon_fence *fence)
840{ 851{
852 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
841 uint32_t cur_pages; 853 uint32_t cur_pages;
842 uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE; 854 uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE;
843 uint32_t pitch; 855 uint32_t pitch;
@@ -855,7 +867,7 @@ int r100_copy_blit(struct radeon_device *rdev,
855 867
856 /* Ask for enough room for blit + flush + fence */ 868 /* Ask for enough room for blit + flush + fence */
857 ndw = 64 + (10 * num_loops); 869 ndw = 64 + (10 * num_loops);
858 r = radeon_ring_lock(rdev, ndw); 870 r = radeon_ring_lock(rdev, ring, ndw);
859 if (r) { 871 if (r) {
860 DRM_ERROR("radeon: moving bo (%d) asking for %u dw.\n", r, ndw); 872 DRM_ERROR("radeon: moving bo (%d) asking for %u dw.\n", r, ndw);
861 return -EINVAL; 873 return -EINVAL;
@@ -869,8 +881,8 @@ int r100_copy_blit(struct radeon_device *rdev,
869 881
870 /* pages are in Y direction - height 882 /* pages are in Y direction - height
871 page width in X direction - width */ 883 page width in X direction - width */
872 radeon_ring_write(rdev, PACKET3(PACKET3_BITBLT_MULTI, 8)); 884 radeon_ring_write(ring, PACKET3(PACKET3_BITBLT_MULTI, 8));
873 radeon_ring_write(rdev, 885 radeon_ring_write(ring,
874 RADEON_GMC_SRC_PITCH_OFFSET_CNTL | 886 RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
875 RADEON_GMC_DST_PITCH_OFFSET_CNTL | 887 RADEON_GMC_DST_PITCH_OFFSET_CNTL |
876 RADEON_GMC_SRC_CLIPPING | 888 RADEON_GMC_SRC_CLIPPING |
@@ -882,26 +894,26 @@ int r100_copy_blit(struct radeon_device *rdev,
882 RADEON_DP_SRC_SOURCE_MEMORY | 894 RADEON_DP_SRC_SOURCE_MEMORY |
883 RADEON_GMC_CLR_CMP_CNTL_DIS | 895 RADEON_GMC_CLR_CMP_CNTL_DIS |
884 RADEON_GMC_WR_MSK_DIS); 896 RADEON_GMC_WR_MSK_DIS);
885 radeon_ring_write(rdev, (pitch << 22) | (src_offset >> 10)); 897 radeon_ring_write(ring, (pitch << 22) | (src_offset >> 10));
886 radeon_ring_write(rdev, (pitch << 22) | (dst_offset >> 10)); 898 radeon_ring_write(ring, (pitch << 22) | (dst_offset >> 10));
887 radeon_ring_write(rdev, (0x1fff) | (0x1fff << 16)); 899 radeon_ring_write(ring, (0x1fff) | (0x1fff << 16));
888 radeon_ring_write(rdev, 0); 900 radeon_ring_write(ring, 0);
889 radeon_ring_write(rdev, (0x1fff) | (0x1fff << 16)); 901 radeon_ring_write(ring, (0x1fff) | (0x1fff << 16));
890 radeon_ring_write(rdev, num_gpu_pages); 902 radeon_ring_write(ring, num_gpu_pages);
891 radeon_ring_write(rdev, num_gpu_pages); 903 radeon_ring_write(ring, num_gpu_pages);
892 radeon_ring_write(rdev, cur_pages | (stride_pixels << 16)); 904 radeon_ring_write(ring, cur_pages | (stride_pixels << 16));
893 } 905 }
894 radeon_ring_write(rdev, PACKET0(RADEON_DSTCACHE_CTLSTAT, 0)); 906 radeon_ring_write(ring, PACKET0(RADEON_DSTCACHE_CTLSTAT, 0));
895 radeon_ring_write(rdev, RADEON_RB2D_DC_FLUSH_ALL); 907 radeon_ring_write(ring, RADEON_RB2D_DC_FLUSH_ALL);
896 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 908 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
897 radeon_ring_write(rdev, 909 radeon_ring_write(ring,
898 RADEON_WAIT_2D_IDLECLEAN | 910 RADEON_WAIT_2D_IDLECLEAN |
899 RADEON_WAIT_HOST_IDLECLEAN | 911 RADEON_WAIT_HOST_IDLECLEAN |
900 RADEON_WAIT_DMA_GUI_IDLE); 912 RADEON_WAIT_DMA_GUI_IDLE);
901 if (fence) { 913 if (fence) {
902 r = radeon_fence_emit(rdev, fence); 914 r = radeon_fence_emit(rdev, fence);
903 } 915 }
904 radeon_ring_unlock_commit(rdev); 916 radeon_ring_unlock_commit(rdev, ring);
905 return r; 917 return r;
906} 918}
907 919
@@ -922,19 +934,20 @@ static int r100_cp_wait_for_idle(struct radeon_device *rdev)
922 934
923void r100_ring_start(struct radeon_device *rdev) 935void r100_ring_start(struct radeon_device *rdev)
924{ 936{
937 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
925 int r; 938 int r;
926 939
927 r = radeon_ring_lock(rdev, 2); 940 r = radeon_ring_lock(rdev, ring, 2);
928 if (r) { 941 if (r) {
929 return; 942 return;
930 } 943 }
931 radeon_ring_write(rdev, PACKET0(RADEON_ISYNC_CNTL, 0)); 944 radeon_ring_write(ring, PACKET0(RADEON_ISYNC_CNTL, 0));
932 radeon_ring_write(rdev, 945 radeon_ring_write(ring,
933 RADEON_ISYNC_ANY2D_IDLE3D | 946 RADEON_ISYNC_ANY2D_IDLE3D |
934 RADEON_ISYNC_ANY3D_IDLE2D | 947 RADEON_ISYNC_ANY3D_IDLE2D |
935 RADEON_ISYNC_WAIT_IDLEGUI | 948 RADEON_ISYNC_WAIT_IDLEGUI |
936 RADEON_ISYNC_CPSCRATCH_IDLEGUI); 949 RADEON_ISYNC_CPSCRATCH_IDLEGUI);
937 radeon_ring_unlock_commit(rdev); 950 radeon_ring_unlock_commit(rdev, ring);
938} 951}
939 952
940 953
@@ -1035,6 +1048,7 @@ static void r100_cp_load_microcode(struct radeon_device *rdev)
1035 1048
1036int r100_cp_init(struct radeon_device *rdev, unsigned ring_size) 1049int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1037{ 1050{
1051 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1038 unsigned rb_bufsz; 1052 unsigned rb_bufsz;
1039 unsigned rb_blksz; 1053 unsigned rb_blksz;
1040 unsigned max_fetch; 1054 unsigned max_fetch;
@@ -1060,7 +1074,9 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1060 rb_bufsz = drm_order(ring_size / 8); 1074 rb_bufsz = drm_order(ring_size / 8);
1061 ring_size = (1 << (rb_bufsz + 1)) * 4; 1075 ring_size = (1 << (rb_bufsz + 1)) * 4;
1062 r100_cp_load_microcode(rdev); 1076 r100_cp_load_microcode(rdev);
1063 r = radeon_ring_init(rdev, ring_size); 1077 r = radeon_ring_init(rdev, ring, ring_size, RADEON_WB_CP_RPTR_OFFSET,
1078 RADEON_CP_RB_RPTR, RADEON_CP_RB_WPTR,
1079 0, 0x7fffff, RADEON_CP_PACKET2);
1064 if (r) { 1080 if (r) {
1065 return r; 1081 return r;
1066 } 1082 }
@@ -1069,7 +1085,7 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1069 rb_blksz = 9; 1085 rb_blksz = 9;
1070 /* cp will read 128bytes at a time (4 dwords) */ 1086 /* cp will read 128bytes at a time (4 dwords) */
1071 max_fetch = 1; 1087 max_fetch = 1;
1072 rdev->cp.align_mask = 16 - 1; 1088 ring->align_mask = 16 - 1;
1073 /* Write to CP_RB_WPTR will be delayed for pre_write_timer clocks */ 1089 /* Write to CP_RB_WPTR will be delayed for pre_write_timer clocks */
1074 pre_write_timer = 64; 1090 pre_write_timer = 64;
1075 /* Force CP_RB_WPTR write if written more than one time before the 1091 /* Force CP_RB_WPTR write if written more than one time before the
@@ -1099,13 +1115,13 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1099 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_NO_UPDATE); 1115 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_NO_UPDATE);
1100 1116
1101 /* Set ring address */ 1117 /* Set ring address */
1102 DRM_INFO("radeon: ring at 0x%016lX\n", (unsigned long)rdev->cp.gpu_addr); 1118 DRM_INFO("radeon: ring at 0x%016lX\n", (unsigned long)ring->gpu_addr);
1103 WREG32(RADEON_CP_RB_BASE, rdev->cp.gpu_addr); 1119 WREG32(RADEON_CP_RB_BASE, ring->gpu_addr);
1104 /* Force read & write ptr to 0 */ 1120 /* Force read & write ptr to 0 */
1105 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA | RADEON_RB_NO_UPDATE); 1121 WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA | RADEON_RB_NO_UPDATE);
1106 WREG32(RADEON_CP_RB_RPTR_WR, 0); 1122 WREG32(RADEON_CP_RB_RPTR_WR, 0);
1107 rdev->cp.wptr = 0; 1123 ring->wptr = 0;
1108 WREG32(RADEON_CP_RB_WPTR, rdev->cp.wptr); 1124 WREG32(RADEON_CP_RB_WPTR, ring->wptr);
1109 1125
1110 /* set the wb address whether it's enabled or not */ 1126 /* set the wb address whether it's enabled or not */
1111 WREG32(R_00070C_CP_RB_RPTR_ADDR, 1127 WREG32(R_00070C_CP_RB_RPTR_ADDR,
@@ -1121,7 +1137,7 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1121 1137
1122 WREG32(RADEON_CP_RB_CNTL, tmp); 1138 WREG32(RADEON_CP_RB_CNTL, tmp);
1123 udelay(10); 1139 udelay(10);
1124 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 1140 ring->rptr = RREG32(RADEON_CP_RB_RPTR);
1125 /* Set cp mode to bus mastering & enable cp*/ 1141 /* Set cp mode to bus mastering & enable cp*/
1126 WREG32(RADEON_CP_CSQ_MODE, 1142 WREG32(RADEON_CP_CSQ_MODE,
1127 REG_SET(RADEON_INDIRECT2_START, indirect2_start) | 1143 REG_SET(RADEON_INDIRECT2_START, indirect2_start) |
@@ -1130,12 +1146,12 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
1130 WREG32(RADEON_CP_CSQ_MODE, 0x00004D4D); 1146 WREG32(RADEON_CP_CSQ_MODE, 0x00004D4D);
1131 WREG32(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM); 1147 WREG32(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIBM_INDBM);
1132 radeon_ring_start(rdev); 1148 radeon_ring_start(rdev);
1133 r = radeon_ring_test(rdev); 1149 r = radeon_ring_test(rdev, ring);
1134 if (r) { 1150 if (r) {
1135 DRM_ERROR("radeon: cp isn't working (%d).\n", r); 1151 DRM_ERROR("radeon: cp isn't working (%d).\n", r);
1136 return r; 1152 return r;
1137 } 1153 }
1138 rdev->cp.ready = true; 1154 ring->ready = true;
1139 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); 1155 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1140 return 0; 1156 return 0;
1141} 1157}
@@ -1147,7 +1163,7 @@ void r100_cp_fini(struct radeon_device *rdev)
1147 } 1163 }
1148 /* Disable ring */ 1164 /* Disable ring */
1149 r100_cp_disable(rdev); 1165 r100_cp_disable(rdev);
1150 radeon_ring_fini(rdev); 1166 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1151 DRM_INFO("radeon: cp finalized\n"); 1167 DRM_INFO("radeon: cp finalized\n");
1152} 1168}
1153 1169
@@ -1155,7 +1171,7 @@ void r100_cp_disable(struct radeon_device *rdev)
1155{ 1171{
1156 /* Disable ring */ 1172 /* Disable ring */
1157 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1173 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1158 rdev->cp.ready = false; 1174 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1159 WREG32(RADEON_CP_CSQ_MODE, 0); 1175 WREG32(RADEON_CP_CSQ_MODE, 0);
1160 WREG32(RADEON_CP_CSQ_CNTL, 0); 1176 WREG32(RADEON_CP_CSQ_CNTL, 0);
1161 WREG32(R_000770_SCRATCH_UMSK, 0); 1177 WREG32(R_000770_SCRATCH_UMSK, 0);
@@ -1165,13 +1181,6 @@ void r100_cp_disable(struct radeon_device *rdev)
1165 } 1181 }
1166} 1182}
1167 1183
1168void r100_cp_commit(struct radeon_device *rdev)
1169{
1170 WREG32(RADEON_CP_RB_WPTR, rdev->cp.wptr);
1171 (void)RREG32(RADEON_CP_RB_WPTR);
1172}
1173
1174
1175/* 1184/*
1176 * CS functions 1185 * CS functions
1177 */ 1186 */
@@ -2099,9 +2108,9 @@ int r100_mc_wait_for_idle(struct radeon_device *rdev)
2099 return -1; 2108 return -1;
2100} 2109}
2101 2110
2102void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_cp *cp) 2111void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_ring *ring)
2103{ 2112{
2104 lockup->last_cp_rptr = cp->rptr; 2113 lockup->last_cp_rptr = ring->rptr;
2105 lockup->last_jiffies = jiffies; 2114 lockup->last_jiffies = jiffies;
2106} 2115}
2107 2116
@@ -2126,20 +2135,20 @@ void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_cp *cp
2126 * false positive when CP is just gived nothing to do. 2135 * false positive when CP is just gived nothing to do.
2127 * 2136 *
2128 **/ 2137 **/
2129bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *lockup, struct radeon_cp *cp) 2138bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *lockup, struct radeon_ring *ring)
2130{ 2139{
2131 unsigned long cjiffies, elapsed; 2140 unsigned long cjiffies, elapsed;
2132 2141
2133 cjiffies = jiffies; 2142 cjiffies = jiffies;
2134 if (!time_after(cjiffies, lockup->last_jiffies)) { 2143 if (!time_after(cjiffies, lockup->last_jiffies)) {
2135 /* likely a wrap around */ 2144 /* likely a wrap around */
2136 lockup->last_cp_rptr = cp->rptr; 2145 lockup->last_cp_rptr = ring->rptr;
2137 lockup->last_jiffies = jiffies; 2146 lockup->last_jiffies = jiffies;
2138 return false; 2147 return false;
2139 } 2148 }
2140 if (cp->rptr != lockup->last_cp_rptr) { 2149 if (ring->rptr != lockup->last_cp_rptr) {
2141 /* CP is still working no lockup */ 2150 /* CP is still working no lockup */
2142 lockup->last_cp_rptr = cp->rptr; 2151 lockup->last_cp_rptr = ring->rptr;
2143 lockup->last_jiffies = jiffies; 2152 lockup->last_jiffies = jiffies;
2144 return false; 2153 return false;
2145 } 2154 }
@@ -2152,26 +2161,26 @@ bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *l
2152 return false; 2161 return false;
2153} 2162}
2154 2163
2155bool r100_gpu_is_lockup(struct radeon_device *rdev) 2164bool r100_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2156{ 2165{
2157 u32 rbbm_status; 2166 u32 rbbm_status;
2158 int r; 2167 int r;
2159 2168
2160 rbbm_status = RREG32(R_000E40_RBBM_STATUS); 2169 rbbm_status = RREG32(R_000E40_RBBM_STATUS);
2161 if (!G_000E40_GUI_ACTIVE(rbbm_status)) { 2170 if (!G_000E40_GUI_ACTIVE(rbbm_status)) {
2162 r100_gpu_lockup_update(&rdev->config.r100.lockup, &rdev->cp); 2171 r100_gpu_lockup_update(&rdev->config.r100.lockup, ring);
2163 return false; 2172 return false;
2164 } 2173 }
2165 /* force CP activities */ 2174 /* force CP activities */
2166 r = radeon_ring_lock(rdev, 2); 2175 r = radeon_ring_lock(rdev, ring, 2);
2167 if (!r) { 2176 if (!r) {
2168 /* PACKET2 NOP */ 2177 /* PACKET2 NOP */
2169 radeon_ring_write(rdev, 0x80000000); 2178 radeon_ring_write(ring, 0x80000000);
2170 radeon_ring_write(rdev, 0x80000000); 2179 radeon_ring_write(ring, 0x80000000);
2171 radeon_ring_unlock_commit(rdev); 2180 radeon_ring_unlock_commit(rdev, ring);
2172 } 2181 }
2173 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 2182 ring->rptr = RREG32(ring->rptr_reg);
2174 return r100_gpu_cp_is_lockup(rdev, &rdev->config.r100.lockup, &rdev->cp); 2183 return r100_gpu_cp_is_lockup(rdev, &rdev->config.r100.lockup, ring);
2175} 2184}
2176 2185
2177void r100_bm_disable(struct radeon_device *rdev) 2186void r100_bm_disable(struct radeon_device *rdev)
@@ -2579,21 +2588,22 @@ static int r100_debugfs_cp_ring_info(struct seq_file *m, void *data)
2579 struct drm_info_node *node = (struct drm_info_node *) m->private; 2588 struct drm_info_node *node = (struct drm_info_node *) m->private;
2580 struct drm_device *dev = node->minor->dev; 2589 struct drm_device *dev = node->minor->dev;
2581 struct radeon_device *rdev = dev->dev_private; 2590 struct radeon_device *rdev = dev->dev_private;
2591 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2582 uint32_t rdp, wdp; 2592 uint32_t rdp, wdp;
2583 unsigned count, i, j; 2593 unsigned count, i, j;
2584 2594
2585 radeon_ring_free_size(rdev); 2595 radeon_ring_free_size(rdev, ring);
2586 rdp = RREG32(RADEON_CP_RB_RPTR); 2596 rdp = RREG32(RADEON_CP_RB_RPTR);
2587 wdp = RREG32(RADEON_CP_RB_WPTR); 2597 wdp = RREG32(RADEON_CP_RB_WPTR);
2588 count = (rdp + rdev->cp.ring_size - wdp) & rdev->cp.ptr_mask; 2598 count = (rdp + ring->ring_size - wdp) & ring->ptr_mask;
2589 seq_printf(m, "CP_STAT 0x%08x\n", RREG32(RADEON_CP_STAT)); 2599 seq_printf(m, "CP_STAT 0x%08x\n", RREG32(RADEON_CP_STAT));
2590 seq_printf(m, "CP_RB_WPTR 0x%08x\n", wdp); 2600 seq_printf(m, "CP_RB_WPTR 0x%08x\n", wdp);
2591 seq_printf(m, "CP_RB_RPTR 0x%08x\n", rdp); 2601 seq_printf(m, "CP_RB_RPTR 0x%08x\n", rdp);
2592 seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw); 2602 seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
2593 seq_printf(m, "%u dwords in ring\n", count); 2603 seq_printf(m, "%u dwords in ring\n", count);
2594 for (j = 0; j <= count; j++) { 2604 for (j = 0; j <= count; j++) {
2595 i = (rdp + j) & rdev->cp.ptr_mask; 2605 i = (rdp + j) & ring->ptr_mask;
2596 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]); 2606 seq_printf(m, "r[%04d]=0x%08x\n", i, ring->ring[i]);
2597 } 2607 }
2598 return 0; 2608 return 0;
2599} 2609}
@@ -3635,7 +3645,7 @@ void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track
3635 } 3645 }
3636} 3646}
3637 3647
3638int r100_ring_test(struct radeon_device *rdev) 3648int r100_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3639{ 3649{
3640 uint32_t scratch; 3650 uint32_t scratch;
3641 uint32_t tmp = 0; 3651 uint32_t tmp = 0;
@@ -3648,15 +3658,15 @@ int r100_ring_test(struct radeon_device *rdev)
3648 return r; 3658 return r;
3649 } 3659 }
3650 WREG32(scratch, 0xCAFEDEAD); 3660 WREG32(scratch, 0xCAFEDEAD);
3651 r = radeon_ring_lock(rdev, 2); 3661 r = radeon_ring_lock(rdev, ring, 2);
3652 if (r) { 3662 if (r) {
3653 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3663 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3654 radeon_scratch_free(rdev, scratch); 3664 radeon_scratch_free(rdev, scratch);
3655 return r; 3665 return r;
3656 } 3666 }
3657 radeon_ring_write(rdev, PACKET0(scratch, 0)); 3667 radeon_ring_write(ring, PACKET0(scratch, 0));
3658 radeon_ring_write(rdev, 0xDEADBEEF); 3668 radeon_ring_write(ring, 0xDEADBEEF);
3659 radeon_ring_unlock_commit(rdev); 3669 radeon_ring_unlock_commit(rdev, ring);
3660 for (i = 0; i < rdev->usec_timeout; i++) { 3670 for (i = 0; i < rdev->usec_timeout; i++) {
3661 tmp = RREG32(scratch); 3671 tmp = RREG32(scratch);
3662 if (tmp == 0xDEADBEEF) { 3672 if (tmp == 0xDEADBEEF) {
@@ -3677,9 +3687,11 @@ int r100_ring_test(struct radeon_device *rdev)
3677 3687
3678void r100_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3688void r100_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3679{ 3689{
3680 radeon_ring_write(rdev, PACKET0(RADEON_CP_IB_BASE, 1)); 3690 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3681 radeon_ring_write(rdev, ib->gpu_addr); 3691
3682 radeon_ring_write(rdev, ib->length_dw); 3692 radeon_ring_write(ring, PACKET0(RADEON_CP_IB_BASE, 1));
3693 radeon_ring_write(ring, ib->gpu_addr);
3694 radeon_ring_write(ring, ib->length_dw);
3683} 3695}
3684 3696
3685int r100_ib_test(struct radeon_device *rdev) 3697int r100_ib_test(struct radeon_device *rdev)
@@ -3696,7 +3708,7 @@ int r100_ib_test(struct radeon_device *rdev)
3696 return r; 3708 return r;
3697 } 3709 }
3698 WREG32(scratch, 0xCAFEDEAD); 3710 WREG32(scratch, 0xCAFEDEAD);
3699 r = radeon_ib_get(rdev, &ib); 3711 r = radeon_ib_get(rdev, RADEON_RING_TYPE_GFX_INDEX, &ib);
3700 if (r) { 3712 if (r) {
3701 return r; 3713 return r;
3702 } 3714 }
@@ -3740,34 +3752,16 @@ int r100_ib_test(struct radeon_device *rdev)
3740 3752
3741void r100_ib_fini(struct radeon_device *rdev) 3753void r100_ib_fini(struct radeon_device *rdev)
3742{ 3754{
3755 radeon_ib_pool_suspend(rdev);
3743 radeon_ib_pool_fini(rdev); 3756 radeon_ib_pool_fini(rdev);
3744} 3757}
3745 3758
3746int r100_ib_init(struct radeon_device *rdev)
3747{
3748 int r;
3749
3750 r = radeon_ib_pool_init(rdev);
3751 if (r) {
3752 dev_err(rdev->dev, "failed initializing IB pool (%d).\n", r);
3753 r100_ib_fini(rdev);
3754 return r;
3755 }
3756 r = r100_ib_test(rdev);
3757 if (r) {
3758 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
3759 r100_ib_fini(rdev);
3760 return r;
3761 }
3762 return 0;
3763}
3764
3765void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save) 3759void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save)
3766{ 3760{
3767 /* Shutdown CP we shouldn't need to do that but better be safe than 3761 /* Shutdown CP we shouldn't need to do that but better be safe than
3768 * sorry 3762 * sorry
3769 */ 3763 */
3770 rdev->cp.ready = false; 3764 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3771 WREG32(R_000740_CP_CSQ_CNTL, 0); 3765 WREG32(R_000740_CP_CSQ_CNTL, 0);
3772 3766
3773 /* Save few CRTC registers */ 3767 /* Save few CRTC registers */
@@ -3905,6 +3899,12 @@ static int r100_startup(struct radeon_device *rdev)
3905 if (r) 3899 if (r)
3906 return r; 3900 return r;
3907 3901
3902 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3903 if (r) {
3904 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3905 return r;
3906 }
3907
3908 /* Enable IRQ */ 3908 /* Enable IRQ */
3909 r100_irq_set(rdev); 3909 r100_irq_set(rdev);
3910 rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 3910 rdev->config.r100.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -3914,11 +3914,18 @@ static int r100_startup(struct radeon_device *rdev)
3914 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 3914 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
3915 return r; 3915 return r;
3916 } 3916 }
3917 r = r100_ib_init(rdev); 3917
3918 r = radeon_ib_pool_start(rdev);
3919 if (r)
3920 return r;
3921
3922 r = r100_ib_test(rdev);
3918 if (r) { 3923 if (r) {
3919 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 3924 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
3925 rdev->accel_working = false;
3920 return r; 3926 return r;
3921 } 3927 }
3928
3922 return 0; 3929 return 0;
3923} 3930}
3924 3931
@@ -3941,11 +3948,14 @@ int r100_resume(struct radeon_device *rdev)
3941 r100_clock_startup(rdev); 3948 r100_clock_startup(rdev);
3942 /* Initialize surface registers */ 3949 /* Initialize surface registers */
3943 radeon_surface_init(rdev); 3950 radeon_surface_init(rdev);
3951
3952 rdev->accel_working = true;
3944 return r100_startup(rdev); 3953 return r100_startup(rdev);
3945} 3954}
3946 3955
3947int r100_suspend(struct radeon_device *rdev) 3956int r100_suspend(struct radeon_device *rdev)
3948{ 3957{
3958 radeon_ib_pool_suspend(rdev);
3949 r100_cp_disable(rdev); 3959 r100_cp_disable(rdev);
3950 radeon_wb_disable(rdev); 3960 radeon_wb_disable(rdev);
3951 r100_irq_disable(rdev); 3961 r100_irq_disable(rdev);
@@ -4064,7 +4074,14 @@ int r100_init(struct radeon_device *rdev)
4064 return r; 4074 return r;
4065 } 4075 }
4066 r100_set_safe_registers(rdev); 4076 r100_set_safe_registers(rdev);
4077
4078 r = radeon_ib_pool_init(rdev);
4067 rdev->accel_working = true; 4079 rdev->accel_working = true;
4080 if (r) {
4081 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
4082 rdev->accel_working = false;
4083 }
4084
4068 r = r100_startup(rdev); 4085 r = r100_startup(rdev);
4069 if (r) { 4086 if (r) {
4070 /* Somethings want wront with the accel init stop accel */ 4087 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c
index a1f3ba063c2..eba4cbfa78f 100644
--- a/drivers/gpu/drm/radeon/r200.c
+++ b/drivers/gpu/drm/radeon/r200.c
@@ -87,6 +87,7 @@ int r200_copy_dma(struct radeon_device *rdev,
87 unsigned num_gpu_pages, 87 unsigned num_gpu_pages,
88 struct radeon_fence *fence) 88 struct radeon_fence *fence)
89{ 89{
90 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
90 uint32_t size; 91 uint32_t size;
91 uint32_t cur_size; 92 uint32_t cur_size;
92 int i, num_loops; 93 int i, num_loops;
@@ -95,33 +96,33 @@ int r200_copy_dma(struct radeon_device *rdev,
95 /* radeon pitch is /64 */ 96 /* radeon pitch is /64 */
96 size = num_gpu_pages << RADEON_GPU_PAGE_SHIFT; 97 size = num_gpu_pages << RADEON_GPU_PAGE_SHIFT;
97 num_loops = DIV_ROUND_UP(size, 0x1FFFFF); 98 num_loops = DIV_ROUND_UP(size, 0x1FFFFF);
98 r = radeon_ring_lock(rdev, num_loops * 4 + 64); 99 r = radeon_ring_lock(rdev, ring, num_loops * 4 + 64);
99 if (r) { 100 if (r) {
100 DRM_ERROR("radeon: moving bo (%d).\n", r); 101 DRM_ERROR("radeon: moving bo (%d).\n", r);
101 return r; 102 return r;
102 } 103 }
103 /* Must wait for 2D idle & clean before DMA or hangs might happen */ 104 /* Must wait for 2D idle & clean before DMA or hangs might happen */
104 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 105 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
105 radeon_ring_write(rdev, (1 << 16)); 106 radeon_ring_write(ring, (1 << 16));
106 for (i = 0; i < num_loops; i++) { 107 for (i = 0; i < num_loops; i++) {
107 cur_size = size; 108 cur_size = size;
108 if (cur_size > 0x1FFFFF) { 109 if (cur_size > 0x1FFFFF) {
109 cur_size = 0x1FFFFF; 110 cur_size = 0x1FFFFF;
110 } 111 }
111 size -= cur_size; 112 size -= cur_size;
112 radeon_ring_write(rdev, PACKET0(0x720, 2)); 113 radeon_ring_write(ring, PACKET0(0x720, 2));
113 radeon_ring_write(rdev, src_offset); 114 radeon_ring_write(ring, src_offset);
114 radeon_ring_write(rdev, dst_offset); 115 radeon_ring_write(ring, dst_offset);
115 radeon_ring_write(rdev, cur_size | (1 << 31) | (1 << 30)); 116 radeon_ring_write(ring, cur_size | (1 << 31) | (1 << 30));
116 src_offset += cur_size; 117 src_offset += cur_size;
117 dst_offset += cur_size; 118 dst_offset += cur_size;
118 } 119 }
119 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 120 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
120 radeon_ring_write(rdev, RADEON_WAIT_DMA_GUI_IDLE); 121 radeon_ring_write(ring, RADEON_WAIT_DMA_GUI_IDLE);
121 if (fence) { 122 if (fence) {
122 r = radeon_fence_emit(rdev, fence); 123 r = radeon_fence_emit(rdev, fence);
123 } 124 }
124 radeon_ring_unlock_commit(rdev); 125 radeon_ring_unlock_commit(rdev, ring);
125 return r; 126 return r;
126} 127}
127 128
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index c93bc64707e..8ad5c6475d5 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -175,37 +175,40 @@ void rv370_pcie_gart_fini(struct radeon_device *rdev)
175void r300_fence_ring_emit(struct radeon_device *rdev, 175void r300_fence_ring_emit(struct radeon_device *rdev,
176 struct radeon_fence *fence) 176 struct radeon_fence *fence)
177{ 177{
178 struct radeon_ring *ring = &rdev->ring[fence->ring];
179
178 /* Who ever call radeon_fence_emit should call ring_lock and ask 180 /* Who ever call radeon_fence_emit should call ring_lock and ask
179 * for enough space (today caller are ib schedule and buffer move) */ 181 * for enough space (today caller are ib schedule and buffer move) */
180 /* Write SC register so SC & US assert idle */ 182 /* Write SC register so SC & US assert idle */
181 radeon_ring_write(rdev, PACKET0(R300_RE_SCISSORS_TL, 0)); 183 radeon_ring_write(ring, PACKET0(R300_RE_SCISSORS_TL, 0));
182 radeon_ring_write(rdev, 0); 184 radeon_ring_write(ring, 0);
183 radeon_ring_write(rdev, PACKET0(R300_RE_SCISSORS_BR, 0)); 185 radeon_ring_write(ring, PACKET0(R300_RE_SCISSORS_BR, 0));
184 radeon_ring_write(rdev, 0); 186 radeon_ring_write(ring, 0);
185 /* Flush 3D cache */ 187 /* Flush 3D cache */
186 radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 188 radeon_ring_write(ring, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
187 radeon_ring_write(rdev, R300_RB3D_DC_FLUSH); 189 radeon_ring_write(ring, R300_RB3D_DC_FLUSH);
188 radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0)); 190 radeon_ring_write(ring, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
189 radeon_ring_write(rdev, R300_ZC_FLUSH); 191 radeon_ring_write(ring, R300_ZC_FLUSH);
190 /* Wait until IDLE & CLEAN */ 192 /* Wait until IDLE & CLEAN */
191 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 193 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
192 radeon_ring_write(rdev, (RADEON_WAIT_3D_IDLECLEAN | 194 radeon_ring_write(ring, (RADEON_WAIT_3D_IDLECLEAN |
193 RADEON_WAIT_2D_IDLECLEAN | 195 RADEON_WAIT_2D_IDLECLEAN |
194 RADEON_WAIT_DMA_GUI_IDLE)); 196 RADEON_WAIT_DMA_GUI_IDLE));
195 radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); 197 radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0));
196 radeon_ring_write(rdev, rdev->config.r300.hdp_cntl | 198 radeon_ring_write(ring, rdev->config.r300.hdp_cntl |
197 RADEON_HDP_READ_BUFFER_INVALIDATE); 199 RADEON_HDP_READ_BUFFER_INVALIDATE);
198 radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0)); 200 radeon_ring_write(ring, PACKET0(RADEON_HOST_PATH_CNTL, 0));
199 radeon_ring_write(rdev, rdev->config.r300.hdp_cntl); 201 radeon_ring_write(ring, rdev->config.r300.hdp_cntl);
200 /* Emit fence sequence & fire IRQ */ 202 /* Emit fence sequence & fire IRQ */
201 radeon_ring_write(rdev, PACKET0(rdev->fence_drv.scratch_reg, 0)); 203 radeon_ring_write(ring, PACKET0(rdev->fence_drv[fence->ring].scratch_reg, 0));
202 radeon_ring_write(rdev, fence->seq); 204 radeon_ring_write(ring, fence->seq);
203 radeon_ring_write(rdev, PACKET0(RADEON_GEN_INT_STATUS, 0)); 205 radeon_ring_write(ring, PACKET0(RADEON_GEN_INT_STATUS, 0));
204 radeon_ring_write(rdev, RADEON_SW_INT_FIRE); 206 radeon_ring_write(ring, RADEON_SW_INT_FIRE);
205} 207}
206 208
207void r300_ring_start(struct radeon_device *rdev) 209void r300_ring_start(struct radeon_device *rdev)
208{ 210{
211 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
209 unsigned gb_tile_config; 212 unsigned gb_tile_config;
210 int r; 213 int r;
211 214
@@ -227,44 +230,44 @@ void r300_ring_start(struct radeon_device *rdev)
227 break; 230 break;
228 } 231 }
229 232
230 r = radeon_ring_lock(rdev, 64); 233 r = radeon_ring_lock(rdev, ring, 64);
231 if (r) { 234 if (r) {
232 return; 235 return;
233 } 236 }
234 radeon_ring_write(rdev, PACKET0(RADEON_ISYNC_CNTL, 0)); 237 radeon_ring_write(ring, PACKET0(RADEON_ISYNC_CNTL, 0));
235 radeon_ring_write(rdev, 238 radeon_ring_write(ring,
236 RADEON_ISYNC_ANY2D_IDLE3D | 239 RADEON_ISYNC_ANY2D_IDLE3D |
237 RADEON_ISYNC_ANY3D_IDLE2D | 240 RADEON_ISYNC_ANY3D_IDLE2D |
238 RADEON_ISYNC_WAIT_IDLEGUI | 241 RADEON_ISYNC_WAIT_IDLEGUI |
239 RADEON_ISYNC_CPSCRATCH_IDLEGUI); 242 RADEON_ISYNC_CPSCRATCH_IDLEGUI);
240 radeon_ring_write(rdev, PACKET0(R300_GB_TILE_CONFIG, 0)); 243 radeon_ring_write(ring, PACKET0(R300_GB_TILE_CONFIG, 0));
241 radeon_ring_write(rdev, gb_tile_config); 244 radeon_ring_write(ring, gb_tile_config);
242 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 245 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
243 radeon_ring_write(rdev, 246 radeon_ring_write(ring,
244 RADEON_WAIT_2D_IDLECLEAN | 247 RADEON_WAIT_2D_IDLECLEAN |
245 RADEON_WAIT_3D_IDLECLEAN); 248 RADEON_WAIT_3D_IDLECLEAN);
246 radeon_ring_write(rdev, PACKET0(R300_DST_PIPE_CONFIG, 0)); 249 radeon_ring_write(ring, PACKET0(R300_DST_PIPE_CONFIG, 0));
247 radeon_ring_write(rdev, R300_PIPE_AUTO_CONFIG); 250 radeon_ring_write(ring, R300_PIPE_AUTO_CONFIG);
248 radeon_ring_write(rdev, PACKET0(R300_GB_SELECT, 0)); 251 radeon_ring_write(ring, PACKET0(R300_GB_SELECT, 0));
249 radeon_ring_write(rdev, 0); 252 radeon_ring_write(ring, 0);
250 radeon_ring_write(rdev, PACKET0(R300_GB_ENABLE, 0)); 253 radeon_ring_write(ring, PACKET0(R300_GB_ENABLE, 0));
251 radeon_ring_write(rdev, 0); 254 radeon_ring_write(ring, 0);
252 radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 255 radeon_ring_write(ring, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
253 radeon_ring_write(rdev, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE); 256 radeon_ring_write(ring, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE);
254 radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0)); 257 radeon_ring_write(ring, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
255 radeon_ring_write(rdev, R300_ZC_FLUSH | R300_ZC_FREE); 258 radeon_ring_write(ring, R300_ZC_FLUSH | R300_ZC_FREE);
256 radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0)); 259 radeon_ring_write(ring, PACKET0(RADEON_WAIT_UNTIL, 0));
257 radeon_ring_write(rdev, 260 radeon_ring_write(ring,
258 RADEON_WAIT_2D_IDLECLEAN | 261 RADEON_WAIT_2D_IDLECLEAN |
259 RADEON_WAIT_3D_IDLECLEAN); 262 RADEON_WAIT_3D_IDLECLEAN);
260 radeon_ring_write(rdev, PACKET0(R300_GB_AA_CONFIG, 0)); 263 radeon_ring_write(ring, PACKET0(R300_GB_AA_CONFIG, 0));
261 radeon_ring_write(rdev, 0); 264 radeon_ring_write(ring, 0);
262 radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 265 radeon_ring_write(ring, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
263 radeon_ring_write(rdev, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE); 266 radeon_ring_write(ring, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE);
264 radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0)); 267 radeon_ring_write(ring, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
265 radeon_ring_write(rdev, R300_ZC_FLUSH | R300_ZC_FREE); 268 radeon_ring_write(ring, R300_ZC_FLUSH | R300_ZC_FREE);
266 radeon_ring_write(rdev, PACKET0(R300_GB_MSPOS0, 0)); 269 radeon_ring_write(ring, PACKET0(R300_GB_MSPOS0, 0));
267 radeon_ring_write(rdev, 270 radeon_ring_write(ring,
268 ((6 << R300_MS_X0_SHIFT) | 271 ((6 << R300_MS_X0_SHIFT) |
269 (6 << R300_MS_Y0_SHIFT) | 272 (6 << R300_MS_Y0_SHIFT) |
270 (6 << R300_MS_X1_SHIFT) | 273 (6 << R300_MS_X1_SHIFT) |
@@ -273,8 +276,8 @@ void r300_ring_start(struct radeon_device *rdev)
273 (6 << R300_MS_Y2_SHIFT) | 276 (6 << R300_MS_Y2_SHIFT) |
274 (6 << R300_MSBD0_Y_SHIFT) | 277 (6 << R300_MSBD0_Y_SHIFT) |
275 (6 << R300_MSBD0_X_SHIFT))); 278 (6 << R300_MSBD0_X_SHIFT)));
276 radeon_ring_write(rdev, PACKET0(R300_GB_MSPOS1, 0)); 279 radeon_ring_write(ring, PACKET0(R300_GB_MSPOS1, 0));
277 radeon_ring_write(rdev, 280 radeon_ring_write(ring,
278 ((6 << R300_MS_X3_SHIFT) | 281 ((6 << R300_MS_X3_SHIFT) |
279 (6 << R300_MS_Y3_SHIFT) | 282 (6 << R300_MS_Y3_SHIFT) |
280 (6 << R300_MS_X4_SHIFT) | 283 (6 << R300_MS_X4_SHIFT) |
@@ -282,16 +285,16 @@ void r300_ring_start(struct radeon_device *rdev)
282 (6 << R300_MS_X5_SHIFT) | 285 (6 << R300_MS_X5_SHIFT) |
283 (6 << R300_MS_Y5_SHIFT) | 286 (6 << R300_MS_Y5_SHIFT) |
284 (6 << R300_MSBD1_SHIFT))); 287 (6 << R300_MSBD1_SHIFT)));
285 radeon_ring_write(rdev, PACKET0(R300_GA_ENHANCE, 0)); 288 radeon_ring_write(ring, PACKET0(R300_GA_ENHANCE, 0));
286 radeon_ring_write(rdev, R300_GA_DEADLOCK_CNTL | R300_GA_FASTSYNC_CNTL); 289 radeon_ring_write(ring, R300_GA_DEADLOCK_CNTL | R300_GA_FASTSYNC_CNTL);
287 radeon_ring_write(rdev, PACKET0(R300_GA_POLY_MODE, 0)); 290 radeon_ring_write(ring, PACKET0(R300_GA_POLY_MODE, 0));
288 radeon_ring_write(rdev, 291 radeon_ring_write(ring,
289 R300_FRONT_PTYPE_TRIANGE | R300_BACK_PTYPE_TRIANGE); 292 R300_FRONT_PTYPE_TRIANGE | R300_BACK_PTYPE_TRIANGE);
290 radeon_ring_write(rdev, PACKET0(R300_GA_ROUND_MODE, 0)); 293 radeon_ring_write(ring, PACKET0(R300_GA_ROUND_MODE, 0));
291 radeon_ring_write(rdev, 294 radeon_ring_write(ring,
292 R300_GEOMETRY_ROUND_NEAREST | 295 R300_GEOMETRY_ROUND_NEAREST |
293 R300_COLOR_ROUND_NEAREST); 296 R300_COLOR_ROUND_NEAREST);
294 radeon_ring_unlock_commit(rdev); 297 radeon_ring_unlock_commit(rdev, ring);
295} 298}
296 299
297void r300_errata(struct radeon_device *rdev) 300void r300_errata(struct radeon_device *rdev)
@@ -375,26 +378,26 @@ void r300_gpu_init(struct radeon_device *rdev)
375 rdev->num_gb_pipes, rdev->num_z_pipes); 378 rdev->num_gb_pipes, rdev->num_z_pipes);
376} 379}
377 380
378bool r300_gpu_is_lockup(struct radeon_device *rdev) 381bool r300_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
379{ 382{
380 u32 rbbm_status; 383 u32 rbbm_status;
381 int r; 384 int r;
382 385
383 rbbm_status = RREG32(R_000E40_RBBM_STATUS); 386 rbbm_status = RREG32(R_000E40_RBBM_STATUS);
384 if (!G_000E40_GUI_ACTIVE(rbbm_status)) { 387 if (!G_000E40_GUI_ACTIVE(rbbm_status)) {
385 r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp); 388 r100_gpu_lockup_update(&rdev->config.r300.lockup, ring);
386 return false; 389 return false;
387 } 390 }
388 /* force CP activities */ 391 /* force CP activities */
389 r = radeon_ring_lock(rdev, 2); 392 r = radeon_ring_lock(rdev, ring, 2);
390 if (!r) { 393 if (!r) {
391 /* PACKET2 NOP */ 394 /* PACKET2 NOP */
392 radeon_ring_write(rdev, 0x80000000); 395 radeon_ring_write(ring, 0x80000000);
393 radeon_ring_write(rdev, 0x80000000); 396 radeon_ring_write(ring, 0x80000000);
394 radeon_ring_unlock_commit(rdev); 397 radeon_ring_unlock_commit(rdev, ring);
395 } 398 }
396 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 399 ring->rptr = RREG32(RADEON_CP_RB_RPTR);
397 return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp); 400 return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, ring);
398} 401}
399 402
400int r300_asic_reset(struct radeon_device *rdev) 403int r300_asic_reset(struct radeon_device *rdev)
@@ -1396,6 +1399,12 @@ static int r300_startup(struct radeon_device *rdev)
1396 if (r) 1399 if (r)
1397 return r; 1400 return r;
1398 1401
1402 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1403 if (r) {
1404 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1405 return r;
1406 }
1407
1399 /* Enable IRQ */ 1408 /* Enable IRQ */
1400 r100_irq_set(rdev); 1409 r100_irq_set(rdev);
1401 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 1410 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -1405,11 +1414,18 @@ static int r300_startup(struct radeon_device *rdev)
1405 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 1414 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
1406 return r; 1415 return r;
1407 } 1416 }
1408 r = r100_ib_init(rdev); 1417
1418 r = radeon_ib_pool_start(rdev);
1419 if (r)
1420 return r;
1421
1422 r = r100_ib_test(rdev);
1409 if (r) { 1423 if (r) {
1410 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 1424 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
1425 rdev->accel_working = false;
1411 return r; 1426 return r;
1412 } 1427 }
1428
1413 return 0; 1429 return 0;
1414} 1430}
1415 1431
@@ -1434,11 +1450,14 @@ int r300_resume(struct radeon_device *rdev)
1434 r300_clock_startup(rdev); 1450 r300_clock_startup(rdev);
1435 /* Initialize surface registers */ 1451 /* Initialize surface registers */
1436 radeon_surface_init(rdev); 1452 radeon_surface_init(rdev);
1453
1454 rdev->accel_working = true;
1437 return r300_startup(rdev); 1455 return r300_startup(rdev);
1438} 1456}
1439 1457
1440int r300_suspend(struct radeon_device *rdev) 1458int r300_suspend(struct radeon_device *rdev)
1441{ 1459{
1460 radeon_ib_pool_suspend(rdev);
1442 r100_cp_disable(rdev); 1461 r100_cp_disable(rdev);
1443 radeon_wb_disable(rdev); 1462 radeon_wb_disable(rdev);
1444 r100_irq_disable(rdev); 1463 r100_irq_disable(rdev);
@@ -1539,7 +1558,14 @@ int r300_init(struct radeon_device *rdev)
1539 return r; 1558 return r;
1540 } 1559 }
1541 r300_set_reg_safe(rdev); 1560 r300_set_reg_safe(rdev);
1561
1562 r = radeon_ib_pool_init(rdev);
1542 rdev->accel_working = true; 1563 rdev->accel_working = true;
1564 if (r) {
1565 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1566 rdev->accel_working = false;
1567 }
1568
1543 r = r300_startup(rdev); 1569 r = r300_startup(rdev);
1544 if (r) { 1570 if (r) {
1545 /* Somethings want wront with the accel init stop accel */ 1571 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index 417fab81812..666e28fe509 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -199,6 +199,8 @@ static void r420_clock_resume(struct radeon_device *rdev)
199 199
200static void r420_cp_errata_init(struct radeon_device *rdev) 200static void r420_cp_errata_init(struct radeon_device *rdev)
201{ 201{
202 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
203
202 /* RV410 and R420 can lock up if CP DMA to host memory happens 204 /* RV410 and R420 can lock up if CP DMA to host memory happens
203 * while the 2D engine is busy. 205 * while the 2D engine is busy.
204 * 206 *
@@ -206,22 +208,24 @@ static void r420_cp_errata_init(struct radeon_device *rdev)
206 * of the CP init, apparently. 208 * of the CP init, apparently.
207 */ 209 */
208 radeon_scratch_get(rdev, &rdev->config.r300.resync_scratch); 210 radeon_scratch_get(rdev, &rdev->config.r300.resync_scratch);
209 radeon_ring_lock(rdev, 8); 211 radeon_ring_lock(rdev, ring, 8);
210 radeon_ring_write(rdev, PACKET0(R300_CP_RESYNC_ADDR, 1)); 212 radeon_ring_write(ring, PACKET0(R300_CP_RESYNC_ADDR, 1));
211 radeon_ring_write(rdev, rdev->config.r300.resync_scratch); 213 radeon_ring_write(ring, rdev->config.r300.resync_scratch);
212 radeon_ring_write(rdev, 0xDEADBEEF); 214 radeon_ring_write(ring, 0xDEADBEEF);
213 radeon_ring_unlock_commit(rdev); 215 radeon_ring_unlock_commit(rdev, ring);
214} 216}
215 217
216static void r420_cp_errata_fini(struct radeon_device *rdev) 218static void r420_cp_errata_fini(struct radeon_device *rdev)
217{ 219{
220 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
221
218 /* Catch the RESYNC we dispatched all the way back, 222 /* Catch the RESYNC we dispatched all the way back,
219 * at the very beginning of the CP init. 223 * at the very beginning of the CP init.
220 */ 224 */
221 radeon_ring_lock(rdev, 8); 225 radeon_ring_lock(rdev, ring, 8);
222 radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0)); 226 radeon_ring_write(ring, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
223 radeon_ring_write(rdev, R300_RB3D_DC_FINISH); 227 radeon_ring_write(ring, R300_RB3D_DC_FINISH);
224 radeon_ring_unlock_commit(rdev); 228 radeon_ring_unlock_commit(rdev, ring);
225 radeon_scratch_free(rdev, rdev->config.r300.resync_scratch); 229 radeon_scratch_free(rdev, rdev->config.r300.resync_scratch);
226} 230}
227 231
@@ -254,6 +258,12 @@ static int r420_startup(struct radeon_device *rdev)
254 if (r) 258 if (r)
255 return r; 259 return r;
256 260
261 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
262 if (r) {
263 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
264 return r;
265 }
266
257 /* Enable IRQ */ 267 /* Enable IRQ */
258 r100_irq_set(rdev); 268 r100_irq_set(rdev);
259 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 269 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -264,11 +274,18 @@ static int r420_startup(struct radeon_device *rdev)
264 return r; 274 return r;
265 } 275 }
266 r420_cp_errata_init(rdev); 276 r420_cp_errata_init(rdev);
267 r = r100_ib_init(rdev); 277
278 r = radeon_ib_pool_start(rdev);
279 if (r)
280 return r;
281
282 r = r100_ib_test(rdev);
268 if (r) { 283 if (r) {
269 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 284 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
285 rdev->accel_working = false;
270 return r; 286 return r;
271 } 287 }
288
272 return 0; 289 return 0;
273} 290}
274 291
@@ -297,11 +314,14 @@ int r420_resume(struct radeon_device *rdev)
297 r420_clock_resume(rdev); 314 r420_clock_resume(rdev);
298 /* Initialize surface registers */ 315 /* Initialize surface registers */
299 radeon_surface_init(rdev); 316 radeon_surface_init(rdev);
317
318 rdev->accel_working = true;
300 return r420_startup(rdev); 319 return r420_startup(rdev);
301} 320}
302 321
303int r420_suspend(struct radeon_device *rdev) 322int r420_suspend(struct radeon_device *rdev)
304{ 323{
324 radeon_ib_pool_suspend(rdev);
305 r420_cp_errata_fini(rdev); 325 r420_cp_errata_fini(rdev);
306 r100_cp_disable(rdev); 326 r100_cp_disable(rdev);
307 radeon_wb_disable(rdev); 327 radeon_wb_disable(rdev);
@@ -414,7 +434,14 @@ int r420_init(struct radeon_device *rdev)
414 return r; 434 return r;
415 } 435 }
416 r420_set_reg_safe(rdev); 436 r420_set_reg_safe(rdev);
437
438 r = radeon_ib_pool_init(rdev);
417 rdev->accel_working = true; 439 rdev->accel_working = true;
440 if (r) {
441 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
442 rdev->accel_working = false;
443 }
444
418 r = r420_startup(rdev); 445 r = r420_startup(rdev);
419 if (r) { 446 if (r) {
420 /* Somethings want wront with the accel init stop accel */ 447 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c
index 3081d07f8de..4ae1615e752 100644
--- a/drivers/gpu/drm/radeon/r520.c
+++ b/drivers/gpu/drm/radeon/r520.c
@@ -187,6 +187,12 @@ static int r520_startup(struct radeon_device *rdev)
187 if (r) 187 if (r)
188 return r; 188 return r;
189 189
190 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
191 if (r) {
192 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
193 return r;
194 }
195
190 /* Enable IRQ */ 196 /* Enable IRQ */
191 rs600_irq_set(rdev); 197 rs600_irq_set(rdev);
192 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 198 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -196,9 +202,15 @@ static int r520_startup(struct radeon_device *rdev)
196 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 202 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
197 return r; 203 return r;
198 } 204 }
199 r = r100_ib_init(rdev); 205
206 r = radeon_ib_pool_start(rdev);
207 if (r)
208 return r;
209
210 r = r100_ib_test(rdev);
200 if (r) { 211 if (r) {
201 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 212 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
213 rdev->accel_working = false;
202 return r; 214 return r;
203 } 215 }
204 return 0; 216 return 0;
@@ -223,6 +235,8 @@ int r520_resume(struct radeon_device *rdev)
223 rv515_clock_startup(rdev); 235 rv515_clock_startup(rdev);
224 /* Initialize surface registers */ 236 /* Initialize surface registers */
225 radeon_surface_init(rdev); 237 radeon_surface_init(rdev);
238
239 rdev->accel_working = true;
226 return r520_startup(rdev); 240 return r520_startup(rdev);
227} 241}
228 242
@@ -292,7 +306,14 @@ int r520_init(struct radeon_device *rdev)
292 if (r) 306 if (r)
293 return r; 307 return r;
294 rv515_set_safe_registers(rdev); 308 rv515_set_safe_registers(rdev);
309
310 r = radeon_ib_pool_init(rdev);
295 rdev->accel_working = true; 311 rdev->accel_working = true;
312 if (r) {
313 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
314 rdev->accel_working = false;
315 }
316
296 r = r520_startup(rdev); 317 r = r520_startup(rdev);
297 if (r) { 318 if (r) {
298 /* Somethings want wront with the accel init stop accel */ 319 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 9cdda0b3b08..951566f1de9 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -1344,7 +1344,7 @@ int r600_gpu_soft_reset(struct radeon_device *rdev)
1344 return 0; 1344 return 0;
1345} 1345}
1346 1346
1347bool r600_gpu_is_lockup(struct radeon_device *rdev) 1347bool r600_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1348{ 1348{
1349 u32 srbm_status; 1349 u32 srbm_status;
1350 u32 grbm_status; 1350 u32 grbm_status;
@@ -1361,19 +1361,19 @@ bool r600_gpu_is_lockup(struct radeon_device *rdev)
1361 grbm_status = RREG32(R_008010_GRBM_STATUS); 1361 grbm_status = RREG32(R_008010_GRBM_STATUS);
1362 grbm_status2 = RREG32(R_008014_GRBM_STATUS2); 1362 grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
1363 if (!G_008010_GUI_ACTIVE(grbm_status)) { 1363 if (!G_008010_GUI_ACTIVE(grbm_status)) {
1364 r100_gpu_lockup_update(lockup, &rdev->cp); 1364 r100_gpu_lockup_update(lockup, ring);
1365 return false; 1365 return false;
1366 } 1366 }
1367 /* force CP activities */ 1367 /* force CP activities */
1368 r = radeon_ring_lock(rdev, 2); 1368 r = radeon_ring_lock(rdev, ring, 2);
1369 if (!r) { 1369 if (!r) {
1370 /* PACKET2 NOP */ 1370 /* PACKET2 NOP */
1371 radeon_ring_write(rdev, 0x80000000); 1371 radeon_ring_write(ring, 0x80000000);
1372 radeon_ring_write(rdev, 0x80000000); 1372 radeon_ring_write(ring, 0x80000000);
1373 radeon_ring_unlock_commit(rdev); 1373 radeon_ring_unlock_commit(rdev, ring);
1374 } 1374 }
1375 rdev->cp.rptr = RREG32(R600_CP_RB_RPTR); 1375 ring->rptr = RREG32(ring->rptr_reg);
1376 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp); 1376 return r100_gpu_cp_is_lockup(rdev, lockup, ring);
1377} 1377}
1378 1378
1379int r600_asic_reset(struct radeon_device *rdev) 1379int r600_asic_reset(struct radeon_device *rdev)
@@ -2144,27 +2144,28 @@ static int r600_cp_load_microcode(struct radeon_device *rdev)
2144 2144
2145int r600_cp_start(struct radeon_device *rdev) 2145int r600_cp_start(struct radeon_device *rdev)
2146{ 2146{
2147 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2147 int r; 2148 int r;
2148 uint32_t cp_me; 2149 uint32_t cp_me;
2149 2150
2150 r = radeon_ring_lock(rdev, 7); 2151 r = radeon_ring_lock(rdev, ring, 7);
2151 if (r) { 2152 if (r) {
2152 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2153 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2153 return r; 2154 return r;
2154 } 2155 }
2155 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5)); 2156 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2156 radeon_ring_write(rdev, 0x1); 2157 radeon_ring_write(ring, 0x1);
2157 if (rdev->family >= CHIP_RV770) { 2158 if (rdev->family >= CHIP_RV770) {
2158 radeon_ring_write(rdev, 0x0); 2159 radeon_ring_write(ring, 0x0);
2159 radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1); 2160 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2160 } else { 2161 } else {
2161 radeon_ring_write(rdev, 0x3); 2162 radeon_ring_write(ring, 0x3);
2162 radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1); 2163 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2163 } 2164 }
2164 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 2165 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2165 radeon_ring_write(rdev, 0); 2166 radeon_ring_write(ring, 0);
2166 radeon_ring_write(rdev, 0); 2167 radeon_ring_write(ring, 0);
2167 radeon_ring_unlock_commit(rdev); 2168 radeon_ring_unlock_commit(rdev, ring);
2168 2169
2169 cp_me = 0xff; 2170 cp_me = 0xff;
2170 WREG32(R_0086D8_CP_ME_CNTL, cp_me); 2171 WREG32(R_0086D8_CP_ME_CNTL, cp_me);
@@ -2173,6 +2174,7 @@ int r600_cp_start(struct radeon_device *rdev)
2173 2174
2174int r600_cp_resume(struct radeon_device *rdev) 2175int r600_cp_resume(struct radeon_device *rdev)
2175{ 2176{
2177 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2176 u32 tmp; 2178 u32 tmp;
2177 u32 rb_bufsz; 2179 u32 rb_bufsz;
2178 int r; 2180 int r;
@@ -2184,13 +2186,13 @@ int r600_cp_resume(struct radeon_device *rdev)
2184 WREG32(GRBM_SOFT_RESET, 0); 2186 WREG32(GRBM_SOFT_RESET, 0);
2185 2187
2186 /* Set ring buffer size */ 2188 /* Set ring buffer size */
2187 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 2189 rb_bufsz = drm_order(ring->ring_size / 8);
2188 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2190 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2189#ifdef __BIG_ENDIAN 2191#ifdef __BIG_ENDIAN
2190 tmp |= BUF_SWAP_32BIT; 2192 tmp |= BUF_SWAP_32BIT;
2191#endif 2193#endif
2192 WREG32(CP_RB_CNTL, tmp); 2194 WREG32(CP_RB_CNTL, tmp);
2193 WREG32(CP_SEM_WAIT_TIMER, 0x4); 2195 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2194 2196
2195 /* Set the write pointer delay */ 2197 /* Set the write pointer delay */
2196 WREG32(CP_RB_WPTR_DELAY, 0); 2198 WREG32(CP_RB_WPTR_DELAY, 0);
@@ -2198,8 +2200,8 @@ int r600_cp_resume(struct radeon_device *rdev)
2198 /* Initialize the ring buffer's read and write pointers */ 2200 /* Initialize the ring buffer's read and write pointers */
2199 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 2201 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2200 WREG32(CP_RB_RPTR_WR, 0); 2202 WREG32(CP_RB_RPTR_WR, 0);
2201 rdev->cp.wptr = 0; 2203 ring->wptr = 0;
2202 WREG32(CP_RB_WPTR, rdev->cp.wptr); 2204 WREG32(CP_RB_WPTR, ring->wptr);
2203 2205
2204 /* set the wb address whether it's enabled or not */ 2206 /* set the wb address whether it's enabled or not */
2205 WREG32(CP_RB_RPTR_ADDR, 2207 WREG32(CP_RB_RPTR_ADDR,
@@ -2217,42 +2219,36 @@ int r600_cp_resume(struct radeon_device *rdev)
2217 mdelay(1); 2219 mdelay(1);
2218 WREG32(CP_RB_CNTL, tmp); 2220 WREG32(CP_RB_CNTL, tmp);
2219 2221
2220 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8); 2222 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2221 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 2223 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2222 2224
2223 rdev->cp.rptr = RREG32(CP_RB_RPTR); 2225 ring->rptr = RREG32(CP_RB_RPTR);
2224 2226
2225 r600_cp_start(rdev); 2227 r600_cp_start(rdev);
2226 rdev->cp.ready = true; 2228 ring->ready = true;
2227 r = radeon_ring_test(rdev); 2229 r = radeon_ring_test(rdev, ring);
2228 if (r) { 2230 if (r) {
2229 rdev->cp.ready = false; 2231 ring->ready = false;
2230 return r; 2232 return r;
2231 } 2233 }
2232 return 0; 2234 return 0;
2233} 2235}
2234 2236
2235void r600_cp_commit(struct radeon_device *rdev) 2237void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2236{
2237 WREG32(CP_RB_WPTR, rdev->cp.wptr);
2238 (void)RREG32(CP_RB_WPTR);
2239}
2240
2241void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
2242{ 2238{
2243 u32 rb_bufsz; 2239 u32 rb_bufsz;
2244 2240
2245 /* Align ring size */ 2241 /* Align ring size */
2246 rb_bufsz = drm_order(ring_size / 8); 2242 rb_bufsz = drm_order(ring_size / 8);
2247 ring_size = (1 << (rb_bufsz + 1)) * 4; 2243 ring_size = (1 << (rb_bufsz + 1)) * 4;
2248 rdev->cp.ring_size = ring_size; 2244 ring->ring_size = ring_size;
2249 rdev->cp.align_mask = 16 - 1; 2245 ring->align_mask = 16 - 1;
2250} 2246}
2251 2247
2252void r600_cp_fini(struct radeon_device *rdev) 2248void r600_cp_fini(struct radeon_device *rdev)
2253{ 2249{
2254 r600_cp_stop(rdev); 2250 r600_cp_stop(rdev);
2255 radeon_ring_fini(rdev); 2251 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2256} 2252}
2257 2253
2258 2254
@@ -2271,11 +2267,11 @@ void r600_scratch_init(struct radeon_device *rdev)
2271 } 2267 }
2272} 2268}
2273 2269
2274int r600_ring_test(struct radeon_device *rdev) 2270int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2275{ 2271{
2276 uint32_t scratch; 2272 uint32_t scratch;
2277 uint32_t tmp = 0; 2273 uint32_t tmp = 0;
2278 unsigned i; 2274 unsigned i, ridx = radeon_ring_index(rdev, ring);
2279 int r; 2275 int r;
2280 2276
2281 r = radeon_scratch_get(rdev, &scratch); 2277 r = radeon_scratch_get(rdev, &scratch);
@@ -2284,16 +2280,16 @@ int r600_ring_test(struct radeon_device *rdev)
2284 return r; 2280 return r;
2285 } 2281 }
2286 WREG32(scratch, 0xCAFEDEAD); 2282 WREG32(scratch, 0xCAFEDEAD);
2287 r = radeon_ring_lock(rdev, 3); 2283 r = radeon_ring_lock(rdev, ring, 3);
2288 if (r) { 2284 if (r) {
2289 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 2285 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ridx, r);
2290 radeon_scratch_free(rdev, scratch); 2286 radeon_scratch_free(rdev, scratch);
2291 return r; 2287 return r;
2292 } 2288 }
2293 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2289 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2294 radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); 2290 radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2295 radeon_ring_write(rdev, 0xDEADBEEF); 2291 radeon_ring_write(ring, 0xDEADBEEF);
2296 radeon_ring_unlock_commit(rdev); 2292 radeon_ring_unlock_commit(rdev, ring);
2297 for (i = 0; i < rdev->usec_timeout; i++) { 2293 for (i = 0; i < rdev->usec_timeout; i++) {
2298 tmp = RREG32(scratch); 2294 tmp = RREG32(scratch);
2299 if (tmp == 0xDEADBEEF) 2295 if (tmp == 0xDEADBEEF)
@@ -2301,10 +2297,10 @@ int r600_ring_test(struct radeon_device *rdev)
2301 DRM_UDELAY(1); 2297 DRM_UDELAY(1);
2302 } 2298 }
2303 if (i < rdev->usec_timeout) { 2299 if (i < rdev->usec_timeout) {
2304 DRM_INFO("ring test succeeded in %d usecs\n", i); 2300 DRM_INFO("ring test on %d succeeded in %d usecs\n", ridx, i);
2305 } else { 2301 } else {
2306 DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n", 2302 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2307 scratch, tmp); 2303 ridx, scratch, tmp);
2308 r = -EINVAL; 2304 r = -EINVAL;
2309 } 2305 }
2310 radeon_scratch_free(rdev, scratch); 2306 radeon_scratch_free(rdev, scratch);
@@ -2314,49 +2310,63 @@ int r600_ring_test(struct radeon_device *rdev)
2314void r600_fence_ring_emit(struct radeon_device *rdev, 2310void r600_fence_ring_emit(struct radeon_device *rdev,
2315 struct radeon_fence *fence) 2311 struct radeon_fence *fence)
2316{ 2312{
2313 struct radeon_ring *ring = &rdev->ring[fence->ring];
2314
2317 if (rdev->wb.use_event) { 2315 if (rdev->wb.use_event) {
2318 u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET + 2316 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2319 (u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
2320 /* flush read cache over gart */ 2317 /* flush read cache over gart */
2321 radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3)); 2318 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2322 radeon_ring_write(rdev, PACKET3_TC_ACTION_ENA | 2319 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2323 PACKET3_VC_ACTION_ENA | 2320 PACKET3_VC_ACTION_ENA |
2324 PACKET3_SH_ACTION_ENA); 2321 PACKET3_SH_ACTION_ENA);
2325 radeon_ring_write(rdev, 0xFFFFFFFF); 2322 radeon_ring_write(ring, 0xFFFFFFFF);
2326 radeon_ring_write(rdev, 0); 2323 radeon_ring_write(ring, 0);
2327 radeon_ring_write(rdev, 10); /* poll interval */ 2324 radeon_ring_write(ring, 10); /* poll interval */
2328 /* EVENT_WRITE_EOP - flush caches, send int */ 2325 /* EVENT_WRITE_EOP - flush caches, send int */
2329 radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 2326 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2330 radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5)); 2327 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2331 radeon_ring_write(rdev, addr & 0xffffffff); 2328 radeon_ring_write(ring, addr & 0xffffffff);
2332 radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 2329 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2333 radeon_ring_write(rdev, fence->seq); 2330 radeon_ring_write(ring, fence->seq);
2334 radeon_ring_write(rdev, 0); 2331 radeon_ring_write(ring, 0);
2335 } else { 2332 } else {
2336 /* flush read cache over gart */ 2333 /* flush read cache over gart */
2337 radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3)); 2334 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2338 radeon_ring_write(rdev, PACKET3_TC_ACTION_ENA | 2335 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2339 PACKET3_VC_ACTION_ENA | 2336 PACKET3_VC_ACTION_ENA |
2340 PACKET3_SH_ACTION_ENA); 2337 PACKET3_SH_ACTION_ENA);
2341 radeon_ring_write(rdev, 0xFFFFFFFF); 2338 radeon_ring_write(ring, 0xFFFFFFFF);
2342 radeon_ring_write(rdev, 0); 2339 radeon_ring_write(ring, 0);
2343 radeon_ring_write(rdev, 10); /* poll interval */ 2340 radeon_ring_write(ring, 10); /* poll interval */
2344 radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0)); 2341 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2345 radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0)); 2342 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2346 /* wait for 3D idle clean */ 2343 /* wait for 3D idle clean */
2347 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2344 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2348 radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); 2345 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2349 radeon_ring_write(rdev, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit); 2346 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2350 /* Emit fence sequence & fire IRQ */ 2347 /* Emit fence sequence & fire IRQ */
2351 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2348 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2352 radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2)); 2349 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2353 radeon_ring_write(rdev, fence->seq); 2350 radeon_ring_write(ring, fence->seq);
2354 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */ 2351 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2355 radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0)); 2352 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2356 radeon_ring_write(rdev, RB_INT_STAT); 2353 radeon_ring_write(ring, RB_INT_STAT);
2357 } 2354 }
2358} 2355}
2359 2356
2357void r600_semaphore_ring_emit(struct radeon_device *rdev,
2358 struct radeon_ring *ring,
2359 struct radeon_semaphore *semaphore,
2360 bool emit_wait)
2361{
2362 uint64_t addr = semaphore->gpu_addr;
2363 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2364
2365 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2366 radeon_ring_write(ring, addr & 0xffffffff);
2367 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2368}
2369
2360int r600_copy_blit(struct radeon_device *rdev, 2370int r600_copy_blit(struct radeon_device *rdev,
2361 uint64_t src_offset, 2371 uint64_t src_offset,
2362 uint64_t dst_offset, 2372 uint64_t dst_offset,
@@ -2409,6 +2419,7 @@ void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2409 2419
2410int r600_startup(struct radeon_device *rdev) 2420int r600_startup(struct radeon_device *rdev)
2411{ 2421{
2422 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2412 int r; 2423 int r;
2413 2424
2414 /* enable pcie gen2 link */ 2425 /* enable pcie gen2 link */
@@ -2447,6 +2458,12 @@ int r600_startup(struct radeon_device *rdev)
2447 if (r) 2458 if (r)
2448 return r; 2459 return r;
2449 2460
2461 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2462 if (r) {
2463 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2464 return r;
2465 }
2466
2450 /* Enable IRQ */ 2467 /* Enable IRQ */
2451 r = r600_irq_init(rdev); 2468 r = r600_irq_init(rdev);
2452 if (r) { 2469 if (r) {
@@ -2456,7 +2473,10 @@ int r600_startup(struct radeon_device *rdev)
2456 } 2473 }
2457 r600_irq_set(rdev); 2474 r600_irq_set(rdev);
2458 2475
2459 r = radeon_ring_init(rdev, rdev->cp.ring_size); 2476 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2477 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2478 0, 0xfffff, RADEON_CP_PACKET2);
2479
2460 if (r) 2480 if (r)
2461 return r; 2481 return r;
2462 r = r600_cp_load_microcode(rdev); 2482 r = r600_cp_load_microcode(rdev);
@@ -2466,6 +2486,17 @@ int r600_startup(struct radeon_device *rdev)
2466 if (r) 2486 if (r)
2467 return r; 2487 return r;
2468 2488
2489 r = radeon_ib_pool_start(rdev);
2490 if (r)
2491 return r;
2492
2493 r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
2494 if (r) {
2495 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
2496 rdev->accel_working = false;
2497 return r;
2498 }
2499
2469 return 0; 2500 return 0;
2470} 2501}
2471 2502
@@ -2494,18 +2525,13 @@ int r600_resume(struct radeon_device *rdev)
2494 /* post card */ 2525 /* post card */
2495 atom_asic_init(rdev->mode_info.atom_context); 2526 atom_asic_init(rdev->mode_info.atom_context);
2496 2527
2528 rdev->accel_working = true;
2497 r = r600_startup(rdev); 2529 r = r600_startup(rdev);
2498 if (r) { 2530 if (r) {
2499 DRM_ERROR("r600 startup failed on resume\n"); 2531 DRM_ERROR("r600 startup failed on resume\n");
2500 return r; 2532 return r;
2501 } 2533 }
2502 2534
2503 r = r600_ib_test(rdev);
2504 if (r) {
2505 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
2506 return r;
2507 }
2508
2509 r = r600_audio_init(rdev); 2535 r = r600_audio_init(rdev);
2510 if (r) { 2536 if (r) {
2511 DRM_ERROR("radeon: audio resume failed\n"); 2537 DRM_ERROR("radeon: audio resume failed\n");
@@ -2518,13 +2544,14 @@ int r600_resume(struct radeon_device *rdev)
2518int r600_suspend(struct radeon_device *rdev) 2544int r600_suspend(struct radeon_device *rdev)
2519{ 2545{
2520 r600_audio_fini(rdev); 2546 r600_audio_fini(rdev);
2547 radeon_ib_pool_suspend(rdev);
2548 r600_blit_suspend(rdev);
2521 /* FIXME: we should wait for ring to be empty */ 2549 /* FIXME: we should wait for ring to be empty */
2522 r600_cp_stop(rdev); 2550 r600_cp_stop(rdev);
2523 rdev->cp.ready = false; 2551 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2524 r600_irq_suspend(rdev); 2552 r600_irq_suspend(rdev);
2525 radeon_wb_disable(rdev); 2553 radeon_wb_disable(rdev);
2526 r600_pcie_gart_disable(rdev); 2554 r600_pcie_gart_disable(rdev);
2527 r600_blit_suspend(rdev);
2528 2555
2529 return 0; 2556 return 0;
2530} 2557}
@@ -2595,8 +2622,8 @@ int r600_init(struct radeon_device *rdev)
2595 if (r) 2622 if (r)
2596 return r; 2623 return r;
2597 2624
2598 rdev->cp.ring_obj = NULL; 2625 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
2599 r600_ring_init(rdev, 1024 * 1024); 2626 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
2600 2627
2601 rdev->ih.ring_obj = NULL; 2628 rdev->ih.ring_obj = NULL;
2602 r600_ih_ring_init(rdev, 64 * 1024); 2629 r600_ih_ring_init(rdev, 64 * 1024);
@@ -2605,30 +2632,24 @@ int r600_init(struct radeon_device *rdev)
2605 if (r) 2632 if (r)
2606 return r; 2633 return r;
2607 2634
2635 r = radeon_ib_pool_init(rdev);
2608 rdev->accel_working = true; 2636 rdev->accel_working = true;
2637 if (r) {
2638 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2639 rdev->accel_working = false;
2640 }
2641
2609 r = r600_startup(rdev); 2642 r = r600_startup(rdev);
2610 if (r) { 2643 if (r) {
2611 dev_err(rdev->dev, "disabling GPU acceleration\n"); 2644 dev_err(rdev->dev, "disabling GPU acceleration\n");
2612 r600_cp_fini(rdev); 2645 r600_cp_fini(rdev);
2613 r600_irq_fini(rdev); 2646 r600_irq_fini(rdev);
2614 radeon_wb_fini(rdev); 2647 radeon_wb_fini(rdev);
2648 r100_ib_fini(rdev);
2615 radeon_irq_kms_fini(rdev); 2649 radeon_irq_kms_fini(rdev);
2616 r600_pcie_gart_fini(rdev); 2650 r600_pcie_gart_fini(rdev);
2617 rdev->accel_working = false; 2651 rdev->accel_working = false;
2618 } 2652 }
2619 if (rdev->accel_working) {
2620 r = radeon_ib_pool_init(rdev);
2621 if (r) {
2622 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2623 rdev->accel_working = false;
2624 } else {
2625 r = r600_ib_test(rdev);
2626 if (r) {
2627 dev_err(rdev->dev, "IB test failed (%d).\n", r);
2628 rdev->accel_working = false;
2629 }
2630 }
2631 }
2632 2653
2633 r = r600_audio_init(rdev); 2654 r = r600_audio_init(rdev);
2634 if (r) 2655 if (r)
@@ -2643,12 +2664,13 @@ void r600_fini(struct radeon_device *rdev)
2643 r600_cp_fini(rdev); 2664 r600_cp_fini(rdev);
2644 r600_irq_fini(rdev); 2665 r600_irq_fini(rdev);
2645 radeon_wb_fini(rdev); 2666 radeon_wb_fini(rdev);
2646 radeon_ib_pool_fini(rdev); 2667 r100_ib_fini(rdev);
2647 radeon_irq_kms_fini(rdev); 2668 radeon_irq_kms_fini(rdev);
2648 r600_pcie_gart_fini(rdev); 2669 r600_pcie_gart_fini(rdev);
2649 r600_vram_scratch_fini(rdev); 2670 r600_vram_scratch_fini(rdev);
2650 radeon_agp_fini(rdev); 2671 radeon_agp_fini(rdev);
2651 radeon_gem_fini(rdev); 2672 radeon_gem_fini(rdev);
2673 radeon_semaphore_driver_fini(rdev);
2652 radeon_fence_driver_fini(rdev); 2674 radeon_fence_driver_fini(rdev);
2653 radeon_bo_fini(rdev); 2675 radeon_bo_fini(rdev);
2654 radeon_atombios_fini(rdev); 2676 radeon_atombios_fini(rdev);
@@ -2662,18 +2684,20 @@ void r600_fini(struct radeon_device *rdev)
2662 */ 2684 */
2663void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 2685void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2664{ 2686{
2687 struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
2688
2665 /* FIXME: implement */ 2689 /* FIXME: implement */
2666 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 2690 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2667 radeon_ring_write(rdev, 2691 radeon_ring_write(ring,
2668#ifdef __BIG_ENDIAN 2692#ifdef __BIG_ENDIAN
2669 (2 << 0) | 2693 (2 << 0) |
2670#endif 2694#endif
2671 (ib->gpu_addr & 0xFFFFFFFC)); 2695 (ib->gpu_addr & 0xFFFFFFFC));
2672 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF); 2696 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2673 radeon_ring_write(rdev, ib->length_dw); 2697 radeon_ring_write(ring, ib->length_dw);
2674} 2698}
2675 2699
2676int r600_ib_test(struct radeon_device *rdev) 2700int r600_ib_test(struct radeon_device *rdev, int ring)
2677{ 2701{
2678 struct radeon_ib *ib; 2702 struct radeon_ib *ib;
2679 uint32_t scratch; 2703 uint32_t scratch;
@@ -2687,7 +2711,7 @@ int r600_ib_test(struct radeon_device *rdev)
2687 return r; 2711 return r;
2688 } 2712 }
2689 WREG32(scratch, 0xCAFEDEAD); 2713 WREG32(scratch, 0xCAFEDEAD);
2690 r = radeon_ib_get(rdev, &ib); 2714 r = radeon_ib_get(rdev, ring, &ib);
2691 if (r) { 2715 if (r) {
2692 DRM_ERROR("radeon: failed to get ib (%d).\n", r); 2716 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2693 return r; 2717 return r;
@@ -2728,7 +2752,7 @@ int r600_ib_test(struct radeon_device *rdev)
2728 DRM_UDELAY(1); 2752 DRM_UDELAY(1);
2729 } 2753 }
2730 if (i < rdev->usec_timeout) { 2754 if (i < rdev->usec_timeout) {
2731 DRM_INFO("ib test succeeded in %u usecs\n", i); 2755 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib->fence->ring, i);
2732 } else { 2756 } else {
2733 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n", 2757 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
2734 scratch, tmp); 2758 scratch, tmp);
@@ -3075,7 +3099,7 @@ int r600_irq_set(struct radeon_device *rdev)
3075 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN; 3099 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3076 } 3100 }
3077 3101
3078 if (rdev->irq.sw_int) { 3102 if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
3079 DRM_DEBUG("r600_irq_set: sw int\n"); 3103 DRM_DEBUG("r600_irq_set: sw int\n");
3080 cp_int_cntl |= RB_INT_ENABLE; 3104 cp_int_cntl |= RB_INT_ENABLE;
3081 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 3105 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
@@ -3459,11 +3483,11 @@ restart_ih:
3459 case 177: /* CP_INT in IB1 */ 3483 case 177: /* CP_INT in IB1 */
3460 case 178: /* CP_INT in IB2 */ 3484 case 178: /* CP_INT in IB2 */
3461 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 3485 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3462 radeon_fence_process(rdev); 3486 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3463 break; 3487 break;
3464 case 181: /* CP EOP event */ 3488 case 181: /* CP EOP event */
3465 DRM_DEBUG("IH: CP EOP\n"); 3489 DRM_DEBUG("IH: CP EOP\n");
3466 radeon_fence_process(rdev); 3490 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3467 break; 3491 break;
3468 case 233: /* GUI IDLE */ 3492 case 233: /* GUI IDLE */
3469 DRM_DEBUG("IH: GUI idle\n"); 3493 DRM_DEBUG("IH: GUI idle\n");
@@ -3496,30 +3520,6 @@ restart_ih:
3496 */ 3520 */
3497#if defined(CONFIG_DEBUG_FS) 3521#if defined(CONFIG_DEBUG_FS)
3498 3522
3499static int r600_debugfs_cp_ring_info(struct seq_file *m, void *data)
3500{
3501 struct drm_info_node *node = (struct drm_info_node *) m->private;
3502 struct drm_device *dev = node->minor->dev;
3503 struct radeon_device *rdev = dev->dev_private;
3504 unsigned count, i, j;
3505
3506 radeon_ring_free_size(rdev);
3507 count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
3508 seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
3509 seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
3510 seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
3511 seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
3512 seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
3513 seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
3514 seq_printf(m, "%u dwords in ring\n", count);
3515 i = rdev->cp.rptr;
3516 for (j = 0; j <= count; j++) {
3517 seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
3518 i = (i + 1) & rdev->cp.ptr_mask;
3519 }
3520 return 0;
3521}
3522
3523static int r600_debugfs_mc_info(struct seq_file *m, void *data) 3523static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3524{ 3524{
3525 struct drm_info_node *node = (struct drm_info_node *) m->private; 3525 struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -3533,7 +3533,6 @@ static int r600_debugfs_mc_info(struct seq_file *m, void *data)
3533 3533
3534static struct drm_info_list r600_mc_info_list[] = { 3534static struct drm_info_list r600_mc_info_list[] = {
3535 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL}, 3535 {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
3536 {"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
3537}; 3536};
3538#endif 3537#endif
3539 3538
diff --git a/drivers/gpu/drm/radeon/r600_blit_kms.c b/drivers/gpu/drm/radeon/r600_blit_kms.c
index e09d2818f94..02a75747029 100644
--- a/drivers/gpu/drm/radeon/r600_blit_kms.c
+++ b/drivers/gpu/drm/radeon/r600_blit_kms.c
@@ -50,6 +50,7 @@ static void
50set_render_target(struct radeon_device *rdev, int format, 50set_render_target(struct radeon_device *rdev, int format,
51 int w, int h, u64 gpu_addr) 51 int w, int h, u64 gpu_addr)
52{ 52{
53 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
53 u32 cb_color_info; 54 u32 cb_color_info;
54 int pitch, slice; 55 int pitch, slice;
55 56
@@ -63,38 +64,38 @@ set_render_target(struct radeon_device *rdev, int format,
63 pitch = (w / 8) - 1; 64 pitch = (w / 8) - 1;
64 slice = ((w * h) / 64) - 1; 65 slice = ((w * h) / 64) - 1;
65 66
66 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 67 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
67 radeon_ring_write(rdev, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 68 radeon_ring_write(ring, (CB_COLOR0_BASE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
68 radeon_ring_write(rdev, gpu_addr >> 8); 69 radeon_ring_write(ring, gpu_addr >> 8);
69 70
70 if (rdev->family > CHIP_R600 && rdev->family < CHIP_RV770) { 71 if (rdev->family > CHIP_R600 && rdev->family < CHIP_RV770) {
71 radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_BASE_UPDATE, 0)); 72 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_BASE_UPDATE, 0));
72 radeon_ring_write(rdev, 2 << 0); 73 radeon_ring_write(ring, 2 << 0);
73 } 74 }
74 75
75 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 76 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
76 radeon_ring_write(rdev, (CB_COLOR0_SIZE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 77 radeon_ring_write(ring, (CB_COLOR0_SIZE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
77 radeon_ring_write(rdev, (pitch << 0) | (slice << 10)); 78 radeon_ring_write(ring, (pitch << 0) | (slice << 10));
78 79
79 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 80 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
80 radeon_ring_write(rdev, (CB_COLOR0_VIEW - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 81 radeon_ring_write(ring, (CB_COLOR0_VIEW - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
81 radeon_ring_write(rdev, 0); 82 radeon_ring_write(ring, 0);
82 83
83 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 84 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
84 radeon_ring_write(rdev, (CB_COLOR0_INFO - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 85 radeon_ring_write(ring, (CB_COLOR0_INFO - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
85 radeon_ring_write(rdev, cb_color_info); 86 radeon_ring_write(ring, cb_color_info);
86 87
87 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 88 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
88 radeon_ring_write(rdev, (CB_COLOR0_TILE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 89 radeon_ring_write(ring, (CB_COLOR0_TILE - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
89 radeon_ring_write(rdev, 0); 90 radeon_ring_write(ring, 0);
90 91
91 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 92 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
92 radeon_ring_write(rdev, (CB_COLOR0_FRAG - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 93 radeon_ring_write(ring, (CB_COLOR0_FRAG - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
93 radeon_ring_write(rdev, 0); 94 radeon_ring_write(ring, 0);
94 95
95 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 96 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
96 radeon_ring_write(rdev, (CB_COLOR0_MASK - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 97 radeon_ring_write(ring, (CB_COLOR0_MASK - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
97 radeon_ring_write(rdev, 0); 98 radeon_ring_write(ring, 0);
98} 99}
99 100
100/* emits 5dw */ 101/* emits 5dw */
@@ -103,6 +104,7 @@ cp_set_surface_sync(struct radeon_device *rdev,
103 u32 sync_type, u32 size, 104 u32 sync_type, u32 size,
104 u64 mc_addr) 105 u64 mc_addr)
105{ 106{
107 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
106 u32 cp_coher_size; 108 u32 cp_coher_size;
107 109
108 if (size == 0xffffffff) 110 if (size == 0xffffffff)
@@ -110,17 +112,18 @@ cp_set_surface_sync(struct radeon_device *rdev,
110 else 112 else
111 cp_coher_size = ((size + 255) >> 8); 113 cp_coher_size = ((size + 255) >> 8);
112 114
113 radeon_ring_write(rdev, PACKET3(PACKET3_SURFACE_SYNC, 3)); 115 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
114 radeon_ring_write(rdev, sync_type); 116 radeon_ring_write(ring, sync_type);
115 radeon_ring_write(rdev, cp_coher_size); 117 radeon_ring_write(ring, cp_coher_size);
116 radeon_ring_write(rdev, mc_addr >> 8); 118 radeon_ring_write(ring, mc_addr >> 8);
117 radeon_ring_write(rdev, 10); /* poll interval */ 119 radeon_ring_write(ring, 10); /* poll interval */
118} 120}
119 121
120/* emits 21dw + 1 surface sync = 26dw */ 122/* emits 21dw + 1 surface sync = 26dw */
121static void 123static void
122set_shaders(struct radeon_device *rdev) 124set_shaders(struct radeon_device *rdev)
123{ 125{
126 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
124 u64 gpu_addr; 127 u64 gpu_addr;
125 u32 sq_pgm_resources; 128 u32 sq_pgm_resources;
126 129
@@ -129,35 +132,35 @@ set_shaders(struct radeon_device *rdev)
129 132
130 /* VS */ 133 /* VS */
131 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; 134 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
132 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 135 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
133 radeon_ring_write(rdev, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 136 radeon_ring_write(ring, (SQ_PGM_START_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
134 radeon_ring_write(rdev, gpu_addr >> 8); 137 radeon_ring_write(ring, gpu_addr >> 8);
135 138
136 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 139 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
137 radeon_ring_write(rdev, (SQ_PGM_RESOURCES_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 140 radeon_ring_write(ring, (SQ_PGM_RESOURCES_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
138 radeon_ring_write(rdev, sq_pgm_resources); 141 radeon_ring_write(ring, sq_pgm_resources);
139 142
140 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 143 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
141 radeon_ring_write(rdev, (SQ_PGM_CF_OFFSET_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 144 radeon_ring_write(ring, (SQ_PGM_CF_OFFSET_VS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
142 radeon_ring_write(rdev, 0); 145 radeon_ring_write(ring, 0);
143 146
144 /* PS */ 147 /* PS */
145 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset; 148 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.ps_offset;
146 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 149 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
147 radeon_ring_write(rdev, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 150 radeon_ring_write(ring, (SQ_PGM_START_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
148 radeon_ring_write(rdev, gpu_addr >> 8); 151 radeon_ring_write(ring, gpu_addr >> 8);
149 152
150 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 153 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
151 radeon_ring_write(rdev, (SQ_PGM_RESOURCES_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 154 radeon_ring_write(ring, (SQ_PGM_RESOURCES_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
152 radeon_ring_write(rdev, sq_pgm_resources | (1 << 28)); 155 radeon_ring_write(ring, sq_pgm_resources | (1 << 28));
153 156
154 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 157 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
155 radeon_ring_write(rdev, (SQ_PGM_EXPORTS_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 158 radeon_ring_write(ring, (SQ_PGM_EXPORTS_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
156 radeon_ring_write(rdev, 2); 159 radeon_ring_write(ring, 2);
157 160
158 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 161 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
159 radeon_ring_write(rdev, (SQ_PGM_CF_OFFSET_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 162 radeon_ring_write(ring, (SQ_PGM_CF_OFFSET_PS - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
160 radeon_ring_write(rdev, 0); 163 radeon_ring_write(ring, 0);
161 164
162 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset; 165 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.vs_offset;
163 cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr); 166 cp_set_surface_sync(rdev, PACKET3_SH_ACTION_ENA, 512, gpu_addr);
@@ -167,6 +170,7 @@ set_shaders(struct radeon_device *rdev)
167static void 170static void
168set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr) 171set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
169{ 172{
173 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
170 u32 sq_vtx_constant_word2; 174 u32 sq_vtx_constant_word2;
171 175
172 sq_vtx_constant_word2 = SQ_VTXC_BASE_ADDR_HI(upper_32_bits(gpu_addr) & 0xff) | 176 sq_vtx_constant_word2 = SQ_VTXC_BASE_ADDR_HI(upper_32_bits(gpu_addr) & 0xff) |
@@ -175,15 +179,15 @@ set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
175 sq_vtx_constant_word2 |= SQ_VTXC_ENDIAN_SWAP(SQ_ENDIAN_8IN32); 179 sq_vtx_constant_word2 |= SQ_VTXC_ENDIAN_SWAP(SQ_ENDIAN_8IN32);
176#endif 180#endif
177 181
178 radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 7)); 182 radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 7));
179 radeon_ring_write(rdev, 0x460); 183 radeon_ring_write(ring, 0x460);
180 radeon_ring_write(rdev, gpu_addr & 0xffffffff); 184 radeon_ring_write(ring, gpu_addr & 0xffffffff);
181 radeon_ring_write(rdev, 48 - 1); 185 radeon_ring_write(ring, 48 - 1);
182 radeon_ring_write(rdev, sq_vtx_constant_word2); 186 radeon_ring_write(ring, sq_vtx_constant_word2);
183 radeon_ring_write(rdev, 1 << 0); 187 radeon_ring_write(ring, 1 << 0);
184 radeon_ring_write(rdev, 0); 188 radeon_ring_write(ring, 0);
185 radeon_ring_write(rdev, 0); 189 radeon_ring_write(ring, 0);
186 radeon_ring_write(rdev, SQ_TEX_VTX_VALID_BUFFER << 30); 190 radeon_ring_write(ring, SQ_TEX_VTX_VALID_BUFFER << 30);
187 191
188 if ((rdev->family == CHIP_RV610) || 192 if ((rdev->family == CHIP_RV610) ||
189 (rdev->family == CHIP_RV620) || 193 (rdev->family == CHIP_RV620) ||
@@ -203,6 +207,7 @@ set_tex_resource(struct radeon_device *rdev,
203 int format, int w, int h, int pitch, 207 int format, int w, int h, int pitch,
204 u64 gpu_addr, u32 size) 208 u64 gpu_addr, u32 size)
205{ 209{
210 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
206 uint32_t sq_tex_resource_word0, sq_tex_resource_word1, sq_tex_resource_word4; 211 uint32_t sq_tex_resource_word0, sq_tex_resource_word1, sq_tex_resource_word4;
207 212
208 if (h < 1) 213 if (h < 1)
@@ -225,15 +230,15 @@ set_tex_resource(struct radeon_device *rdev,
225 cp_set_surface_sync(rdev, 230 cp_set_surface_sync(rdev,
226 PACKET3_TC_ACTION_ENA, size, gpu_addr); 231 PACKET3_TC_ACTION_ENA, size, gpu_addr);
227 232
228 radeon_ring_write(rdev, PACKET3(PACKET3_SET_RESOURCE, 7)); 233 radeon_ring_write(ring, PACKET3(PACKET3_SET_RESOURCE, 7));
229 radeon_ring_write(rdev, 0); 234 radeon_ring_write(ring, 0);
230 radeon_ring_write(rdev, sq_tex_resource_word0); 235 radeon_ring_write(ring, sq_tex_resource_word0);
231 radeon_ring_write(rdev, sq_tex_resource_word1); 236 radeon_ring_write(ring, sq_tex_resource_word1);
232 radeon_ring_write(rdev, gpu_addr >> 8); 237 radeon_ring_write(ring, gpu_addr >> 8);
233 radeon_ring_write(rdev, gpu_addr >> 8); 238 radeon_ring_write(ring, gpu_addr >> 8);
234 radeon_ring_write(rdev, sq_tex_resource_word4); 239 radeon_ring_write(ring, sq_tex_resource_word4);
235 radeon_ring_write(rdev, 0); 240 radeon_ring_write(ring, 0);
236 radeon_ring_write(rdev, SQ_TEX_VTX_VALID_TEXTURE << 30); 241 radeon_ring_write(ring, SQ_TEX_VTX_VALID_TEXTURE << 30);
237} 242}
238 243
239/* emits 12 */ 244/* emits 12 */
@@ -241,43 +246,45 @@ static void
241set_scissors(struct radeon_device *rdev, int x1, int y1, 246set_scissors(struct radeon_device *rdev, int x1, int y1,
242 int x2, int y2) 247 int x2, int y2)
243{ 248{
244 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 249 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
245 radeon_ring_write(rdev, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 250 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
246 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16)); 251 radeon_ring_write(ring, (PA_SC_SCREEN_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
247 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 252 radeon_ring_write(ring, (x1 << 0) | (y1 << 16));
248 253 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
249 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 254
250 radeon_ring_write(rdev, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 255 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
251 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31)); 256 radeon_ring_write(ring, (PA_SC_GENERIC_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
252 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 257 radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31));
253 258 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
254 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 259
255 radeon_ring_write(rdev, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2); 260 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
256 radeon_ring_write(rdev, (x1 << 0) | (y1 << 16) | (1 << 31)); 261 radeon_ring_write(ring, (PA_SC_WINDOW_SCISSOR_TL - PACKET3_SET_CONTEXT_REG_OFFSET) >> 2);
257 radeon_ring_write(rdev, (x2 << 0) | (y2 << 16)); 262 radeon_ring_write(ring, (x1 << 0) | (y1 << 16) | (1 << 31));
263 radeon_ring_write(ring, (x2 << 0) | (y2 << 16));
258} 264}
259 265
260/* emits 10 */ 266/* emits 10 */
261static void 267static void
262draw_auto(struct radeon_device *rdev) 268draw_auto(struct radeon_device *rdev)
263{ 269{
264 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 270 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
265 radeon_ring_write(rdev, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); 271 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
266 radeon_ring_write(rdev, DI_PT_RECTLIST); 272 radeon_ring_write(ring, (VGT_PRIMITIVE_TYPE - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
273 radeon_ring_write(ring, DI_PT_RECTLIST);
267 274
268 radeon_ring_write(rdev, PACKET3(PACKET3_INDEX_TYPE, 0)); 275 radeon_ring_write(ring, PACKET3(PACKET3_INDEX_TYPE, 0));
269 radeon_ring_write(rdev, 276 radeon_ring_write(ring,
270#ifdef __BIG_ENDIAN 277#ifdef __BIG_ENDIAN
271 (2 << 2) | 278 (2 << 2) |
272#endif 279#endif
273 DI_INDEX_SIZE_16_BIT); 280 DI_INDEX_SIZE_16_BIT);
274 281
275 radeon_ring_write(rdev, PACKET3(PACKET3_NUM_INSTANCES, 0)); 282 radeon_ring_write(ring, PACKET3(PACKET3_NUM_INSTANCES, 0));
276 radeon_ring_write(rdev, 1); 283 radeon_ring_write(ring, 1);
277 284
278 radeon_ring_write(rdev, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1)); 285 radeon_ring_write(ring, PACKET3(PACKET3_DRAW_INDEX_AUTO, 1));
279 radeon_ring_write(rdev, 3); 286 radeon_ring_write(ring, 3);
280 radeon_ring_write(rdev, DI_SRC_SEL_AUTO_INDEX); 287 radeon_ring_write(ring, DI_SRC_SEL_AUTO_INDEX);
281 288
282} 289}
283 290
@@ -285,6 +292,7 @@ draw_auto(struct radeon_device *rdev)
285static void 292static void
286set_default_state(struct radeon_device *rdev) 293set_default_state(struct radeon_device *rdev)
287{ 294{
295 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
288 u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2; 296 u32 sq_config, sq_gpr_resource_mgmt_1, sq_gpr_resource_mgmt_2;
289 u32 sq_thread_resource_mgmt, sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2; 297 u32 sq_thread_resource_mgmt, sq_stack_resource_mgmt_1, sq_stack_resource_mgmt_2;
290 int num_ps_gprs, num_vs_gprs, num_temp_gprs, num_gs_gprs, num_es_gprs; 298 int num_ps_gprs, num_vs_gprs, num_temp_gprs, num_gs_gprs, num_es_gprs;
@@ -440,24 +448,24 @@ set_default_state(struct radeon_device *rdev)
440 /* emit an IB pointing at default state */ 448 /* emit an IB pointing at default state */
441 dwords = ALIGN(rdev->r600_blit.state_len, 0x10); 449 dwords = ALIGN(rdev->r600_blit.state_len, 0x10);
442 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset; 450 gpu_addr = rdev->r600_blit.shader_gpu_addr + rdev->r600_blit.state_offset;
443 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 451 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
444 radeon_ring_write(rdev, 452 radeon_ring_write(ring,
445#ifdef __BIG_ENDIAN 453#ifdef __BIG_ENDIAN
446 (2 << 0) | 454 (2 << 0) |
447#endif 455#endif
448 (gpu_addr & 0xFFFFFFFC)); 456 (gpu_addr & 0xFFFFFFFC));
449 radeon_ring_write(rdev, upper_32_bits(gpu_addr) & 0xFF); 457 radeon_ring_write(ring, upper_32_bits(gpu_addr) & 0xFF);
450 radeon_ring_write(rdev, dwords); 458 radeon_ring_write(ring, dwords);
451 459
452 /* SQ config */ 460 /* SQ config */
453 radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 6)); 461 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 6));
454 radeon_ring_write(rdev, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); 462 radeon_ring_write(ring, (SQ_CONFIG - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
455 radeon_ring_write(rdev, sq_config); 463 radeon_ring_write(ring, sq_config);
456 radeon_ring_write(rdev, sq_gpr_resource_mgmt_1); 464 radeon_ring_write(ring, sq_gpr_resource_mgmt_1);
457 radeon_ring_write(rdev, sq_gpr_resource_mgmt_2); 465 radeon_ring_write(ring, sq_gpr_resource_mgmt_2);
458 radeon_ring_write(rdev, sq_thread_resource_mgmt); 466 radeon_ring_write(ring, sq_thread_resource_mgmt);
459 radeon_ring_write(rdev, sq_stack_resource_mgmt_1); 467 radeon_ring_write(ring, sq_stack_resource_mgmt_1);
460 radeon_ring_write(rdev, sq_stack_resource_mgmt_2); 468 radeon_ring_write(ring, sq_stack_resource_mgmt_2);
461} 469}
462 470
463static uint32_t i2f(uint32_t input) 471static uint32_t i2f(uint32_t input)
@@ -614,7 +622,7 @@ void r600_blit_fini(struct radeon_device *rdev)
614static int r600_vb_ib_get(struct radeon_device *rdev) 622static int r600_vb_ib_get(struct radeon_device *rdev)
615{ 623{
616 int r; 624 int r;
617 r = radeon_ib_get(rdev, &rdev->r600_blit.vb_ib); 625 r = radeon_ib_get(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->r600_blit.vb_ib);
618 if (r) { 626 if (r) {
619 DRM_ERROR("failed to get IB for vertex buffer\n"); 627 DRM_ERROR("failed to get IB for vertex buffer\n");
620 return r; 628 return r;
@@ -679,6 +687,7 @@ static unsigned r600_blit_create_rect(unsigned num_gpu_pages,
679 687
680int r600_blit_prepare_copy(struct radeon_device *rdev, unsigned num_gpu_pages) 688int r600_blit_prepare_copy(struct radeon_device *rdev, unsigned num_gpu_pages)
681{ 689{
690 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
682 int r; 691 int r;
683 int ring_size; 692 int ring_size;
684 int num_loops = 0; 693 int num_loops = 0;
@@ -699,7 +708,7 @@ int r600_blit_prepare_copy(struct radeon_device *rdev, unsigned num_gpu_pages)
699 /* calculate number of loops correctly */ 708 /* calculate number of loops correctly */
700 ring_size = num_loops * dwords_per_loop; 709 ring_size = num_loops * dwords_per_loop;
701 ring_size += rdev->r600_blit.ring_size_common; 710 ring_size += rdev->r600_blit.ring_size_common;
702 r = radeon_ring_lock(rdev, ring_size); 711 r = radeon_ring_lock(rdev, ring, ring_size);
703 if (r) 712 if (r)
704 return r; 713 return r;
705 714
@@ -718,7 +727,7 @@ void r600_blit_done_copy(struct radeon_device *rdev, struct radeon_fence *fence)
718 if (fence) 727 if (fence)
719 r = radeon_fence_emit(rdev, fence); 728 r = radeon_fence_emit(rdev, fence);
720 729
721 radeon_ring_unlock_commit(rdev); 730 radeon_ring_unlock_commit(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
722} 731}
723 732
724void r600_kms_blit_copy(struct radeon_device *rdev, 733void r600_kms_blit_copy(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/r600_cp.c b/drivers/gpu/drm/radeon/r600_cp.c
index c9db4931913..84c54625095 100644
--- a/drivers/gpu/drm/radeon/r600_cp.c
+++ b/drivers/gpu/drm/radeon/r600_cp.c
@@ -1815,7 +1815,7 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
1815 dev_priv->ring.size_l2qw); 1815 dev_priv->ring.size_l2qw);
1816#endif 1816#endif
1817 1817
1818 RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x4); 1818 RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x0);
1819 1819
1820 /* Set the write pointer delay */ 1820 /* Set the write pointer delay */
1821 RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0); 1821 RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index bfe1b5d92af..3ee1fd7ef39 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -831,6 +831,8 @@
831#define PACKET3_STRMOUT_BUFFER_UPDATE 0x34 831#define PACKET3_STRMOUT_BUFFER_UPDATE 0x34
832#define PACKET3_INDIRECT_BUFFER_MP 0x38 832#define PACKET3_INDIRECT_BUFFER_MP 0x38
833#define PACKET3_MEM_SEMAPHORE 0x39 833#define PACKET3_MEM_SEMAPHORE 0x39
834# define PACKET3_SEM_SEL_SIGNAL (0x6 << 29)
835# define PACKET3_SEM_SEL_WAIT (0x7 << 29)
834#define PACKET3_MPEG_INDEX 0x3A 836#define PACKET3_MPEG_INDEX 0x3A
835#define PACKET3_WAIT_REG_MEM 0x3C 837#define PACKET3_WAIT_REG_MEM 0x3C
836#define PACKET3_MEM_WRITE 0x3D 838#define PACKET3_MEM_WRITE 0x3D
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index c8f4dbd2d17..f29edbf6296 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -107,6 +107,17 @@ extern int radeon_msi;
107#define RADEONFB_CONN_LIMIT 4 107#define RADEONFB_CONN_LIMIT 4
108#define RADEON_BIOS_NUM_SCRATCH 8 108#define RADEON_BIOS_NUM_SCRATCH 8
109 109
110/* max number of rings */
111#define RADEON_NUM_RINGS 3
112
113/* internal ring indices */
114/* r1xx+ has gfx CP ring */
115#define RADEON_RING_TYPE_GFX_INDEX 0
116
117/* cayman has 2 compute CP rings */
118#define CAYMAN_RING_TYPE_CP1_INDEX 1
119#define CAYMAN_RING_TYPE_CP2_INDEX 2
120
110/* 121/*
111 * Errata workarounds. 122 * Errata workarounds.
112 */ 123 */
@@ -192,14 +203,15 @@ extern int sumo_get_temp(struct radeon_device *rdev);
192 */ 203 */
193struct radeon_fence_driver { 204struct radeon_fence_driver {
194 uint32_t scratch_reg; 205 uint32_t scratch_reg;
206 uint64_t gpu_addr;
207 volatile uint32_t *cpu_addr;
195 atomic_t seq; 208 atomic_t seq;
196 uint32_t last_seq; 209 uint32_t last_seq;
197 unsigned long last_jiffies; 210 unsigned long last_jiffies;
198 unsigned long last_timeout; 211 unsigned long last_timeout;
199 wait_queue_head_t queue; 212 wait_queue_head_t queue;
200 rwlock_t lock;
201 struct list_head created; 213 struct list_head created;
202 struct list_head emited; 214 struct list_head emitted;
203 struct list_head signaled; 215 struct list_head signaled;
204 bool initialized; 216 bool initialized;
205}; 217};
@@ -210,21 +222,51 @@ struct radeon_fence {
210 struct list_head list; 222 struct list_head list;
211 /* protected by radeon_fence.lock */ 223 /* protected by radeon_fence.lock */
212 uint32_t seq; 224 uint32_t seq;
213 bool emited; 225 bool emitted;
214 bool signaled; 226 bool signaled;
227 /* RB, DMA, etc. */
228 int ring;
215}; 229};
216 230
231int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
217int radeon_fence_driver_init(struct radeon_device *rdev); 232int radeon_fence_driver_init(struct radeon_device *rdev);
218void radeon_fence_driver_fini(struct radeon_device *rdev); 233void radeon_fence_driver_fini(struct radeon_device *rdev);
219int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence); 234int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence, int ring);
220int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence); 235int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence);
221void radeon_fence_process(struct radeon_device *rdev); 236void radeon_fence_process(struct radeon_device *rdev, int ring);
222bool radeon_fence_signaled(struct radeon_fence *fence); 237bool radeon_fence_signaled(struct radeon_fence *fence);
223int radeon_fence_wait(struct radeon_fence *fence, bool interruptible); 238int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
224int radeon_fence_wait_next(struct radeon_device *rdev); 239int radeon_fence_wait_next(struct radeon_device *rdev, int ring);
225int radeon_fence_wait_last(struct radeon_device *rdev); 240int radeon_fence_wait_last(struct radeon_device *rdev, int ring);
226struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence); 241struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence);
227void radeon_fence_unref(struct radeon_fence **fence); 242void radeon_fence_unref(struct radeon_fence **fence);
243int radeon_fence_count_emitted(struct radeon_device *rdev, int ring);
244
245/*
246 * Semaphores.
247 */
248struct radeon_ring;
249
250struct radeon_semaphore_driver {
251 rwlock_t lock;
252 struct list_head free;
253};
254
255struct radeon_semaphore {
256 struct radeon_bo *robj;
257 struct list_head list;
258 uint64_t gpu_addr;
259};
260
261void radeon_semaphore_driver_fini(struct radeon_device *rdev);
262int radeon_semaphore_create(struct radeon_device *rdev,
263 struct radeon_semaphore **semaphore);
264void radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
265 struct radeon_semaphore *semaphore);
266void radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
267 struct radeon_semaphore *semaphore);
268void radeon_semaphore_free(struct radeon_device *rdev,
269 struct radeon_semaphore *semaphore);
228 270
229/* 271/*
230 * Tiling registers 272 * Tiling registers
@@ -274,6 +316,48 @@ struct radeon_bo_list {
274 u32 tiling_flags; 316 u32 tiling_flags;
275}; 317};
276 318
319/* sub-allocation manager, it has to be protected by another lock.
320 * By conception this is an helper for other part of the driver
321 * like the indirect buffer or semaphore, which both have their
322 * locking.
323 *
324 * Principe is simple, we keep a list of sub allocation in offset
325 * order (first entry has offset == 0, last entry has the highest
326 * offset).
327 *
328 * When allocating new object we first check if there is room at
329 * the end total_size - (last_object_offset + last_object_size) >=
330 * alloc_size. If so we allocate new object there.
331 *
332 * When there is not enough room at the end, we start waiting for
333 * each sub object until we reach object_offset+object_size >=
334 * alloc_size, this object then become the sub object we return.
335 *
336 * Alignment can't be bigger than page size.
337 *
338 * Hole are not considered for allocation to keep things simple.
339 * Assumption is that there won't be hole (all object on same
340 * alignment).
341 */
342struct radeon_sa_manager {
343 struct radeon_bo *bo;
344 struct list_head sa_bo;
345 unsigned size;
346 uint64_t gpu_addr;
347 void *cpu_ptr;
348 uint32_t domain;
349};
350
351struct radeon_sa_bo;
352
353/* sub-allocation buffer */
354struct radeon_sa_bo {
355 struct list_head list;
356 struct radeon_sa_manager *manager;
357 unsigned offset;
358 unsigned size;
359};
360
277/* 361/*
278 * GEM objects. 362 * GEM objects.
279 */ 363 */
@@ -433,7 +517,7 @@ union radeon_irq_stat_regs {
433 517
434struct radeon_irq { 518struct radeon_irq {
435 bool installed; 519 bool installed;
436 bool sw_int; 520 bool sw_int[RADEON_NUM_RINGS];
437 bool crtc_vblank_int[RADEON_MAX_CRTCS]; 521 bool crtc_vblank_int[RADEON_MAX_CRTCS];
438 bool pflip[RADEON_MAX_CRTCS]; 522 bool pflip[RADEON_MAX_CRTCS];
439 wait_queue_head_t vblank_queue; 523 wait_queue_head_t vblank_queue;
@@ -443,7 +527,7 @@ struct radeon_irq {
443 wait_queue_head_t idle_queue; 527 wait_queue_head_t idle_queue;
444 bool hdmi[RADEON_MAX_HDMI_BLOCKS]; 528 bool hdmi[RADEON_MAX_HDMI_BLOCKS];
445 spinlock_t sw_lock; 529 spinlock_t sw_lock;
446 int sw_refcount; 530 int sw_refcount[RADEON_NUM_RINGS];
447 union radeon_irq_stat_regs stat_regs; 531 union radeon_irq_stat_regs stat_regs;
448 spinlock_t pflip_lock[RADEON_MAX_CRTCS]; 532 spinlock_t pflip_lock[RADEON_MAX_CRTCS];
449 int pflip_refcount[RADEON_MAX_CRTCS]; 533 int pflip_refcount[RADEON_MAX_CRTCS];
@@ -451,22 +535,22 @@ struct radeon_irq {
451 535
452int radeon_irq_kms_init(struct radeon_device *rdev); 536int radeon_irq_kms_init(struct radeon_device *rdev);
453void radeon_irq_kms_fini(struct radeon_device *rdev); 537void radeon_irq_kms_fini(struct radeon_device *rdev);
454void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev); 538void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring);
455void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev); 539void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring);
456void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc); 540void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
457void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); 541void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
458 542
459/* 543/*
460 * CP & ring. 544 * CP & rings.
461 */ 545 */
546
462struct radeon_ib { 547struct radeon_ib {
463 struct list_head list; 548 struct radeon_sa_bo sa_bo;
464 unsigned idx; 549 unsigned idx;
550 uint32_t length_dw;
465 uint64_t gpu_addr; 551 uint64_t gpu_addr;
466 struct radeon_fence *fence;
467 uint32_t *ptr; 552 uint32_t *ptr;
468 uint32_t length_dw; 553 struct radeon_fence *fence;
469 bool free;
470}; 554};
471 555
472/* 556/*
@@ -474,20 +558,22 @@ struct radeon_ib {
474 * mutex protects scheduled_ibs, ready, alloc_bm 558 * mutex protects scheduled_ibs, ready, alloc_bm
475 */ 559 */
476struct radeon_ib_pool { 560struct radeon_ib_pool {
477 struct mutex mutex; 561 struct mutex mutex;
478 struct radeon_bo *robj; 562 struct radeon_sa_manager sa_manager;
479 struct list_head bogus_ib; 563 struct radeon_ib ibs[RADEON_IB_POOL_SIZE];
480 struct radeon_ib ibs[RADEON_IB_POOL_SIZE]; 564 bool ready;
481 bool ready; 565 unsigned head_id;
482 unsigned head_id;
483}; 566};
484 567
485struct radeon_cp { 568struct radeon_ring {
486 struct radeon_bo *ring_obj; 569 struct radeon_bo *ring_obj;
487 volatile uint32_t *ring; 570 volatile uint32_t *ring;
488 unsigned rptr; 571 unsigned rptr;
572 unsigned rptr_offs;
573 unsigned rptr_reg;
489 unsigned wptr; 574 unsigned wptr;
490 unsigned wptr_old; 575 unsigned wptr_old;
576 unsigned wptr_reg;
491 unsigned ring_size; 577 unsigned ring_size;
492 unsigned ring_free_dw; 578 unsigned ring_free_dw;
493 int count_dw; 579 int count_dw;
@@ -496,6 +582,9 @@ struct radeon_cp {
496 uint32_t ptr_mask; 582 uint32_t ptr_mask;
497 struct mutex mutex; 583 struct mutex mutex;
498 bool ready; 584 bool ready;
585 u32 ptr_reg_shift;
586 u32 ptr_reg_mask;
587 u32 nop;
499}; 588};
500 589
501/* 590/*
@@ -505,6 +594,7 @@ struct r600_ih {
505 struct radeon_bo *ring_obj; 594 struct radeon_bo *ring_obj;
506 volatile uint32_t *ring; 595 volatile uint32_t *ring;
507 unsigned rptr; 596 unsigned rptr;
597 unsigned rptr_offs;
508 unsigned wptr; 598 unsigned wptr;
509 unsigned wptr_old; 599 unsigned wptr_old;
510 unsigned ring_size; 600 unsigned ring_size;
@@ -548,23 +638,27 @@ struct r600_blit {
548 638
549void r600_blit_suspend(struct radeon_device *rdev); 639void r600_blit_suspend(struct radeon_device *rdev);
550 640
551int radeon_ib_get(struct radeon_device *rdev, struct radeon_ib **ib); 641int radeon_ib_get(struct radeon_device *rdev, int ring, struct radeon_ib **ib);
552void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib); 642void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib);
553int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib); 643int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib);
554int radeon_ib_pool_init(struct radeon_device *rdev); 644int radeon_ib_pool_init(struct radeon_device *rdev);
555void radeon_ib_pool_fini(struct radeon_device *rdev); 645void radeon_ib_pool_fini(struct radeon_device *rdev);
646int radeon_ib_pool_start(struct radeon_device *rdev);
647int radeon_ib_pool_suspend(struct radeon_device *rdev);
556int radeon_ib_test(struct radeon_device *rdev); 648int radeon_ib_test(struct radeon_device *rdev);
557extern void radeon_ib_bogus_add(struct radeon_device *rdev, struct radeon_ib *ib);
558/* Ring access between begin & end cannot sleep */ 649/* Ring access between begin & end cannot sleep */
559void radeon_ring_free_size(struct radeon_device *rdev); 650int radeon_ring_index(struct radeon_device *rdev, struct radeon_ring *cp);
560int radeon_ring_alloc(struct radeon_device *rdev, unsigned ndw); 651void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *cp);
561int radeon_ring_lock(struct radeon_device *rdev, unsigned ndw); 652int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
562void radeon_ring_commit(struct radeon_device *rdev); 653int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ndw);
563void radeon_ring_unlock_commit(struct radeon_device *rdev); 654void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *cp);
564void radeon_ring_unlock_undo(struct radeon_device *rdev); 655void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *cp);
565int radeon_ring_test(struct radeon_device *rdev); 656void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp);
566int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size); 657int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
567void radeon_ring_fini(struct radeon_device *rdev); 658int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size,
659 unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg,
660 u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop);
661void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *cp);
568 662
569 663
570/* 664/*
@@ -868,11 +962,20 @@ void radeon_benchmark(struct radeon_device *rdev, int test_number);
868 * Testing 962 * Testing
869 */ 963 */
870void radeon_test_moves(struct radeon_device *rdev); 964void radeon_test_moves(struct radeon_device *rdev);
965void radeon_test_ring_sync(struct radeon_device *rdev,
966 struct radeon_ring *cpA,
967 struct radeon_ring *cpB);
968void radeon_test_syncing(struct radeon_device *rdev);
871 969
872 970
873/* 971/*
874 * Debugfs 972 * Debugfs
875 */ 973 */
974struct radeon_debugfs {
975 struct drm_info_list *files;
976 unsigned num_files;
977};
978
876int radeon_debugfs_add_files(struct radeon_device *rdev, 979int radeon_debugfs_add_files(struct radeon_device *rdev,
877 struct drm_info_list *files, 980 struct drm_info_list *files,
878 unsigned nfiles); 981 unsigned nfiles);
@@ -888,21 +991,26 @@ struct radeon_asic {
888 int (*resume)(struct radeon_device *rdev); 991 int (*resume)(struct radeon_device *rdev);
889 int (*suspend)(struct radeon_device *rdev); 992 int (*suspend)(struct radeon_device *rdev);
890 void (*vga_set_state)(struct radeon_device *rdev, bool state); 993 void (*vga_set_state)(struct radeon_device *rdev, bool state);
891 bool (*gpu_is_lockup)(struct radeon_device *rdev); 994 bool (*gpu_is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
892 int (*asic_reset)(struct radeon_device *rdev); 995 int (*asic_reset)(struct radeon_device *rdev);
893 void (*gart_tlb_flush)(struct radeon_device *rdev); 996 void (*gart_tlb_flush)(struct radeon_device *rdev);
894 int (*gart_set_page)(struct radeon_device *rdev, int i, uint64_t addr); 997 int (*gart_set_page)(struct radeon_device *rdev, int i, uint64_t addr);
895 int (*cp_init)(struct radeon_device *rdev, unsigned ring_size); 998 int (*cp_init)(struct radeon_device *rdev, unsigned ring_size);
896 void (*cp_fini)(struct radeon_device *rdev); 999 void (*cp_fini)(struct radeon_device *rdev);
897 void (*cp_disable)(struct radeon_device *rdev); 1000 void (*cp_disable)(struct radeon_device *rdev);
898 void (*cp_commit)(struct radeon_device *rdev);
899 void (*ring_start)(struct radeon_device *rdev); 1001 void (*ring_start)(struct radeon_device *rdev);
900 int (*ring_test)(struct radeon_device *rdev); 1002
901 void (*ring_ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib); 1003 struct {
1004 void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib);
1005 void (*emit_fence)(struct radeon_device *rdev, struct radeon_fence *fence);
1006 void (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp,
1007 struct radeon_semaphore *semaphore, bool emit_wait);
1008 } ring[RADEON_NUM_RINGS];
1009
1010 int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
902 int (*irq_set)(struct radeon_device *rdev); 1011 int (*irq_set)(struct radeon_device *rdev);
903 int (*irq_process)(struct radeon_device *rdev); 1012 int (*irq_process)(struct radeon_device *rdev);
904 u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc); 1013 u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
905 void (*fence_ring_emit)(struct radeon_device *rdev, struct radeon_fence *fence);
906 int (*cs_parse)(struct radeon_cs_parser *p); 1014 int (*cs_parse)(struct radeon_cs_parser *p);
907 int (*copy_blit)(struct radeon_device *rdev, 1015 int (*copy_blit)(struct radeon_device *rdev,
908 uint64_t src_offset, 1016 uint64_t src_offset,
@@ -1230,11 +1338,10 @@ struct radeon_device {
1230 struct radeon_mode_info mode_info; 1338 struct radeon_mode_info mode_info;
1231 struct radeon_scratch scratch; 1339 struct radeon_scratch scratch;
1232 struct radeon_mman mman; 1340 struct radeon_mman mman;
1233 struct radeon_fence_driver fence_drv; 1341 rwlock_t fence_lock;
1234 struct radeon_cp cp; 1342 struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS];
1235 /* cayman compute rings */ 1343 struct radeon_semaphore_driver semaphore_drv;
1236 struct radeon_cp cp1; 1344 struct radeon_ring ring[RADEON_NUM_RINGS];
1237 struct radeon_cp cp2;
1238 struct radeon_ib_pool ib_pool; 1345 struct radeon_ib_pool ib_pool;
1239 struct radeon_irq irq; 1346 struct radeon_irq irq;
1240 struct radeon_asic *asic; 1347 struct radeon_asic *asic;
@@ -1278,6 +1385,9 @@ struct radeon_device {
1278 struct drm_file *cmask_filp; 1385 struct drm_file *cmask_filp;
1279 /* i2c buses */ 1386 /* i2c buses */
1280 struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS]; 1387 struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
1388 /* debugfs */
1389 struct radeon_debugfs debugfs[RADEON_DEBUGFS_MAX_COMPONENTS];
1390 unsigned debugfs_count;
1281}; 1391};
1282 1392
1283int radeon_device_init(struct radeon_device *rdev, 1393int radeon_device_init(struct radeon_device *rdev,
@@ -1413,18 +1523,17 @@ void radeon_atombios_fini(struct radeon_device *rdev);
1413/* 1523/*
1414 * RING helpers. 1524 * RING helpers.
1415 */ 1525 */
1416
1417#if DRM_DEBUG_CODE == 0 1526#if DRM_DEBUG_CODE == 0
1418static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v) 1527static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
1419{ 1528{
1420 rdev->cp.ring[rdev->cp.wptr++] = v; 1529 ring->ring[ring->wptr++] = v;
1421 rdev->cp.wptr &= rdev->cp.ptr_mask; 1530 ring->wptr &= ring->ptr_mask;
1422 rdev->cp.count_dw--; 1531 ring->count_dw--;
1423 rdev->cp.ring_free_dw--; 1532 ring->ring_free_dw--;
1424} 1533}
1425#else 1534#else
1426/* With debugging this is just too big to inline */ 1535/* With debugging this is just too big to inline */
1427void radeon_ring_write(struct radeon_device *rdev, uint32_t v); 1536void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
1428#endif 1537#endif
1429 1538
1430/* 1539/*
@@ -1436,18 +1545,18 @@ void radeon_ring_write(struct radeon_device *rdev, uint32_t v);
1436#define radeon_suspend(rdev) (rdev)->asic->suspend((rdev)) 1545#define radeon_suspend(rdev) (rdev)->asic->suspend((rdev))
1437#define radeon_cs_parse(p) rdev->asic->cs_parse((p)) 1546#define radeon_cs_parse(p) rdev->asic->cs_parse((p))
1438#define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state)) 1547#define radeon_vga_set_state(rdev, state) (rdev)->asic->vga_set_state((rdev), (state))
1439#define radeon_gpu_is_lockup(rdev) (rdev)->asic->gpu_is_lockup((rdev)) 1548#define radeon_gpu_is_lockup(rdev, cp) (rdev)->asic->gpu_is_lockup((rdev), (cp))
1440#define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev)) 1549#define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev))
1441#define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart_tlb_flush((rdev)) 1550#define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart_tlb_flush((rdev))
1442#define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart_set_page((rdev), (i), (p)) 1551#define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart_set_page((rdev), (i), (p))
1443#define radeon_cp_commit(rdev) (rdev)->asic->cp_commit((rdev))
1444#define radeon_ring_start(rdev) (rdev)->asic->ring_start((rdev)) 1552#define radeon_ring_start(rdev) (rdev)->asic->ring_start((rdev))
1445#define radeon_ring_test(rdev) (rdev)->asic->ring_test((rdev)) 1553#define radeon_ring_test(rdev, cp) (rdev)->asic->ring_test((rdev), (cp))
1446#define radeon_ring_ib_execute(rdev, ib) (rdev)->asic->ring_ib_execute((rdev), (ib)) 1554#define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)].ib_execute((rdev), (ib))
1447#define radeon_irq_set(rdev) (rdev)->asic->irq_set((rdev)) 1555#define radeon_irq_set(rdev) (rdev)->asic->irq_set((rdev))
1448#define radeon_irq_process(rdev) (rdev)->asic->irq_process((rdev)) 1556#define radeon_irq_process(rdev) (rdev)->asic->irq_process((rdev))
1449#define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->get_vblank_counter((rdev), (crtc)) 1557#define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->get_vblank_counter((rdev), (crtc))
1450#define radeon_fence_ring_emit(rdev, fence) (rdev)->asic->fence_ring_emit((rdev), (fence)) 1558#define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)].emit_fence((rdev), (fence))
1559#define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)].emit_semaphore((rdev), (cp), (semaphore), (emit_wait))
1451#define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy_blit((rdev), (s), (d), (np), (f)) 1560#define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy_blit((rdev), (s), (d), (np), (f))
1452#define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy_dma((rdev), (s), (d), (np), (f)) 1561#define radeon_copy_dma(rdev, s, d, np, f) (rdev)->asic->copy_dma((rdev), (s), (d), (np), (f))
1453#define radeon_copy(rdev, s, d, np, f) (rdev)->asic->copy((rdev), (s), (d), (np), (f)) 1562#define radeon_copy(rdev, s, d, np, f) (rdev)->asic->copy((rdev), (s), (d), (np), (f))
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index a2e1eae114e..8493d406f5e 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -138,14 +138,18 @@ static struct radeon_asic r100_asic = {
138 .asic_reset = &r100_asic_reset, 138 .asic_reset = &r100_asic_reset,
139 .gart_tlb_flush = &r100_pci_gart_tlb_flush, 139 .gart_tlb_flush = &r100_pci_gart_tlb_flush,
140 .gart_set_page = &r100_pci_gart_set_page, 140 .gart_set_page = &r100_pci_gart_set_page,
141 .cp_commit = &r100_cp_commit,
142 .ring_start = &r100_ring_start, 141 .ring_start = &r100_ring_start,
143 .ring_test = &r100_ring_test, 142 .ring_test = &r100_ring_test,
144 .ring_ib_execute = &r100_ring_ib_execute, 143 .ring = {
144 [RADEON_RING_TYPE_GFX_INDEX] = {
145 .ib_execute = &r100_ring_ib_execute,
146 .emit_fence = &r100_fence_ring_emit,
147 .emit_semaphore = &r100_semaphore_ring_emit,
148 }
149 },
145 .irq_set = &r100_irq_set, 150 .irq_set = &r100_irq_set,
146 .irq_process = &r100_irq_process, 151 .irq_process = &r100_irq_process,
147 .get_vblank_counter = &r100_get_vblank_counter, 152 .get_vblank_counter = &r100_get_vblank_counter,
148 .fence_ring_emit = &r100_fence_ring_emit,
149 .cs_parse = &r100_cs_parse, 153 .cs_parse = &r100_cs_parse,
150 .copy_blit = &r100_copy_blit, 154 .copy_blit = &r100_copy_blit,
151 .copy_dma = NULL, 155 .copy_dma = NULL,
@@ -186,14 +190,18 @@ static struct radeon_asic r200_asic = {
186 .asic_reset = &r100_asic_reset, 190 .asic_reset = &r100_asic_reset,
187 .gart_tlb_flush = &r100_pci_gart_tlb_flush, 191 .gart_tlb_flush = &r100_pci_gart_tlb_flush,
188 .gart_set_page = &r100_pci_gart_set_page, 192 .gart_set_page = &r100_pci_gart_set_page,
189 .cp_commit = &r100_cp_commit,
190 .ring_start = &r100_ring_start, 193 .ring_start = &r100_ring_start,
191 .ring_test = &r100_ring_test, 194 .ring_test = &r100_ring_test,
192 .ring_ib_execute = &r100_ring_ib_execute, 195 .ring = {
196 [RADEON_RING_TYPE_GFX_INDEX] = {
197 .ib_execute = &r100_ring_ib_execute,
198 .emit_fence = &r100_fence_ring_emit,
199 .emit_semaphore = &r100_semaphore_ring_emit,
200 }
201 },
193 .irq_set = &r100_irq_set, 202 .irq_set = &r100_irq_set,
194 .irq_process = &r100_irq_process, 203 .irq_process = &r100_irq_process,
195 .get_vblank_counter = &r100_get_vblank_counter, 204 .get_vblank_counter = &r100_get_vblank_counter,
196 .fence_ring_emit = &r100_fence_ring_emit,
197 .cs_parse = &r100_cs_parse, 205 .cs_parse = &r100_cs_parse,
198 .copy_blit = &r100_copy_blit, 206 .copy_blit = &r100_copy_blit,
199 .copy_dma = &r200_copy_dma, 207 .copy_dma = &r200_copy_dma,
@@ -233,14 +241,18 @@ static struct radeon_asic r300_asic = {
233 .asic_reset = &r300_asic_reset, 241 .asic_reset = &r300_asic_reset,
234 .gart_tlb_flush = &r100_pci_gart_tlb_flush, 242 .gart_tlb_flush = &r100_pci_gart_tlb_flush,
235 .gart_set_page = &r100_pci_gart_set_page, 243 .gart_set_page = &r100_pci_gart_set_page,
236 .cp_commit = &r100_cp_commit,
237 .ring_start = &r300_ring_start, 244 .ring_start = &r300_ring_start,
238 .ring_test = &r100_ring_test, 245 .ring_test = &r100_ring_test,
239 .ring_ib_execute = &r100_ring_ib_execute, 246 .ring = {
247 [RADEON_RING_TYPE_GFX_INDEX] = {
248 .ib_execute = &r100_ring_ib_execute,
249 .emit_fence = &r300_fence_ring_emit,
250 .emit_semaphore = &r100_semaphore_ring_emit,
251 }
252 },
240 .irq_set = &r100_irq_set, 253 .irq_set = &r100_irq_set,
241 .irq_process = &r100_irq_process, 254 .irq_process = &r100_irq_process,
242 .get_vblank_counter = &r100_get_vblank_counter, 255 .get_vblank_counter = &r100_get_vblank_counter,
243 .fence_ring_emit = &r300_fence_ring_emit,
244 .cs_parse = &r300_cs_parse, 256 .cs_parse = &r300_cs_parse,
245 .copy_blit = &r100_copy_blit, 257 .copy_blit = &r100_copy_blit,
246 .copy_dma = &r200_copy_dma, 258 .copy_dma = &r200_copy_dma,
@@ -281,14 +293,18 @@ static struct radeon_asic r300_asic_pcie = {
281 .asic_reset = &r300_asic_reset, 293 .asic_reset = &r300_asic_reset,
282 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush, 294 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush,
283 .gart_set_page = &rv370_pcie_gart_set_page, 295 .gart_set_page = &rv370_pcie_gart_set_page,
284 .cp_commit = &r100_cp_commit,
285 .ring_start = &r300_ring_start, 296 .ring_start = &r300_ring_start,
286 .ring_test = &r100_ring_test, 297 .ring_test = &r100_ring_test,
287 .ring_ib_execute = &r100_ring_ib_execute, 298 .ring = {
299 [RADEON_RING_TYPE_GFX_INDEX] = {
300 .ib_execute = &r100_ring_ib_execute,
301 .emit_fence = &r300_fence_ring_emit,
302 .emit_semaphore = &r100_semaphore_ring_emit,
303 }
304 },
288 .irq_set = &r100_irq_set, 305 .irq_set = &r100_irq_set,
289 .irq_process = &r100_irq_process, 306 .irq_process = &r100_irq_process,
290 .get_vblank_counter = &r100_get_vblank_counter, 307 .get_vblank_counter = &r100_get_vblank_counter,
291 .fence_ring_emit = &r300_fence_ring_emit,
292 .cs_parse = &r300_cs_parse, 308 .cs_parse = &r300_cs_parse,
293 .copy_blit = &r100_copy_blit, 309 .copy_blit = &r100_copy_blit,
294 .copy_dma = &r200_copy_dma, 310 .copy_dma = &r200_copy_dma,
@@ -328,14 +344,18 @@ static struct radeon_asic r420_asic = {
328 .asic_reset = &r300_asic_reset, 344 .asic_reset = &r300_asic_reset,
329 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush, 345 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush,
330 .gart_set_page = &rv370_pcie_gart_set_page, 346 .gart_set_page = &rv370_pcie_gart_set_page,
331 .cp_commit = &r100_cp_commit,
332 .ring_start = &r300_ring_start, 347 .ring_start = &r300_ring_start,
333 .ring_test = &r100_ring_test, 348 .ring_test = &r100_ring_test,
334 .ring_ib_execute = &r100_ring_ib_execute, 349 .ring = {
350 [RADEON_RING_TYPE_GFX_INDEX] = {
351 .ib_execute = &r100_ring_ib_execute,
352 .emit_fence = &r300_fence_ring_emit,
353 .emit_semaphore = &r100_semaphore_ring_emit,
354 }
355 },
335 .irq_set = &r100_irq_set, 356 .irq_set = &r100_irq_set,
336 .irq_process = &r100_irq_process, 357 .irq_process = &r100_irq_process,
337 .get_vblank_counter = &r100_get_vblank_counter, 358 .get_vblank_counter = &r100_get_vblank_counter,
338 .fence_ring_emit = &r300_fence_ring_emit,
339 .cs_parse = &r300_cs_parse, 359 .cs_parse = &r300_cs_parse,
340 .copy_blit = &r100_copy_blit, 360 .copy_blit = &r100_copy_blit,
341 .copy_dma = &r200_copy_dma, 361 .copy_dma = &r200_copy_dma,
@@ -376,14 +396,18 @@ static struct radeon_asic rs400_asic = {
376 .asic_reset = &r300_asic_reset, 396 .asic_reset = &r300_asic_reset,
377 .gart_tlb_flush = &rs400_gart_tlb_flush, 397 .gart_tlb_flush = &rs400_gart_tlb_flush,
378 .gart_set_page = &rs400_gart_set_page, 398 .gart_set_page = &rs400_gart_set_page,
379 .cp_commit = &r100_cp_commit,
380 .ring_start = &r300_ring_start, 399 .ring_start = &r300_ring_start,
381 .ring_test = &r100_ring_test, 400 .ring_test = &r100_ring_test,
382 .ring_ib_execute = &r100_ring_ib_execute, 401 .ring = {
402 [RADEON_RING_TYPE_GFX_INDEX] = {
403 .ib_execute = &r100_ring_ib_execute,
404 .emit_fence = &r300_fence_ring_emit,
405 .emit_semaphore = &r100_semaphore_ring_emit,
406 }
407 },
383 .irq_set = &r100_irq_set, 408 .irq_set = &r100_irq_set,
384 .irq_process = &r100_irq_process, 409 .irq_process = &r100_irq_process,
385 .get_vblank_counter = &r100_get_vblank_counter, 410 .get_vblank_counter = &r100_get_vblank_counter,
386 .fence_ring_emit = &r300_fence_ring_emit,
387 .cs_parse = &r300_cs_parse, 411 .cs_parse = &r300_cs_parse,
388 .copy_blit = &r100_copy_blit, 412 .copy_blit = &r100_copy_blit,
389 .copy_dma = &r200_copy_dma, 413 .copy_dma = &r200_copy_dma,
@@ -424,14 +448,18 @@ static struct radeon_asic rs600_asic = {
424 .asic_reset = &rs600_asic_reset, 448 .asic_reset = &rs600_asic_reset,
425 .gart_tlb_flush = &rs600_gart_tlb_flush, 449 .gart_tlb_flush = &rs600_gart_tlb_flush,
426 .gart_set_page = &rs600_gart_set_page, 450 .gart_set_page = &rs600_gart_set_page,
427 .cp_commit = &r100_cp_commit,
428 .ring_start = &r300_ring_start, 451 .ring_start = &r300_ring_start,
429 .ring_test = &r100_ring_test, 452 .ring_test = &r100_ring_test,
430 .ring_ib_execute = &r100_ring_ib_execute, 453 .ring = {
454 [RADEON_RING_TYPE_GFX_INDEX] = {
455 .ib_execute = &r100_ring_ib_execute,
456 .emit_fence = &r300_fence_ring_emit,
457 .emit_semaphore = &r100_semaphore_ring_emit,
458 }
459 },
431 .irq_set = &rs600_irq_set, 460 .irq_set = &rs600_irq_set,
432 .irq_process = &rs600_irq_process, 461 .irq_process = &rs600_irq_process,
433 .get_vblank_counter = &rs600_get_vblank_counter, 462 .get_vblank_counter = &rs600_get_vblank_counter,
434 .fence_ring_emit = &r300_fence_ring_emit,
435 .cs_parse = &r300_cs_parse, 463 .cs_parse = &r300_cs_parse,
436 .copy_blit = &r100_copy_blit, 464 .copy_blit = &r100_copy_blit,
437 .copy_dma = &r200_copy_dma, 465 .copy_dma = &r200_copy_dma,
@@ -472,14 +500,18 @@ static struct radeon_asic rs690_asic = {
472 .asic_reset = &rs600_asic_reset, 500 .asic_reset = &rs600_asic_reset,
473 .gart_tlb_flush = &rs400_gart_tlb_flush, 501 .gart_tlb_flush = &rs400_gart_tlb_flush,
474 .gart_set_page = &rs400_gart_set_page, 502 .gart_set_page = &rs400_gart_set_page,
475 .cp_commit = &r100_cp_commit,
476 .ring_start = &r300_ring_start, 503 .ring_start = &r300_ring_start,
477 .ring_test = &r100_ring_test, 504 .ring_test = &r100_ring_test,
478 .ring_ib_execute = &r100_ring_ib_execute, 505 .ring = {
506 [RADEON_RING_TYPE_GFX_INDEX] = {
507 .ib_execute = &r100_ring_ib_execute,
508 .emit_fence = &r300_fence_ring_emit,
509 .emit_semaphore = &r100_semaphore_ring_emit,
510 }
511 },
479 .irq_set = &rs600_irq_set, 512 .irq_set = &rs600_irq_set,
480 .irq_process = &rs600_irq_process, 513 .irq_process = &rs600_irq_process,
481 .get_vblank_counter = &rs600_get_vblank_counter, 514 .get_vblank_counter = &rs600_get_vblank_counter,
482 .fence_ring_emit = &r300_fence_ring_emit,
483 .cs_parse = &r300_cs_parse, 515 .cs_parse = &r300_cs_parse,
484 .copy_blit = &r100_copy_blit, 516 .copy_blit = &r100_copy_blit,
485 .copy_dma = &r200_copy_dma, 517 .copy_dma = &r200_copy_dma,
@@ -520,14 +552,18 @@ static struct radeon_asic rv515_asic = {
520 .asic_reset = &rs600_asic_reset, 552 .asic_reset = &rs600_asic_reset,
521 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush, 553 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush,
522 .gart_set_page = &rv370_pcie_gart_set_page, 554 .gart_set_page = &rv370_pcie_gart_set_page,
523 .cp_commit = &r100_cp_commit,
524 .ring_start = &rv515_ring_start, 555 .ring_start = &rv515_ring_start,
525 .ring_test = &r100_ring_test, 556 .ring_test = &r100_ring_test,
526 .ring_ib_execute = &r100_ring_ib_execute, 557 .ring = {
558 [RADEON_RING_TYPE_GFX_INDEX] = {
559 .ib_execute = &r100_ring_ib_execute,
560 .emit_fence = &r300_fence_ring_emit,
561 .emit_semaphore = &r100_semaphore_ring_emit,
562 }
563 },
527 .irq_set = &rs600_irq_set, 564 .irq_set = &rs600_irq_set,
528 .irq_process = &rs600_irq_process, 565 .irq_process = &rs600_irq_process,
529 .get_vblank_counter = &rs600_get_vblank_counter, 566 .get_vblank_counter = &rs600_get_vblank_counter,
530 .fence_ring_emit = &r300_fence_ring_emit,
531 .cs_parse = &r300_cs_parse, 567 .cs_parse = &r300_cs_parse,
532 .copy_blit = &r100_copy_blit, 568 .copy_blit = &r100_copy_blit,
533 .copy_dma = &r200_copy_dma, 569 .copy_dma = &r200_copy_dma,
@@ -568,14 +604,18 @@ static struct radeon_asic r520_asic = {
568 .asic_reset = &rs600_asic_reset, 604 .asic_reset = &rs600_asic_reset,
569 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush, 605 .gart_tlb_flush = &rv370_pcie_gart_tlb_flush,
570 .gart_set_page = &rv370_pcie_gart_set_page, 606 .gart_set_page = &rv370_pcie_gart_set_page,
571 .cp_commit = &r100_cp_commit,
572 .ring_start = &rv515_ring_start, 607 .ring_start = &rv515_ring_start,
573 .ring_test = &r100_ring_test, 608 .ring_test = &r100_ring_test,
574 .ring_ib_execute = &r100_ring_ib_execute, 609 .ring = {
610 [RADEON_RING_TYPE_GFX_INDEX] = {
611 .ib_execute = &r100_ring_ib_execute,
612 .emit_fence = &r300_fence_ring_emit,
613 .emit_semaphore = &r100_semaphore_ring_emit,
614 }
615 },
575 .irq_set = &rs600_irq_set, 616 .irq_set = &rs600_irq_set,
576 .irq_process = &rs600_irq_process, 617 .irq_process = &rs600_irq_process,
577 .get_vblank_counter = &rs600_get_vblank_counter, 618 .get_vblank_counter = &rs600_get_vblank_counter,
578 .fence_ring_emit = &r300_fence_ring_emit,
579 .cs_parse = &r300_cs_parse, 619 .cs_parse = &r300_cs_parse,
580 .copy_blit = &r100_copy_blit, 620 .copy_blit = &r100_copy_blit,
581 .copy_dma = &r200_copy_dma, 621 .copy_dma = &r200_copy_dma,
@@ -611,18 +651,22 @@ static struct radeon_asic r600_asic = {
611 .fini = &r600_fini, 651 .fini = &r600_fini,
612 .suspend = &r600_suspend, 652 .suspend = &r600_suspend,
613 .resume = &r600_resume, 653 .resume = &r600_resume,
614 .cp_commit = &r600_cp_commit,
615 .vga_set_state = &r600_vga_set_state, 654 .vga_set_state = &r600_vga_set_state,
616 .gpu_is_lockup = &r600_gpu_is_lockup, 655 .gpu_is_lockup = &r600_gpu_is_lockup,
617 .asic_reset = &r600_asic_reset, 656 .asic_reset = &r600_asic_reset,
618 .gart_tlb_flush = &r600_pcie_gart_tlb_flush, 657 .gart_tlb_flush = &r600_pcie_gart_tlb_flush,
619 .gart_set_page = &rs600_gart_set_page, 658 .gart_set_page = &rs600_gart_set_page,
620 .ring_test = &r600_ring_test, 659 .ring_test = &r600_ring_test,
621 .ring_ib_execute = &r600_ring_ib_execute, 660 .ring = {
661 [RADEON_RING_TYPE_GFX_INDEX] = {
662 .ib_execute = &r600_ring_ib_execute,
663 .emit_fence = &r600_fence_ring_emit,
664 .emit_semaphore = &r600_semaphore_ring_emit,
665 }
666 },
622 .irq_set = &r600_irq_set, 667 .irq_set = &r600_irq_set,
623 .irq_process = &r600_irq_process, 668 .irq_process = &r600_irq_process,
624 .get_vblank_counter = &rs600_get_vblank_counter, 669 .get_vblank_counter = &rs600_get_vblank_counter,
625 .fence_ring_emit = &r600_fence_ring_emit,
626 .cs_parse = &r600_cs_parse, 670 .cs_parse = &r600_cs_parse,
627 .copy_blit = &r600_copy_blit, 671 .copy_blit = &r600_copy_blit,
628 .copy_dma = NULL, 672 .copy_dma = NULL,
@@ -658,18 +702,22 @@ static struct radeon_asic rs780_asic = {
658 .fini = &r600_fini, 702 .fini = &r600_fini,
659 .suspend = &r600_suspend, 703 .suspend = &r600_suspend,
660 .resume = &r600_resume, 704 .resume = &r600_resume,
661 .cp_commit = &r600_cp_commit,
662 .gpu_is_lockup = &r600_gpu_is_lockup, 705 .gpu_is_lockup = &r600_gpu_is_lockup,
663 .vga_set_state = &r600_vga_set_state, 706 .vga_set_state = &r600_vga_set_state,
664 .asic_reset = &r600_asic_reset, 707 .asic_reset = &r600_asic_reset,
665 .gart_tlb_flush = &r600_pcie_gart_tlb_flush, 708 .gart_tlb_flush = &r600_pcie_gart_tlb_flush,
666 .gart_set_page = &rs600_gart_set_page, 709 .gart_set_page = &rs600_gart_set_page,
667 .ring_test = &r600_ring_test, 710 .ring_test = &r600_ring_test,
668 .ring_ib_execute = &r600_ring_ib_execute, 711 .ring = {
712 [RADEON_RING_TYPE_GFX_INDEX] = {
713 .ib_execute = &r600_ring_ib_execute,
714 .emit_fence = &r600_fence_ring_emit,
715 .emit_semaphore = &r600_semaphore_ring_emit,
716 }
717 },
669 .irq_set = &r600_irq_set, 718 .irq_set = &r600_irq_set,
670 .irq_process = &r600_irq_process, 719 .irq_process = &r600_irq_process,
671 .get_vblank_counter = &rs600_get_vblank_counter, 720 .get_vblank_counter = &rs600_get_vblank_counter,
672 .fence_ring_emit = &r600_fence_ring_emit,
673 .cs_parse = &r600_cs_parse, 721 .cs_parse = &r600_cs_parse,
674 .copy_blit = &r600_copy_blit, 722 .copy_blit = &r600_copy_blit,
675 .copy_dma = NULL, 723 .copy_dma = NULL,
@@ -705,18 +753,22 @@ static struct radeon_asic rv770_asic = {
705 .fini = &rv770_fini, 753 .fini = &rv770_fini,
706 .suspend = &rv770_suspend, 754 .suspend = &rv770_suspend,
707 .resume = &rv770_resume, 755 .resume = &rv770_resume,
708 .cp_commit = &r600_cp_commit,
709 .asic_reset = &r600_asic_reset, 756 .asic_reset = &r600_asic_reset,
710 .gpu_is_lockup = &r600_gpu_is_lockup, 757 .gpu_is_lockup = &r600_gpu_is_lockup,
711 .vga_set_state = &r600_vga_set_state, 758 .vga_set_state = &r600_vga_set_state,
712 .gart_tlb_flush = &r600_pcie_gart_tlb_flush, 759 .gart_tlb_flush = &r600_pcie_gart_tlb_flush,
713 .gart_set_page = &rs600_gart_set_page, 760 .gart_set_page = &rs600_gart_set_page,
714 .ring_test = &r600_ring_test, 761 .ring_test = &r600_ring_test,
715 .ring_ib_execute = &r600_ring_ib_execute, 762 .ring = {
763 [RADEON_RING_TYPE_GFX_INDEX] = {
764 .ib_execute = &r600_ring_ib_execute,
765 .emit_fence = &r600_fence_ring_emit,
766 .emit_semaphore = &r600_semaphore_ring_emit,
767 }
768 },
716 .irq_set = &r600_irq_set, 769 .irq_set = &r600_irq_set,
717 .irq_process = &r600_irq_process, 770 .irq_process = &r600_irq_process,
718 .get_vblank_counter = &rs600_get_vblank_counter, 771 .get_vblank_counter = &rs600_get_vblank_counter,
719 .fence_ring_emit = &r600_fence_ring_emit,
720 .cs_parse = &r600_cs_parse, 772 .cs_parse = &r600_cs_parse,
721 .copy_blit = &r600_copy_blit, 773 .copy_blit = &r600_copy_blit,
722 .copy_dma = NULL, 774 .copy_dma = NULL,
@@ -752,18 +804,22 @@ static struct radeon_asic evergreen_asic = {
752 .fini = &evergreen_fini, 804 .fini = &evergreen_fini,
753 .suspend = &evergreen_suspend, 805 .suspend = &evergreen_suspend,
754 .resume = &evergreen_resume, 806 .resume = &evergreen_resume,
755 .cp_commit = &r600_cp_commit,
756 .gpu_is_lockup = &evergreen_gpu_is_lockup, 807 .gpu_is_lockup = &evergreen_gpu_is_lockup,
757 .asic_reset = &evergreen_asic_reset, 808 .asic_reset = &evergreen_asic_reset,
758 .vga_set_state = &r600_vga_set_state, 809 .vga_set_state = &r600_vga_set_state,
759 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush, 810 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
760 .gart_set_page = &rs600_gart_set_page, 811 .gart_set_page = &rs600_gart_set_page,
761 .ring_test = &r600_ring_test, 812 .ring_test = &r600_ring_test,
762 .ring_ib_execute = &evergreen_ring_ib_execute, 813 .ring = {
814 [RADEON_RING_TYPE_GFX_INDEX] = {
815 .ib_execute = &evergreen_ring_ib_execute,
816 .emit_fence = &r600_fence_ring_emit,
817 .emit_semaphore = &r600_semaphore_ring_emit,
818 }
819 },
763 .irq_set = &evergreen_irq_set, 820 .irq_set = &evergreen_irq_set,
764 .irq_process = &evergreen_irq_process, 821 .irq_process = &evergreen_irq_process,
765 .get_vblank_counter = &evergreen_get_vblank_counter, 822 .get_vblank_counter = &evergreen_get_vblank_counter,
766 .fence_ring_emit = &r600_fence_ring_emit,
767 .cs_parse = &evergreen_cs_parse, 823 .cs_parse = &evergreen_cs_parse,
768 .copy_blit = &r600_copy_blit, 824 .copy_blit = &r600_copy_blit,
769 .copy_dma = NULL, 825 .copy_dma = NULL,
@@ -799,18 +855,22 @@ static struct radeon_asic sumo_asic = {
799 .fini = &evergreen_fini, 855 .fini = &evergreen_fini,
800 .suspend = &evergreen_suspend, 856 .suspend = &evergreen_suspend,
801 .resume = &evergreen_resume, 857 .resume = &evergreen_resume,
802 .cp_commit = &r600_cp_commit,
803 .gpu_is_lockup = &evergreen_gpu_is_lockup, 858 .gpu_is_lockup = &evergreen_gpu_is_lockup,
804 .asic_reset = &evergreen_asic_reset, 859 .asic_reset = &evergreen_asic_reset,
805 .vga_set_state = &r600_vga_set_state, 860 .vga_set_state = &r600_vga_set_state,
806 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush, 861 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
807 .gart_set_page = &rs600_gart_set_page, 862 .gart_set_page = &rs600_gart_set_page,
808 .ring_test = &r600_ring_test, 863 .ring_test = &r600_ring_test,
809 .ring_ib_execute = &evergreen_ring_ib_execute, 864 .ring = {
865 [RADEON_RING_TYPE_GFX_INDEX] = {
866 .ib_execute = &evergreen_ring_ib_execute,
867 .emit_fence = &r600_fence_ring_emit,
868 .emit_semaphore = &r600_semaphore_ring_emit,
869 }
870 },
810 .irq_set = &evergreen_irq_set, 871 .irq_set = &evergreen_irq_set,
811 .irq_process = &evergreen_irq_process, 872 .irq_process = &evergreen_irq_process,
812 .get_vblank_counter = &evergreen_get_vblank_counter, 873 .get_vblank_counter = &evergreen_get_vblank_counter,
813 .fence_ring_emit = &r600_fence_ring_emit,
814 .cs_parse = &evergreen_cs_parse, 874 .cs_parse = &evergreen_cs_parse,
815 .copy_blit = &r600_copy_blit, 875 .copy_blit = &r600_copy_blit,
816 .copy_dma = NULL, 876 .copy_dma = NULL,
@@ -846,18 +906,22 @@ static struct radeon_asic btc_asic = {
846 .fini = &evergreen_fini, 906 .fini = &evergreen_fini,
847 .suspend = &evergreen_suspend, 907 .suspend = &evergreen_suspend,
848 .resume = &evergreen_resume, 908 .resume = &evergreen_resume,
849 .cp_commit = &r600_cp_commit,
850 .gpu_is_lockup = &evergreen_gpu_is_lockup, 909 .gpu_is_lockup = &evergreen_gpu_is_lockup,
851 .asic_reset = &evergreen_asic_reset, 910 .asic_reset = &evergreen_asic_reset,
852 .vga_set_state = &r600_vga_set_state, 911 .vga_set_state = &r600_vga_set_state,
853 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush, 912 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
854 .gart_set_page = &rs600_gart_set_page, 913 .gart_set_page = &rs600_gart_set_page,
855 .ring_test = &r600_ring_test, 914 .ring_test = &r600_ring_test,
856 .ring_ib_execute = &evergreen_ring_ib_execute, 915 .ring = {
916 [RADEON_RING_TYPE_GFX_INDEX] = {
917 .ib_execute = &evergreen_ring_ib_execute,
918 .emit_fence = &r600_fence_ring_emit,
919 .emit_semaphore = &r600_semaphore_ring_emit,
920 }
921 },
857 .irq_set = &evergreen_irq_set, 922 .irq_set = &evergreen_irq_set,
858 .irq_process = &evergreen_irq_process, 923 .irq_process = &evergreen_irq_process,
859 .get_vblank_counter = &evergreen_get_vblank_counter, 924 .get_vblank_counter = &evergreen_get_vblank_counter,
860 .fence_ring_emit = &r600_fence_ring_emit,
861 .cs_parse = &evergreen_cs_parse, 925 .cs_parse = &evergreen_cs_parse,
862 .copy_blit = &r600_copy_blit, 926 .copy_blit = &r600_copy_blit,
863 .copy_dma = NULL, 927 .copy_dma = NULL,
@@ -893,18 +957,32 @@ static struct radeon_asic cayman_asic = {
893 .fini = &cayman_fini, 957 .fini = &cayman_fini,
894 .suspend = &cayman_suspend, 958 .suspend = &cayman_suspend,
895 .resume = &cayman_resume, 959 .resume = &cayman_resume,
896 .cp_commit = &r600_cp_commit,
897 .gpu_is_lockup = &cayman_gpu_is_lockup, 960 .gpu_is_lockup = &cayman_gpu_is_lockup,
898 .asic_reset = &cayman_asic_reset, 961 .asic_reset = &cayman_asic_reset,
899 .vga_set_state = &r600_vga_set_state, 962 .vga_set_state = &r600_vga_set_state,
900 .gart_tlb_flush = &cayman_pcie_gart_tlb_flush, 963 .gart_tlb_flush = &cayman_pcie_gart_tlb_flush,
901 .gart_set_page = &rs600_gart_set_page, 964 .gart_set_page = &rs600_gart_set_page,
902 .ring_test = &r600_ring_test, 965 .ring_test = &r600_ring_test,
903 .ring_ib_execute = &evergreen_ring_ib_execute, 966 .ring = {
967 [RADEON_RING_TYPE_GFX_INDEX] = {
968 .ib_execute = &evergreen_ring_ib_execute,
969 .emit_fence = &cayman_fence_ring_emit,
970 .emit_semaphore = &r600_semaphore_ring_emit,
971 },
972 [CAYMAN_RING_TYPE_CP1_INDEX] = {
973 .ib_execute = &r600_ring_ib_execute,
974 .emit_fence = &cayman_fence_ring_emit,
975 .emit_semaphore = &r600_semaphore_ring_emit,
976 },
977 [CAYMAN_RING_TYPE_CP2_INDEX] = {
978 .ib_execute = &r600_ring_ib_execute,
979 .emit_fence = &cayman_fence_ring_emit,
980 .emit_semaphore = &r600_semaphore_ring_emit,
981 }
982 },
904 .irq_set = &evergreen_irq_set, 983 .irq_set = &evergreen_irq_set,
905 .irq_process = &evergreen_irq_process, 984 .irq_process = &evergreen_irq_process,
906 .get_vblank_counter = &evergreen_get_vblank_counter, 985 .get_vblank_counter = &evergreen_get_vblank_counter,
907 .fence_ring_emit = &r600_fence_ring_emit,
908 .cs_parse = &evergreen_cs_parse, 986 .cs_parse = &evergreen_cs_parse,
909 .copy_blit = &r600_copy_blit, 987 .copy_blit = &r600_copy_blit,
910 .copy_dma = NULL, 988 .copy_dma = NULL,
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 59914842a72..c002ed1c448 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -58,17 +58,20 @@ void r100_fini(struct radeon_device *rdev);
58int r100_suspend(struct radeon_device *rdev); 58int r100_suspend(struct radeon_device *rdev);
59int r100_resume(struct radeon_device *rdev); 59int r100_resume(struct radeon_device *rdev);
60void r100_vga_set_state(struct radeon_device *rdev, bool state); 60void r100_vga_set_state(struct radeon_device *rdev, bool state);
61bool r100_gpu_is_lockup(struct radeon_device *rdev); 61bool r100_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
62int r100_asic_reset(struct radeon_device *rdev); 62int r100_asic_reset(struct radeon_device *rdev);
63u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc); 63u32 r100_get_vblank_counter(struct radeon_device *rdev, int crtc);
64void r100_pci_gart_tlb_flush(struct radeon_device *rdev); 64void r100_pci_gart_tlb_flush(struct radeon_device *rdev);
65int r100_pci_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); 65int r100_pci_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
66void r100_cp_commit(struct radeon_device *rdev);
67void r100_ring_start(struct radeon_device *rdev); 66void r100_ring_start(struct radeon_device *rdev);
68int r100_irq_set(struct radeon_device *rdev); 67int r100_irq_set(struct radeon_device *rdev);
69int r100_irq_process(struct radeon_device *rdev); 68int r100_irq_process(struct radeon_device *rdev);
70void r100_fence_ring_emit(struct radeon_device *rdev, 69void r100_fence_ring_emit(struct radeon_device *rdev,
71 struct radeon_fence *fence); 70 struct radeon_fence *fence);
71void r100_semaphore_ring_emit(struct radeon_device *rdev,
72 struct radeon_ring *cp,
73 struct radeon_semaphore *semaphore,
74 bool emit_wait);
72int r100_cs_parse(struct radeon_cs_parser *p); 75int r100_cs_parse(struct radeon_cs_parser *p);
73void r100_pll_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); 76void r100_pll_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
74uint32_t r100_pll_rreg(struct radeon_device *rdev, uint32_t reg); 77uint32_t r100_pll_rreg(struct radeon_device *rdev, uint32_t reg);
@@ -83,7 +86,7 @@ int r100_set_surface_reg(struct radeon_device *rdev, int reg,
83void r100_clear_surface_reg(struct radeon_device *rdev, int reg); 86void r100_clear_surface_reg(struct radeon_device *rdev, int reg);
84void r100_bandwidth_update(struct radeon_device *rdev); 87void r100_bandwidth_update(struct radeon_device *rdev);
85void r100_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); 88void r100_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
86int r100_ring_test(struct radeon_device *rdev); 89int r100_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
87void r100_hpd_init(struct radeon_device *rdev); 90void r100_hpd_init(struct radeon_device *rdev);
88void r100_hpd_fini(struct radeon_device *rdev); 91void r100_hpd_fini(struct radeon_device *rdev);
89bool r100_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd); 92bool r100_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd);
@@ -101,12 +104,12 @@ void r100_pci_gart_disable(struct radeon_device *rdev);
101int r100_debugfs_mc_info_init(struct radeon_device *rdev); 104int r100_debugfs_mc_info_init(struct radeon_device *rdev);
102int r100_gui_wait_for_idle(struct radeon_device *rdev); 105int r100_gui_wait_for_idle(struct radeon_device *rdev);
103void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, 106void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup,
104 struct radeon_cp *cp); 107 struct radeon_ring *cp);
105bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, 108bool r100_gpu_cp_is_lockup(struct radeon_device *rdev,
106 struct r100_gpu_lockup *lockup, 109 struct r100_gpu_lockup *lockup,
107 struct radeon_cp *cp); 110 struct radeon_ring *cp);
108void r100_ib_fini(struct radeon_device *rdev); 111void r100_ib_fini(struct radeon_device *rdev);
109int r100_ib_init(struct radeon_device *rdev); 112int r100_ib_test(struct radeon_device *rdev);
110void r100_irq_disable(struct radeon_device *rdev); 113void r100_irq_disable(struct radeon_device *rdev);
111void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save); 114void r100_mc_stop(struct radeon_device *rdev, struct r100_mc_save *save);
112void r100_mc_resume(struct radeon_device *rdev, struct r100_mc_save *save); 115void r100_mc_resume(struct radeon_device *rdev, struct r100_mc_save *save);
@@ -154,7 +157,7 @@ extern int r300_init(struct radeon_device *rdev);
154extern void r300_fini(struct radeon_device *rdev); 157extern void r300_fini(struct radeon_device *rdev);
155extern int r300_suspend(struct radeon_device *rdev); 158extern int r300_suspend(struct radeon_device *rdev);
156extern int r300_resume(struct radeon_device *rdev); 159extern int r300_resume(struct radeon_device *rdev);
157extern bool r300_gpu_is_lockup(struct radeon_device *rdev); 160extern bool r300_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
158extern int r300_asic_reset(struct radeon_device *rdev); 161extern int r300_asic_reset(struct radeon_device *rdev);
159extern void r300_ring_start(struct radeon_device *rdev); 162extern void r300_ring_start(struct radeon_device *rdev);
160extern void r300_fence_ring_emit(struct radeon_device *rdev, 163extern void r300_fence_ring_emit(struct radeon_device *rdev,
@@ -293,22 +296,25 @@ int r600_resume(struct radeon_device *rdev);
293void r600_vga_set_state(struct radeon_device *rdev, bool state); 296void r600_vga_set_state(struct radeon_device *rdev, bool state);
294int r600_wb_init(struct radeon_device *rdev); 297int r600_wb_init(struct radeon_device *rdev);
295void r600_wb_fini(struct radeon_device *rdev); 298void r600_wb_fini(struct radeon_device *rdev);
296void r600_cp_commit(struct radeon_device *rdev);
297void r600_pcie_gart_tlb_flush(struct radeon_device *rdev); 299void r600_pcie_gart_tlb_flush(struct radeon_device *rdev);
298uint32_t r600_pciep_rreg(struct radeon_device *rdev, uint32_t reg); 300uint32_t r600_pciep_rreg(struct radeon_device *rdev, uint32_t reg);
299void r600_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); 301void r600_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
300int r600_cs_parse(struct radeon_cs_parser *p); 302int r600_cs_parse(struct radeon_cs_parser *p);
301void r600_fence_ring_emit(struct radeon_device *rdev, 303void r600_fence_ring_emit(struct radeon_device *rdev,
302 struct radeon_fence *fence); 304 struct radeon_fence *fence);
303bool r600_gpu_is_lockup(struct radeon_device *rdev); 305void r600_semaphore_ring_emit(struct radeon_device *rdev,
306 struct radeon_ring *cp,
307 struct radeon_semaphore *semaphore,
308 bool emit_wait);
309bool r600_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
304int r600_asic_reset(struct radeon_device *rdev); 310int r600_asic_reset(struct radeon_device *rdev);
305int r600_set_surface_reg(struct radeon_device *rdev, int reg, 311int r600_set_surface_reg(struct radeon_device *rdev, int reg,
306 uint32_t tiling_flags, uint32_t pitch, 312 uint32_t tiling_flags, uint32_t pitch,
307 uint32_t offset, uint32_t obj_size); 313 uint32_t offset, uint32_t obj_size);
308void r600_clear_surface_reg(struct radeon_device *rdev, int reg); 314void r600_clear_surface_reg(struct radeon_device *rdev, int reg);
309int r600_ib_test(struct radeon_device *rdev); 315int r600_ib_test(struct radeon_device *rdev, int ring);
310void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); 316void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
311int r600_ring_test(struct radeon_device *rdev); 317int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
312int r600_copy_blit(struct radeon_device *rdev, 318int r600_copy_blit(struct radeon_device *rdev,
313 uint64_t src_offset, uint64_t dst_offset, 319 uint64_t src_offset, uint64_t dst_offset,
314 unsigned num_gpu_pages, struct radeon_fence *fence); 320 unsigned num_gpu_pages, struct radeon_fence *fence);
@@ -328,7 +334,7 @@ extern int r600_get_pcie_lanes(struct radeon_device *rdev);
328bool r600_card_posted(struct radeon_device *rdev); 334bool r600_card_posted(struct radeon_device *rdev);
329void r600_cp_stop(struct radeon_device *rdev); 335void r600_cp_stop(struct radeon_device *rdev);
330int r600_cp_start(struct radeon_device *rdev); 336int r600_cp_start(struct radeon_device *rdev);
331void r600_ring_init(struct radeon_device *rdev, unsigned ring_size); 337void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *cp, unsigned ring_size);
332int r600_cp_resume(struct radeon_device *rdev); 338int r600_cp_resume(struct radeon_device *rdev);
333void r600_cp_fini(struct radeon_device *rdev); 339void r600_cp_fini(struct radeon_device *rdev);
334int r600_count_pipe_bits(uint32_t val); 340int r600_count_pipe_bits(uint32_t val);
@@ -397,7 +403,7 @@ int evergreen_init(struct radeon_device *rdev);
397void evergreen_fini(struct radeon_device *rdev); 403void evergreen_fini(struct radeon_device *rdev);
398int evergreen_suspend(struct radeon_device *rdev); 404int evergreen_suspend(struct radeon_device *rdev);
399int evergreen_resume(struct radeon_device *rdev); 405int evergreen_resume(struct radeon_device *rdev);
400bool evergreen_gpu_is_lockup(struct radeon_device *rdev); 406bool evergreen_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
401int evergreen_asic_reset(struct radeon_device *rdev); 407int evergreen_asic_reset(struct radeon_device *rdev);
402void evergreen_bandwidth_update(struct radeon_device *rdev); 408void evergreen_bandwidth_update(struct radeon_device *rdev);
403void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); 409void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
@@ -423,12 +429,14 @@ int evergreen_blit_init(struct radeon_device *rdev);
423/* 429/*
424 * cayman 430 * cayman
425 */ 431 */
432void cayman_fence_ring_emit(struct radeon_device *rdev,
433 struct radeon_fence *fence);
426void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev); 434void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev);
427int cayman_init(struct radeon_device *rdev); 435int cayman_init(struct radeon_device *rdev);
428void cayman_fini(struct radeon_device *rdev); 436void cayman_fini(struct radeon_device *rdev);
429int cayman_suspend(struct radeon_device *rdev); 437int cayman_suspend(struct radeon_device *rdev);
430int cayman_resume(struct radeon_device *rdev); 438int cayman_resume(struct radeon_device *rdev);
431bool cayman_gpu_is_lockup(struct radeon_device *rdev); 439bool cayman_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *cp);
432int cayman_asic_reset(struct radeon_device *rdev); 440int cayman_asic_reset(struct radeon_device *rdev);
433 441
434#endif 442#endif
diff --git a/drivers/gpu/drm/radeon/radeon_benchmark.c b/drivers/gpu/drm/radeon/radeon_benchmark.c
index 17e1a9b2d8f..4f749a69c9c 100644
--- a/drivers/gpu/drm/radeon/radeon_benchmark.c
+++ b/drivers/gpu/drm/radeon/radeon_benchmark.c
@@ -43,7 +43,7 @@ static int radeon_benchmark_do_move(struct radeon_device *rdev, unsigned size,
43 43
44 start_jiffies = jiffies; 44 start_jiffies = jiffies;
45 for (i = 0; i < n; i++) { 45 for (i = 0; i < n; i++) {
46 r = radeon_fence_create(rdev, &fence); 46 r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
47 if (r) 47 if (r)
48 return r; 48 return r;
49 49
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 29afd71e084..09ef48636e5 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -58,7 +58,7 @@ int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
58 58
59 duplicate = false; 59 duplicate = false;
60 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4]; 60 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
61 for (j = 0; j < p->nrelocs; j++) { 61 for (j = 0; j < i; j++) {
62 if (r->handle == p->relocs[j].handle) { 62 if (r->handle == p->relocs[j].handle) {
63 p->relocs_ptr[i] = &p->relocs[j]; 63 p->relocs_ptr[i] = &p->relocs[j];
64 duplicate = true; 64 duplicate = true;
@@ -84,7 +84,8 @@ int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
84 p->relocs[i].flags = r->flags; 84 p->relocs[i].flags = r->flags;
85 radeon_bo_list_add_object(&p->relocs[i].lobj, 85 radeon_bo_list_add_object(&p->relocs[i].lobj,
86 &p->validated); 86 &p->validated);
87 } 87 } else
88 p->relocs[i].handle = 0;
88 } 89 }
89 return radeon_bo_list_validate(&p->validated); 90 return radeon_bo_list_validate(&p->validated);
90} 91}
@@ -245,7 +246,7 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
245 radeon_mutex_unlock(&rdev->cs_mutex); 246 radeon_mutex_unlock(&rdev->cs_mutex);
246 return r; 247 return r;
247 } 248 }
248 r = radeon_ib_get(rdev, &parser.ib); 249 r = radeon_ib_get(rdev, RADEON_RING_TYPE_GFX_INDEX, &parser.ib);
249 if (r) { 250 if (r) {
250 DRM_ERROR("Failed to get ib !\n"); 251 DRM_ERROR("Failed to get ib !\n");
251 radeon_cs_parser_fini(&parser, r); 252 radeon_cs_parser_fini(&parser, r);
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index fb347a80486..e81c333e0f9 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -718,17 +718,20 @@ int radeon_device_init(struct radeon_device *rdev,
718 * can recall function without having locking issues */ 718 * can recall function without having locking issues */
719 radeon_mutex_init(&rdev->cs_mutex); 719 radeon_mutex_init(&rdev->cs_mutex);
720 mutex_init(&rdev->ib_pool.mutex); 720 mutex_init(&rdev->ib_pool.mutex);
721 mutex_init(&rdev->cp.mutex); 721 for (i = 0; i < RADEON_NUM_RINGS; ++i)
722 mutex_init(&rdev->ring[i].mutex);
722 mutex_init(&rdev->dc_hw_i2c_mutex); 723 mutex_init(&rdev->dc_hw_i2c_mutex);
723 if (rdev->family >= CHIP_R600) 724 if (rdev->family >= CHIP_R600)
724 spin_lock_init(&rdev->ih.lock); 725 spin_lock_init(&rdev->ih.lock);
725 mutex_init(&rdev->gem.mutex); 726 mutex_init(&rdev->gem.mutex);
726 mutex_init(&rdev->pm.mutex); 727 mutex_init(&rdev->pm.mutex);
727 mutex_init(&rdev->vram_mutex); 728 mutex_init(&rdev->vram_mutex);
728 rwlock_init(&rdev->fence_drv.lock); 729 rwlock_init(&rdev->fence_lock);
730 rwlock_init(&rdev->semaphore_drv.lock);
729 INIT_LIST_HEAD(&rdev->gem.objects); 731 INIT_LIST_HEAD(&rdev->gem.objects);
730 init_waitqueue_head(&rdev->irq.vblank_queue); 732 init_waitqueue_head(&rdev->irq.vblank_queue);
731 init_waitqueue_head(&rdev->irq.idle_queue); 733 init_waitqueue_head(&rdev->irq.idle_queue);
734 INIT_LIST_HEAD(&rdev->semaphore_drv.free);
732 735
733 /* Set asic functions */ 736 /* Set asic functions */
734 r = radeon_asic_init(rdev); 737 r = radeon_asic_init(rdev);
@@ -820,15 +823,20 @@ int radeon_device_init(struct radeon_device *rdev,
820 if (r) 823 if (r)
821 return r; 824 return r;
822 } 825 }
823 if (radeon_testing) { 826 if ((radeon_testing & 1)) {
824 radeon_test_moves(rdev); 827 radeon_test_moves(rdev);
825 } 828 }
829 if ((radeon_testing & 2)) {
830 radeon_test_syncing(rdev);
831 }
826 if (radeon_benchmarking) { 832 if (radeon_benchmarking) {
827 radeon_benchmark(rdev, radeon_benchmarking); 833 radeon_benchmark(rdev, radeon_benchmarking);
828 } 834 }
829 return 0; 835 return 0;
830} 836}
831 837
838static void radeon_debugfs_remove_files(struct radeon_device *rdev);
839
832void radeon_device_fini(struct radeon_device *rdev) 840void radeon_device_fini(struct radeon_device *rdev)
833{ 841{
834 DRM_INFO("radeon: finishing device.\n"); 842 DRM_INFO("radeon: finishing device.\n");
@@ -843,6 +851,7 @@ void radeon_device_fini(struct radeon_device *rdev)
843 rdev->rio_mem = NULL; 851 rdev->rio_mem = NULL;
844 iounmap(rdev->rmmio); 852 iounmap(rdev->rmmio);
845 rdev->rmmio = NULL; 853 rdev->rmmio = NULL;
854 radeon_debugfs_remove_files(rdev);
846} 855}
847 856
848 857
@@ -854,7 +863,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
854 struct radeon_device *rdev; 863 struct radeon_device *rdev;
855 struct drm_crtc *crtc; 864 struct drm_crtc *crtc;
856 struct drm_connector *connector; 865 struct drm_connector *connector;
857 int r; 866 int i, r;
858 867
859 if (dev == NULL || dev->dev_private == NULL) { 868 if (dev == NULL || dev->dev_private == NULL) {
860 return -ENODEV; 869 return -ENODEV;
@@ -893,7 +902,8 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
893 /* evict vram memory */ 902 /* evict vram memory */
894 radeon_bo_evict_vram(rdev); 903 radeon_bo_evict_vram(rdev);
895 /* wait for gpu to finish processing current batch */ 904 /* wait for gpu to finish processing current batch */
896 radeon_fence_wait_last(rdev); 905 for (i = 0; i < RADEON_NUM_RINGS; i++)
906 radeon_fence_wait_last(rdev, i);
897 907
898 radeon_save_bios_scratch_regs(rdev); 908 radeon_save_bios_scratch_regs(rdev);
899 909
@@ -992,36 +1002,29 @@ int radeon_gpu_reset(struct radeon_device *rdev)
992/* 1002/*
993 * Debugfs 1003 * Debugfs
994 */ 1004 */
995struct radeon_debugfs {
996 struct drm_info_list *files;
997 unsigned num_files;
998};
999static struct radeon_debugfs _radeon_debugfs[RADEON_DEBUGFS_MAX_COMPONENTS];
1000static unsigned _radeon_debugfs_count = 0;
1001
1002int radeon_debugfs_add_files(struct radeon_device *rdev, 1005int radeon_debugfs_add_files(struct radeon_device *rdev,
1003 struct drm_info_list *files, 1006 struct drm_info_list *files,
1004 unsigned nfiles) 1007 unsigned nfiles)
1005{ 1008{
1006 unsigned i; 1009 unsigned i;
1007 1010
1008 for (i = 0; i < _radeon_debugfs_count; i++) { 1011 for (i = 0; i < rdev->debugfs_count; i++) {
1009 if (_radeon_debugfs[i].files == files) { 1012 if (rdev->debugfs[i].files == files) {
1010 /* Already registered */ 1013 /* Already registered */
1011 return 0; 1014 return 0;
1012 } 1015 }
1013 } 1016 }
1014 1017
1015 i = _radeon_debugfs_count + 1; 1018 i = rdev->debugfs_count + 1;
1016 if (i > RADEON_DEBUGFS_MAX_COMPONENTS) { 1019 if (i > RADEON_DEBUGFS_MAX_COMPONENTS) {
1017 DRM_ERROR("Reached maximum number of debugfs components.\n"); 1020 DRM_ERROR("Reached maximum number of debugfs components.\n");
1018 DRM_ERROR("Report so we increase " 1021 DRM_ERROR("Report so we increase "
1019 "RADEON_DEBUGFS_MAX_COMPONENTS.\n"); 1022 "RADEON_DEBUGFS_MAX_COMPONENTS.\n");
1020 return -EINVAL; 1023 return -EINVAL;
1021 } 1024 }
1022 _radeon_debugfs[_radeon_debugfs_count].files = files; 1025 rdev->debugfs[rdev->debugfs_count].files = files;
1023 _radeon_debugfs[_radeon_debugfs_count].num_files = nfiles; 1026 rdev->debugfs[rdev->debugfs_count].num_files = nfiles;
1024 _radeon_debugfs_count = i; 1027 rdev->debugfs_count = i;
1025#if defined(CONFIG_DEBUG_FS) 1028#if defined(CONFIG_DEBUG_FS)
1026 drm_debugfs_create_files(files, nfiles, 1029 drm_debugfs_create_files(files, nfiles,
1027 rdev->ddev->control->debugfs_root, 1030 rdev->ddev->control->debugfs_root,
@@ -1033,6 +1036,22 @@ int radeon_debugfs_add_files(struct radeon_device *rdev,
1033 return 0; 1036 return 0;
1034} 1037}
1035 1038
1039static void radeon_debugfs_remove_files(struct radeon_device *rdev)
1040{
1041#if defined(CONFIG_DEBUG_FS)
1042 unsigned i;
1043
1044 for (i = 0; i < rdev->debugfs_count; i++) {
1045 drm_debugfs_remove_files(rdev->debugfs[i].files,
1046 rdev->debugfs[i].num_files,
1047 rdev->ddev->control);
1048 drm_debugfs_remove_files(rdev->debugfs[i].files,
1049 rdev->debugfs[i].num_files,
1050 rdev->ddev->primary);
1051 }
1052#endif
1053}
1054
1036#if defined(CONFIG_DEBUG_FS) 1055#if defined(CONFIG_DEBUG_FS)
1037int radeon_debugfs_init(struct drm_minor *minor) 1056int radeon_debugfs_init(struct drm_minor *minor)
1038{ 1057{
@@ -1041,11 +1060,5 @@ int radeon_debugfs_init(struct drm_minor *minor)
1041 1060
1042void radeon_debugfs_cleanup(struct drm_minor *minor) 1061void radeon_debugfs_cleanup(struct drm_minor *minor)
1043{ 1062{
1044 unsigned i;
1045
1046 for (i = 0; i < _radeon_debugfs_count; i++) {
1047 drm_debugfs_remove_files(_radeon_debugfs[i].files,
1048 _radeon_debugfs[i].num_files, minor);
1049 }
1050} 1063}
1051#endif 1064#endif
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 76ec0e9ed8a..ae9e3da594a 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -40,32 +40,24 @@
40#include "radeon.h" 40#include "radeon.h"
41#include "radeon_trace.h" 41#include "radeon_trace.h"
42 42
43static void radeon_fence_write(struct radeon_device *rdev, u32 seq) 43static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
44{ 44{
45 if (rdev->wb.enabled) { 45 if (rdev->wb.enabled) {
46 u32 scratch_index; 46 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
47 if (rdev->wb.use_event) 47 } else {
48 scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; 48 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
49 else 49 }
50 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
51 rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq);
52 } else
53 WREG32(rdev->fence_drv.scratch_reg, seq);
54} 50}
55 51
56static u32 radeon_fence_read(struct radeon_device *rdev) 52static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
57{ 53{
58 u32 seq; 54 u32 seq = 0;
59 55
60 if (rdev->wb.enabled) { 56 if (rdev->wb.enabled) {
61 u32 scratch_index; 57 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
62 if (rdev->wb.use_event) 58 } else {
63 scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base; 59 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
64 else 60 }
65 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
66 seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]);
67 } else
68 seq = RREG32(rdev->fence_drv.scratch_reg);
69 return seq; 61 return seq;
70} 62}
71 63
@@ -73,28 +65,28 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
73{ 65{
74 unsigned long irq_flags; 66 unsigned long irq_flags;
75 67
76 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 68 write_lock_irqsave(&rdev->fence_lock, irq_flags);
77 if (fence->emited) { 69 if (fence->emitted) {
78 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 70 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
79 return 0; 71 return 0;
80 } 72 }
81 fence->seq = atomic_add_return(1, &rdev->fence_drv.seq); 73 fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
82 if (!rdev->cp.ready) 74 if (!rdev->ring[fence->ring].ready)
83 /* FIXME: cp is not running assume everythings is done right 75 /* FIXME: cp is not running assume everythings is done right
84 * away 76 * away
85 */ 77 */
86 radeon_fence_write(rdev, fence->seq); 78 radeon_fence_write(rdev, fence->seq, fence->ring);
87 else 79 else
88 radeon_fence_ring_emit(rdev, fence); 80 radeon_fence_ring_emit(rdev, fence->ring, fence);
89 81
90 trace_radeon_fence_emit(rdev->ddev, fence->seq); 82 trace_radeon_fence_emit(rdev->ddev, fence->seq);
91 fence->emited = true; 83 fence->emitted = true;
92 list_move_tail(&fence->list, &rdev->fence_drv.emited); 84 list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
93 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 85 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
94 return 0; 86 return 0;
95} 87}
96 88
97static bool radeon_fence_poll_locked(struct radeon_device *rdev) 89static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
98{ 90{
99 struct radeon_fence *fence; 91 struct radeon_fence *fence;
100 struct list_head *i, *n; 92 struct list_head *i, *n;
@@ -102,34 +94,34 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev)
102 bool wake = false; 94 bool wake = false;
103 unsigned long cjiffies; 95 unsigned long cjiffies;
104 96
105 seq = radeon_fence_read(rdev); 97 seq = radeon_fence_read(rdev, ring);
106 if (seq != rdev->fence_drv.last_seq) { 98 if (seq != rdev->fence_drv[ring].last_seq) {
107 rdev->fence_drv.last_seq = seq; 99 rdev->fence_drv[ring].last_seq = seq;
108 rdev->fence_drv.last_jiffies = jiffies; 100 rdev->fence_drv[ring].last_jiffies = jiffies;
109 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 101 rdev->fence_drv[ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
110 } else { 102 } else {
111 cjiffies = jiffies; 103 cjiffies = jiffies;
112 if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) { 104 if (time_after(cjiffies, rdev->fence_drv[ring].last_jiffies)) {
113 cjiffies -= rdev->fence_drv.last_jiffies; 105 cjiffies -= rdev->fence_drv[ring].last_jiffies;
114 if (time_after(rdev->fence_drv.last_timeout, cjiffies)) { 106 if (time_after(rdev->fence_drv[ring].last_timeout, cjiffies)) {
115 /* update the timeout */ 107 /* update the timeout */
116 rdev->fence_drv.last_timeout -= cjiffies; 108 rdev->fence_drv[ring].last_timeout -= cjiffies;
117 } else { 109 } else {
118 /* the 500ms timeout is elapsed we should test 110 /* the 500ms timeout is elapsed we should test
119 * for GPU lockup 111 * for GPU lockup
120 */ 112 */
121 rdev->fence_drv.last_timeout = 1; 113 rdev->fence_drv[ring].last_timeout = 1;
122 } 114 }
123 } else { 115 } else {
124 /* wrap around update last jiffies, we will just wait 116 /* wrap around update last jiffies, we will just wait
125 * a little longer 117 * a little longer
126 */ 118 */
127 rdev->fence_drv.last_jiffies = cjiffies; 119 rdev->fence_drv[ring].last_jiffies = cjiffies;
128 } 120 }
129 return false; 121 return false;
130 } 122 }
131 n = NULL; 123 n = NULL;
132 list_for_each(i, &rdev->fence_drv.emited) { 124 list_for_each(i, &rdev->fence_drv[ring].emitted) {
133 fence = list_entry(i, struct radeon_fence, list); 125 fence = list_entry(i, struct radeon_fence, list);
134 if (fence->seq == seq) { 126 if (fence->seq == seq) {
135 n = i; 127 n = i;
@@ -141,11 +133,11 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev)
141 i = n; 133 i = n;
142 do { 134 do {
143 n = i->prev; 135 n = i->prev;
144 list_move_tail(i, &rdev->fence_drv.signaled); 136 list_move_tail(i, &rdev->fence_drv[ring].signaled);
145 fence = list_entry(i, struct radeon_fence, list); 137 fence = list_entry(i, struct radeon_fence, list);
146 fence->signaled = true; 138 fence->signaled = true;
147 i = n; 139 i = n;
148 } while (i != &rdev->fence_drv.emited); 140 } while (i != &rdev->fence_drv[ring].emitted);
149 wake = true; 141 wake = true;
150 } 142 }
151 return wake; 143 return wake;
@@ -157,14 +149,16 @@ static void radeon_fence_destroy(struct kref *kref)
157 struct radeon_fence *fence; 149 struct radeon_fence *fence;
158 150
159 fence = container_of(kref, struct radeon_fence, kref); 151 fence = container_of(kref, struct radeon_fence, kref);
160 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 152 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
161 list_del(&fence->list); 153 list_del(&fence->list);
162 fence->emited = false; 154 fence->emitted = false;
163 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 155 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
164 kfree(fence); 156 kfree(fence);
165} 157}
166 158
167int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence) 159int radeon_fence_create(struct radeon_device *rdev,
160 struct radeon_fence **fence,
161 int ring)
168{ 162{
169 unsigned long irq_flags; 163 unsigned long irq_flags;
170 164
@@ -174,18 +168,18 @@ int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence)
174 } 168 }
175 kref_init(&((*fence)->kref)); 169 kref_init(&((*fence)->kref));
176 (*fence)->rdev = rdev; 170 (*fence)->rdev = rdev;
177 (*fence)->emited = false; 171 (*fence)->emitted = false;
178 (*fence)->signaled = false; 172 (*fence)->signaled = false;
179 (*fence)->seq = 0; 173 (*fence)->seq = 0;
174 (*fence)->ring = ring;
180 INIT_LIST_HEAD(&(*fence)->list); 175 INIT_LIST_HEAD(&(*fence)->list);
181 176
182 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 177 write_lock_irqsave(&rdev->fence_lock, irq_flags);
183 list_add_tail(&(*fence)->list, &rdev->fence_drv.created); 178 list_add_tail(&(*fence)->list, &rdev->fence_drv[ring].created);
184 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 179 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
185 return 0; 180 return 0;
186} 181}
187 182
188
189bool radeon_fence_signaled(struct radeon_fence *fence) 183bool radeon_fence_signaled(struct radeon_fence *fence)
190{ 184{
191 unsigned long irq_flags; 185 unsigned long irq_flags;
@@ -197,21 +191,21 @@ bool radeon_fence_signaled(struct radeon_fence *fence)
197 if (fence->rdev->gpu_lockup) 191 if (fence->rdev->gpu_lockup)
198 return true; 192 return true;
199 193
200 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags); 194 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
201 signaled = fence->signaled; 195 signaled = fence->signaled;
202 /* if we are shuting down report all fence as signaled */ 196 /* if we are shuting down report all fence as signaled */
203 if (fence->rdev->shutdown) { 197 if (fence->rdev->shutdown) {
204 signaled = true; 198 signaled = true;
205 } 199 }
206 if (!fence->emited) { 200 if (!fence->emitted) {
207 WARN(1, "Querying an unemited fence : %p !\n", fence); 201 WARN(1, "Querying an unemitted fence : %p !\n", fence);
208 signaled = true; 202 signaled = true;
209 } 203 }
210 if (!signaled) { 204 if (!signaled) {
211 radeon_fence_poll_locked(fence->rdev); 205 radeon_fence_poll_locked(fence->rdev, fence->ring);
212 signaled = fence->signaled; 206 signaled = fence->signaled;
213 } 207 }
214 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags); 208 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
215 return signaled; 209 return signaled;
216} 210}
217 211
@@ -230,24 +224,24 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
230 if (radeon_fence_signaled(fence)) { 224 if (radeon_fence_signaled(fence)) {
231 return 0; 225 return 0;
232 } 226 }
233 timeout = rdev->fence_drv.last_timeout; 227 timeout = rdev->fence_drv[fence->ring].last_timeout;
234retry: 228retry:
235 /* save current sequence used to check for GPU lockup */ 229 /* save current sequence used to check for GPU lockup */
236 seq = rdev->fence_drv.last_seq; 230 seq = rdev->fence_drv[fence->ring].last_seq;
237 trace_radeon_fence_wait_begin(rdev->ddev, seq); 231 trace_radeon_fence_wait_begin(rdev->ddev, seq);
238 if (intr) { 232 if (intr) {
239 radeon_irq_kms_sw_irq_get(rdev); 233 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
240 r = wait_event_interruptible_timeout(rdev->fence_drv.queue, 234 r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue,
241 radeon_fence_signaled(fence), timeout); 235 radeon_fence_signaled(fence), timeout);
242 radeon_irq_kms_sw_irq_put(rdev); 236 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
243 if (unlikely(r < 0)) { 237 if (unlikely(r < 0)) {
244 return r; 238 return r;
245 } 239 }
246 } else { 240 } else {
247 radeon_irq_kms_sw_irq_get(rdev); 241 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
248 r = wait_event_timeout(rdev->fence_drv.queue, 242 r = wait_event_timeout(rdev->fence_drv[fence->ring].queue,
249 radeon_fence_signaled(fence), timeout); 243 radeon_fence_signaled(fence), timeout);
250 radeon_irq_kms_sw_irq_put(rdev); 244 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
251 } 245 }
252 trace_radeon_fence_wait_end(rdev->ddev, seq); 246 trace_radeon_fence_wait_end(rdev->ddev, seq);
253 if (unlikely(!radeon_fence_signaled(fence))) { 247 if (unlikely(!radeon_fence_signaled(fence))) {
@@ -258,10 +252,11 @@ retry:
258 timeout = r; 252 timeout = r;
259 goto retry; 253 goto retry;
260 } 254 }
261 /* don't protect read access to rdev->fence_drv.last_seq 255 /* don't protect read access to rdev->fence_drv[t].last_seq
262 * if we experiencing a lockup the value doesn't change 256 * if we experiencing a lockup the value doesn't change
263 */ 257 */
264 if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) { 258 if (seq == rdev->fence_drv[fence->ring].last_seq &&
259 radeon_gpu_is_lockup(rdev, &rdev->ring[fence->ring])) {
265 /* good news we believe it's a lockup */ 260 /* good news we believe it's a lockup */
266 printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n", 261 printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
267 fence->seq, seq); 262 fence->seq, seq);
@@ -272,20 +267,20 @@ retry:
272 r = radeon_gpu_reset(rdev); 267 r = radeon_gpu_reset(rdev);
273 if (r) 268 if (r)
274 return r; 269 return r;
275 radeon_fence_write(rdev, fence->seq); 270 radeon_fence_write(rdev, fence->seq, fence->ring);
276 rdev->gpu_lockup = false; 271 rdev->gpu_lockup = false;
277 } 272 }
278 timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 273 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
279 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 274 write_lock_irqsave(&rdev->fence_lock, irq_flags);
280 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT; 275 rdev->fence_drv[fence->ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
281 rdev->fence_drv.last_jiffies = jiffies; 276 rdev->fence_drv[fence->ring].last_jiffies = jiffies;
282 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 277 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
283 goto retry; 278 goto retry;
284 } 279 }
285 return 0; 280 return 0;
286} 281}
287 282
288int radeon_fence_wait_next(struct radeon_device *rdev) 283int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
289{ 284{
290 unsigned long irq_flags; 285 unsigned long irq_flags;
291 struct radeon_fence *fence; 286 struct radeon_fence *fence;
@@ -294,21 +289,21 @@ int radeon_fence_wait_next(struct radeon_device *rdev)
294 if (rdev->gpu_lockup) { 289 if (rdev->gpu_lockup) {
295 return 0; 290 return 0;
296 } 291 }
297 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 292 write_lock_irqsave(&rdev->fence_lock, irq_flags);
298 if (list_empty(&rdev->fence_drv.emited)) { 293 if (list_empty(&rdev->fence_drv[ring].emitted)) {
299 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 294 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
300 return 0; 295 return 0;
301 } 296 }
302 fence = list_entry(rdev->fence_drv.emited.next, 297 fence = list_entry(rdev->fence_drv[ring].emitted.next,
303 struct radeon_fence, list); 298 struct radeon_fence, list);
304 radeon_fence_ref(fence); 299 radeon_fence_ref(fence);
305 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 300 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
306 r = radeon_fence_wait(fence, false); 301 r = radeon_fence_wait(fence, false);
307 radeon_fence_unref(&fence); 302 radeon_fence_unref(&fence);
308 return r; 303 return r;
309} 304}
310 305
311int radeon_fence_wait_last(struct radeon_device *rdev) 306int radeon_fence_wait_last(struct radeon_device *rdev, int ring)
312{ 307{
313 unsigned long irq_flags; 308 unsigned long irq_flags;
314 struct radeon_fence *fence; 309 struct radeon_fence *fence;
@@ -317,15 +312,15 @@ int radeon_fence_wait_last(struct radeon_device *rdev)
317 if (rdev->gpu_lockup) { 312 if (rdev->gpu_lockup) {
318 return 0; 313 return 0;
319 } 314 }
320 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 315 write_lock_irqsave(&rdev->fence_lock, irq_flags);
321 if (list_empty(&rdev->fence_drv.emited)) { 316 if (list_empty(&rdev->fence_drv[ring].emitted)) {
322 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 317 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
323 return 0; 318 return 0;
324 } 319 }
325 fence = list_entry(rdev->fence_drv.emited.prev, 320 fence = list_entry(rdev->fence_drv[ring].emitted.prev,
326 struct radeon_fence, list); 321 struct radeon_fence, list);
327 radeon_fence_ref(fence); 322 radeon_fence_ref(fence);
328 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 323 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
329 r = radeon_fence_wait(fence, false); 324 r = radeon_fence_wait(fence, false);
330 radeon_fence_unref(&fence); 325 radeon_fence_unref(&fence);
331 return r; 326 return r;
@@ -347,39 +342,95 @@ void radeon_fence_unref(struct radeon_fence **fence)
347 } 342 }
348} 343}
349 344
350void radeon_fence_process(struct radeon_device *rdev) 345void radeon_fence_process(struct radeon_device *rdev, int ring)
351{ 346{
352 unsigned long irq_flags; 347 unsigned long irq_flags;
353 bool wake; 348 bool wake;
354 349
355 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 350 write_lock_irqsave(&rdev->fence_lock, irq_flags);
356 wake = radeon_fence_poll_locked(rdev); 351 wake = radeon_fence_poll_locked(rdev, ring);
357 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 352 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
358 if (wake) { 353 if (wake) {
359 wake_up_all(&rdev->fence_drv.queue); 354 wake_up_all(&rdev->fence_drv[ring].queue);
360 } 355 }
361} 356}
362 357
363int radeon_fence_driver_init(struct radeon_device *rdev) 358int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
359{
360 unsigned long irq_flags;
361 int not_processed = 0;
362
363 read_lock_irqsave(&rdev->fence_lock, irq_flags);
364 if (!rdev->fence_drv[ring].initialized)
365 return 0;
366
367 if (!list_empty(&rdev->fence_drv[ring].emitted)) {
368 struct list_head *ptr;
369 list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
370 /* count up to 3, that's enought info */
371 if (++not_processed >= 3)
372 break;
373 }
374 }
375 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
376 return not_processed;
377}
378
379int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
364{ 380{
365 unsigned long irq_flags; 381 unsigned long irq_flags;
382 uint64_t index;
366 int r; 383 int r;
367 384
368 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 385 write_lock_irqsave(&rdev->fence_lock, irq_flags);
369 r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg); 386 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
370 if (r) { 387 if (rdev->wb.use_event) {
371 dev_err(rdev->dev, "fence failed to get scratch register\n"); 388 rdev->fence_drv[ring].scratch_reg = 0;
372 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 389 index = R600_WB_EVENT_OFFSET + ring * 4;
373 return r; 390 } else {
391 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
392 if (r) {
393 dev_err(rdev->dev, "fence failed to get scratch register\n");
394 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
395 return r;
396 }
397 index = RADEON_WB_SCRATCH_OFFSET +
398 rdev->fence_drv[ring].scratch_reg -
399 rdev->scratch.reg_base;
374 } 400 }
375 radeon_fence_write(rdev, 0); 401 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
376 atomic_set(&rdev->fence_drv.seq, 0); 402 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
377 INIT_LIST_HEAD(&rdev->fence_drv.created); 403 radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
378 INIT_LIST_HEAD(&rdev->fence_drv.emited); 404 rdev->fence_drv[ring].initialized = true;
379 INIT_LIST_HEAD(&rdev->fence_drv.signaled); 405 DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
380 init_waitqueue_head(&rdev->fence_drv.queue); 406 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
381 rdev->fence_drv.initialized = true; 407 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
382 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 408 return 0;
409}
410
411static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
412{
413 rdev->fence_drv[ring].scratch_reg = -1;
414 rdev->fence_drv[ring].cpu_addr = NULL;
415 rdev->fence_drv[ring].gpu_addr = 0;
416 atomic_set(&rdev->fence_drv[ring].seq, 0);
417 INIT_LIST_HEAD(&rdev->fence_drv[ring].created);
418 INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
419 INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
420 init_waitqueue_head(&rdev->fence_drv[ring].queue);
421 rdev->fence_drv[ring].initialized = false;
422}
423
424int radeon_fence_driver_init(struct radeon_device *rdev)
425{
426 unsigned long irq_flags;
427 int ring;
428
429 write_lock_irqsave(&rdev->fence_lock, irq_flags);
430 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
431 radeon_fence_driver_init_ring(rdev, ring);
432 }
433 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
383 if (radeon_debugfs_fence_init(rdev)) { 434 if (radeon_debugfs_fence_init(rdev)) {
384 dev_err(rdev->dev, "fence debugfs file creation failed\n"); 435 dev_err(rdev->dev, "fence debugfs file creation failed\n");
385 } 436 }
@@ -389,14 +440,18 @@ int radeon_fence_driver_init(struct radeon_device *rdev)
389void radeon_fence_driver_fini(struct radeon_device *rdev) 440void radeon_fence_driver_fini(struct radeon_device *rdev)
390{ 441{
391 unsigned long irq_flags; 442 unsigned long irq_flags;
392 443 int ring;
393 if (!rdev->fence_drv.initialized) 444
394 return; 445 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
395 wake_up_all(&rdev->fence_drv.queue); 446 if (!rdev->fence_drv[ring].initialized)
396 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 447 continue;
397 radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg); 448 radeon_fence_wait_last(rdev, ring);
398 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags); 449 wake_up_all(&rdev->fence_drv[ring].queue);
399 rdev->fence_drv.initialized = false; 450 write_lock_irqsave(&rdev->fence_lock, irq_flags);
451 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
452 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
453 rdev->fence_drv[ring].initialized = false;
454 }
400} 455}
401 456
402 457
@@ -410,14 +465,21 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
410 struct drm_device *dev = node->minor->dev; 465 struct drm_device *dev = node->minor->dev;
411 struct radeon_device *rdev = dev->dev_private; 466 struct radeon_device *rdev = dev->dev_private;
412 struct radeon_fence *fence; 467 struct radeon_fence *fence;
413 468 int i;
414 seq_printf(m, "Last signaled fence 0x%08X\n", 469
415 radeon_fence_read(rdev)); 470 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
416 if (!list_empty(&rdev->fence_drv.emited)) { 471 if (!rdev->fence_drv[i].initialized)
417 fence = list_entry(rdev->fence_drv.emited.prev, 472 continue;
418 struct radeon_fence, list); 473
419 seq_printf(m, "Last emited fence %p with 0x%08X\n", 474 seq_printf(m, "--- ring %d ---\n", i);
420 fence, fence->seq); 475 seq_printf(m, "Last signaled fence 0x%08X\n",
476 radeon_fence_read(rdev, i));
477 if (!list_empty(&rdev->fence_drv[i].emitted)) {
478 fence = list_entry(rdev->fence_drv[i].emitted.prev,
479 struct radeon_fence, list);
480 seq_printf(m, "Last emitted fence %p with 0x%08X\n",
481 fence, fence->seq);
482 }
421 } 483 }
422 return 0; 484 return 0;
423} 485}
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index aa1ca2dea42..ae321975283 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -152,6 +152,7 @@ int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
152 struct radeon_device *rdev = dev->dev_private; 152 struct radeon_device *rdev = dev->dev_private;
153 struct drm_radeon_gem_info *args = data; 153 struct drm_radeon_gem_info *args = data;
154 struct ttm_mem_type_manager *man; 154 struct ttm_mem_type_manager *man;
155 unsigned i;
155 156
156 man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 157 man = &rdev->mman.bdev.man[TTM_PL_VRAM];
157 158
@@ -160,8 +161,9 @@ int radeon_gem_info_ioctl(struct drm_device *dev, void *data,
160 if (rdev->stollen_vga_memory) 161 if (rdev->stollen_vga_memory)
161 args->vram_visible -= radeon_bo_size(rdev->stollen_vga_memory); 162 args->vram_visible -= radeon_bo_size(rdev->stollen_vga_memory);
162 args->vram_visible -= radeon_fbdev_total_size(rdev); 163 args->vram_visible -= radeon_fbdev_total_size(rdev);
163 args->gart_size = rdev->mc.gtt_size - rdev->cp.ring_size - 4096 - 164 args->gart_size = rdev->mc.gtt_size - 4096 - RADEON_IB_POOL_SIZE*64*1024;
164 RADEON_IB_POOL_SIZE*64*1024; 165 for(i = 0; i < RADEON_NUM_RINGS; ++i)
166 args->gart_size -= rdev->ring[i].ring_size;
165 return 0; 167 return 0;
166} 168}
167 169
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 8f86aeb2669..be38921bf76 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -65,7 +65,8 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
65 unsigned i; 65 unsigned i;
66 66
67 /* Disable *all* interrupts */ 67 /* Disable *all* interrupts */
68 rdev->irq.sw_int = false; 68 for (i = 0; i < RADEON_NUM_RINGS; i++)
69 rdev->irq.sw_int[i] = false;
69 rdev->irq.gui_idle = false; 70 rdev->irq.gui_idle = false;
70 for (i = 0; i < RADEON_MAX_HPD_PINS; i++) 71 for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
71 rdev->irq.hpd[i] = false; 72 rdev->irq.hpd[i] = false;
@@ -81,9 +82,11 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
81int radeon_driver_irq_postinstall_kms(struct drm_device *dev) 82int radeon_driver_irq_postinstall_kms(struct drm_device *dev)
82{ 83{
83 struct radeon_device *rdev = dev->dev_private; 84 struct radeon_device *rdev = dev->dev_private;
85 unsigned i;
84 86
85 dev->max_vblank_count = 0x001fffff; 87 dev->max_vblank_count = 0x001fffff;
86 rdev->irq.sw_int = true; 88 for (i = 0; i < RADEON_NUM_RINGS; i++)
89 rdev->irq.sw_int[i] = true;
87 radeon_irq_set(rdev); 90 radeon_irq_set(rdev);
88 return 0; 91 return 0;
89} 92}
@@ -97,7 +100,8 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
97 return; 100 return;
98 } 101 }
99 /* Disable *all* interrupts */ 102 /* Disable *all* interrupts */
100 rdev->irq.sw_int = false; 103 for (i = 0; i < RADEON_NUM_RINGS; i++)
104 rdev->irq.sw_int[i] = false;
101 rdev->irq.gui_idle = false; 105 rdev->irq.gui_idle = false;
102 for (i = 0; i < RADEON_MAX_HPD_PINS; i++) 106 for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
103 rdev->irq.hpd[i] = false; 107 rdev->irq.hpd[i] = false;
@@ -194,26 +198,26 @@ void radeon_irq_kms_fini(struct radeon_device *rdev)
194 flush_work_sync(&rdev->hotplug_work); 198 flush_work_sync(&rdev->hotplug_work);
195} 199}
196 200
197void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev) 201void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring)
198{ 202{
199 unsigned long irqflags; 203 unsigned long irqflags;
200 204
201 spin_lock_irqsave(&rdev->irq.sw_lock, irqflags); 205 spin_lock_irqsave(&rdev->irq.sw_lock, irqflags);
202 if (rdev->ddev->irq_enabled && (++rdev->irq.sw_refcount == 1)) { 206 if (rdev->ddev->irq_enabled && (++rdev->irq.sw_refcount[ring] == 1)) {
203 rdev->irq.sw_int = true; 207 rdev->irq.sw_int[ring] = true;
204 radeon_irq_set(rdev); 208 radeon_irq_set(rdev);
205 } 209 }
206 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags); 210 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
207} 211}
208 212
209void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev) 213void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring)
210{ 214{
211 unsigned long irqflags; 215 unsigned long irqflags;
212 216
213 spin_lock_irqsave(&rdev->irq.sw_lock, irqflags); 217 spin_lock_irqsave(&rdev->irq.sw_lock, irqflags);
214 BUG_ON(rdev->ddev->irq_enabled && rdev->irq.sw_refcount <= 0); 218 BUG_ON(rdev->ddev->irq_enabled && rdev->irq.sw_refcount[ring] <= 0);
215 if (rdev->ddev->irq_enabled && (--rdev->irq.sw_refcount == 0)) { 219 if (rdev->ddev->irq_enabled && (--rdev->irq.sw_refcount[ring] == 0)) {
216 rdev->irq.sw_int = false; 220 rdev->irq.sw_int[ring] = false;
217 radeon_irq_set(rdev); 221 radeon_irq_set(rdev);
218 } 222 }
219 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags); 223 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index b07f0f9b862..cc236fb128a 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -128,4 +128,24 @@ extern void radeon_bo_move_notify(struct ttm_buffer_object *bo,
128 struct ttm_mem_reg *mem); 128 struct ttm_mem_reg *mem);
129extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo); 129extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
130extern int radeon_bo_get_surface_reg(struct radeon_bo *bo); 130extern int radeon_bo_get_surface_reg(struct radeon_bo *bo);
131
132/*
133 * sub allocation
134 */
135extern int radeon_sa_bo_manager_init(struct radeon_device *rdev,
136 struct radeon_sa_manager *sa_manager,
137 unsigned size, u32 domain);
138extern void radeon_sa_bo_manager_fini(struct radeon_device *rdev,
139 struct radeon_sa_manager *sa_manager);
140extern int radeon_sa_bo_manager_start(struct radeon_device *rdev,
141 struct radeon_sa_manager *sa_manager);
142extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
143 struct radeon_sa_manager *sa_manager);
144extern int radeon_sa_bo_new(struct radeon_device *rdev,
145 struct radeon_sa_manager *sa_manager,
146 struct radeon_sa_bo *sa_bo,
147 unsigned size, unsigned align);
148extern void radeon_sa_bo_free(struct radeon_device *rdev,
149 struct radeon_sa_bo *sa_bo);
150
131#endif 151#endif
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 78a665bd951..095148e29a1 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -252,7 +252,10 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
252 252
253 mutex_lock(&rdev->ddev->struct_mutex); 253 mutex_lock(&rdev->ddev->struct_mutex);
254 mutex_lock(&rdev->vram_mutex); 254 mutex_lock(&rdev->vram_mutex);
255 mutex_lock(&rdev->cp.mutex); 255 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
256 if (rdev->ring[i].ring_obj)
257 mutex_lock(&rdev->ring[i].mutex);
258 }
256 259
257 /* gui idle int has issues on older chips it seems */ 260 /* gui idle int has issues on older chips it seems */
258 if (rdev->family >= CHIP_R600) { 261 if (rdev->family >= CHIP_R600) {
@@ -268,12 +271,13 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
268 radeon_irq_set(rdev); 271 radeon_irq_set(rdev);
269 } 272 }
270 } else { 273 } else {
271 if (rdev->cp.ready) { 274 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
275 if (ring->ready) {
272 struct radeon_fence *fence; 276 struct radeon_fence *fence;
273 radeon_ring_alloc(rdev, 64); 277 radeon_ring_alloc(rdev, ring, 64);
274 radeon_fence_create(rdev, &fence); 278 radeon_fence_create(rdev, &fence, radeon_ring_index(rdev, ring));
275 radeon_fence_emit(rdev, fence); 279 radeon_fence_emit(rdev, fence);
276 radeon_ring_commit(rdev); 280 radeon_ring_commit(rdev, ring);
277 radeon_fence_wait(fence, false); 281 radeon_fence_wait(fence, false);
278 radeon_fence_unref(&fence); 282 radeon_fence_unref(&fence);
279 } 283 }
@@ -307,7 +311,10 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
307 311
308 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE; 312 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
309 313
310 mutex_unlock(&rdev->cp.mutex); 314 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
315 if (rdev->ring[i].ring_obj)
316 mutex_unlock(&rdev->ring[i].mutex);
317 }
311 mutex_unlock(&rdev->vram_mutex); 318 mutex_unlock(&rdev->vram_mutex);
312 mutex_unlock(&rdev->ddev->struct_mutex); 319 mutex_unlock(&rdev->ddev->struct_mutex);
313} 320}
@@ -795,19 +802,14 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work)
795 resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev); 802 resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);
796 mutex_lock(&rdev->pm.mutex); 803 mutex_lock(&rdev->pm.mutex);
797 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE) { 804 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE) {
798 unsigned long irq_flags;
799 int not_processed = 0; 805 int not_processed = 0;
806 int i;
800 807
801 read_lock_irqsave(&rdev->fence_drv.lock, irq_flags); 808 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
802 if (!list_empty(&rdev->fence_drv.emited)) { 809 not_processed += radeon_fence_count_emitted(rdev, i);
803 struct list_head *ptr; 810 if (not_processed >= 3)
804 list_for_each(ptr, &rdev->fence_drv.emited) { 811 break;
805 /* count up to 3, that's enought info */
806 if (++not_processed >= 3)
807 break;
808 }
809 } 812 }
810 read_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
811 813
812 if (not_processed >= 3) { /* should upclock */ 814 if (not_processed >= 3) { /* should upclock */
813 if (rdev->pm.dynpm_planned_action == DYNPM_ACTION_DOWNCLOCK) { 815 if (rdev->pm.dynpm_planned_action == DYNPM_ACTION_DOWNCLOCK) {
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 49d58202202..f6a4fbd102a 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -34,6 +34,7 @@
34#include "atom.h" 34#include "atom.h"
35 35
36int radeon_debugfs_ib_init(struct radeon_device *rdev); 36int radeon_debugfs_ib_init(struct radeon_device *rdev);
37int radeon_debugfs_ring_init(struct radeon_device *rdev);
37 38
38u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) 39u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
39{ 40{
@@ -60,105 +61,103 @@ u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
60 return idx_value; 61 return idx_value;
61} 62}
62 63
63void radeon_ring_write(struct radeon_device *rdev, uint32_t v) 64void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
64{ 65{
65#if DRM_DEBUG_CODE 66#if DRM_DEBUG_CODE
66 if (rdev->cp.count_dw <= 0) { 67 if (ring->count_dw <= 0) {
67 DRM_ERROR("radeon: writting more dword to ring than expected !\n"); 68 DRM_ERROR("radeon: writting more dword to ring than expected !\n");
68 } 69 }
69#endif 70#endif
70 rdev->cp.ring[rdev->cp.wptr++] = v; 71 ring->ring[ring->wptr++] = v;
71 rdev->cp.wptr &= rdev->cp.ptr_mask; 72 ring->wptr &= ring->ptr_mask;
72 rdev->cp.count_dw--; 73 ring->count_dw--;
73 rdev->cp.ring_free_dw--; 74 ring->ring_free_dw--;
74} 75}
75 76
76void radeon_ib_bogus_cleanup(struct radeon_device *rdev) 77/*
77{ 78 * IB.
78 struct radeon_ib *ib, *n; 79 */
79 80static bool radeon_ib_try_free(struct radeon_device *rdev,
80 list_for_each_entry_safe(ib, n, &rdev->ib_pool.bogus_ib, list) { 81 struct radeon_ib *ib)
81 list_del(&ib->list);
82 vfree(ib->ptr);
83 kfree(ib);
84 }
85}
86
87void radeon_ib_bogus_add(struct radeon_device *rdev, struct radeon_ib *ib)
88{ 82{
89 struct radeon_ib *bib; 83 bool done = false;
90 84
91 bib = kmalloc(sizeof(*bib), GFP_KERNEL); 85 /* only free ib which have been emited */
92 if (bib == NULL) 86 if (ib->fence && ib->fence->emitted) {
93 return; 87 if (radeon_fence_signaled(ib->fence)) {
94 bib->ptr = vmalloc(ib->length_dw * 4); 88 radeon_fence_unref(&ib->fence);
95 if (bib->ptr == NULL) { 89 radeon_sa_bo_free(rdev, &ib->sa_bo);
96 kfree(bib); 90 done = true;
97 return; 91 }
98 } 92 }
99 memcpy(bib->ptr, ib->ptr, ib->length_dw * 4); 93 return done;
100 bib->length_dw = ib->length_dw;
101 mutex_lock(&rdev->ib_pool.mutex);
102 list_add_tail(&bib->list, &rdev->ib_pool.bogus_ib);
103 mutex_unlock(&rdev->ib_pool.mutex);
104} 94}
105 95
106/* 96int radeon_ib_get(struct radeon_device *rdev, int ring, struct radeon_ib **ib)
107 * IB.
108 */
109int radeon_ib_get(struct radeon_device *rdev, struct radeon_ib **ib)
110{ 97{
111 struct radeon_fence *fence; 98 struct radeon_fence *fence;
112 struct radeon_ib *nib; 99 unsigned cretry = 0;
113 int r = 0, i, c; 100 int r = 0, i, idx;
114 101
115 *ib = NULL; 102 *ib = NULL;
116 r = radeon_fence_create(rdev, &fence); 103
104 r = radeon_fence_create(rdev, &fence, ring);
117 if (r) { 105 if (r) {
118 dev_err(rdev->dev, "failed to create fence for new IB\n"); 106 dev_err(rdev->dev, "failed to create fence for new IB\n");
119 return r; 107 return r;
120 } 108 }
109
121 mutex_lock(&rdev->ib_pool.mutex); 110 mutex_lock(&rdev->ib_pool.mutex);
122 for (i = rdev->ib_pool.head_id, c = 0, nib = NULL; c < RADEON_IB_POOL_SIZE; c++, i++) { 111 idx = rdev->ib_pool.head_id;
123 i &= (RADEON_IB_POOL_SIZE - 1); 112retry:
124 if (rdev->ib_pool.ibs[i].free) { 113 if (cretry > 5) {
125 nib = &rdev->ib_pool.ibs[i]; 114 dev_err(rdev->dev, "failed to get an ib after 5 retry\n");
126 break;
127 }
128 }
129 if (nib == NULL) {
130 /* This should never happen, it means we allocated all
131 * IB and haven't scheduled one yet, return EBUSY to
132 * userspace hoping that on ioctl recall we get better
133 * luck
134 */
135 dev_err(rdev->dev, "no free indirect buffer !\n");
136 mutex_unlock(&rdev->ib_pool.mutex); 115 mutex_unlock(&rdev->ib_pool.mutex);
137 radeon_fence_unref(&fence); 116 radeon_fence_unref(&fence);
138 return -EBUSY; 117 return -ENOMEM;
139 } 118 }
140 rdev->ib_pool.head_id = (nib->idx + 1) & (RADEON_IB_POOL_SIZE - 1); 119 cretry++;
141 nib->free = false; 120 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
142 if (nib->fence) { 121 radeon_ib_try_free(rdev, &rdev->ib_pool.ibs[idx]);
143 mutex_unlock(&rdev->ib_pool.mutex); 122 if (rdev->ib_pool.ibs[idx].fence == NULL) {
144 r = radeon_fence_wait(nib->fence, false); 123 r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager,
145 if (r) { 124 &rdev->ib_pool.ibs[idx].sa_bo,
146 dev_err(rdev->dev, "error waiting fence of IB(%u:0x%016lX:%u)\n", 125 64*1024, 64);
147 nib->idx, (unsigned long)nib->gpu_addr, nib->length_dw); 126 if (!r) {
148 mutex_lock(&rdev->ib_pool.mutex); 127 *ib = &rdev->ib_pool.ibs[idx];
149 nib->free = true; 128 (*ib)->ptr = rdev->ib_pool.sa_manager.cpu_ptr;
150 mutex_unlock(&rdev->ib_pool.mutex); 129 (*ib)->ptr += ((*ib)->sa_bo.offset >> 2);
151 radeon_fence_unref(&fence); 130 (*ib)->gpu_addr = rdev->ib_pool.sa_manager.gpu_addr;
152 return r; 131 (*ib)->gpu_addr += (*ib)->sa_bo.offset;
132 (*ib)->fence = fence;
133 /* ib are most likely to be allocated in a ring fashion
134 * thus rdev->ib_pool.head_id should be the id of the
135 * oldest ib
136 */
137 rdev->ib_pool.head_id = (1 + idx);
138 rdev->ib_pool.head_id &= (RADEON_IB_POOL_SIZE - 1);
139 mutex_unlock(&rdev->ib_pool.mutex);
140 return 0;
141 }
153 } 142 }
154 mutex_lock(&rdev->ib_pool.mutex); 143 idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1);
144 }
145 /* this should be rare event, ie all ib scheduled none signaled yet.
146 */
147 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
148 if (rdev->ib_pool.ibs[idx].fence) {
149 r = radeon_fence_wait(rdev->ib_pool.ibs[idx].fence, false);
150 if (!r) {
151 goto retry;
152 }
153 /* an error happened */
154 break;
155 }
156 idx = (idx + 1) & (RADEON_IB_POOL_SIZE - 1);
155 } 157 }
156 radeon_fence_unref(&nib->fence);
157 nib->fence = fence;
158 nib->length_dw = 0;
159 mutex_unlock(&rdev->ib_pool.mutex); 158 mutex_unlock(&rdev->ib_pool.mutex);
160 *ib = nib; 159 radeon_fence_unref(&fence);
161 return 0; 160 return r;
162} 161}
163 162
164void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib) 163void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
@@ -169,247 +168,255 @@ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib **ib)
169 if (tmp == NULL) { 168 if (tmp == NULL) {
170 return; 169 return;
171 } 170 }
172 if (!tmp->fence->emited)
173 radeon_fence_unref(&tmp->fence);
174 mutex_lock(&rdev->ib_pool.mutex); 171 mutex_lock(&rdev->ib_pool.mutex);
175 tmp->free = true; 172 if (tmp->fence && !tmp->fence->emitted) {
173 radeon_sa_bo_free(rdev, &tmp->sa_bo);
174 radeon_fence_unref(&tmp->fence);
175 }
176 mutex_unlock(&rdev->ib_pool.mutex); 176 mutex_unlock(&rdev->ib_pool.mutex);
177} 177}
178 178
179int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib) 179int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib)
180{ 180{
181 struct radeon_ring *ring = &rdev->ring[ib->fence->ring];
181 int r = 0; 182 int r = 0;
182 183
183 if (!ib->length_dw || !rdev->cp.ready) { 184 if (!ib->length_dw || !ring->ready) {
184 /* TODO: Nothings in the ib we should report. */ 185 /* TODO: Nothings in the ib we should report. */
185 DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx); 186 DRM_ERROR("radeon: couldn't schedule IB(%u).\n", ib->idx);
186 return -EINVAL; 187 return -EINVAL;
187 } 188 }
188 189
189 /* 64 dwords should be enough for fence too */ 190 /* 64 dwords should be enough for fence too */
190 r = radeon_ring_lock(rdev, 64); 191 r = radeon_ring_lock(rdev, ring, 64);
191 if (r) { 192 if (r) {
192 DRM_ERROR("radeon: scheduling IB failed (%d).\n", r); 193 DRM_ERROR("radeon: scheduling IB failed (%d).\n", r);
193 return r; 194 return r;
194 } 195 }
195 radeon_ring_ib_execute(rdev, ib); 196 radeon_ring_ib_execute(rdev, ib->fence->ring, ib);
196 radeon_fence_emit(rdev, ib->fence); 197 radeon_fence_emit(rdev, ib->fence);
197 mutex_lock(&rdev->ib_pool.mutex); 198 radeon_ring_unlock_commit(rdev, ring);
198 /* once scheduled IB is considered free and protected by the fence */
199 ib->free = true;
200 mutex_unlock(&rdev->ib_pool.mutex);
201 radeon_ring_unlock_commit(rdev);
202 return 0; 199 return 0;
203} 200}
204 201
205int radeon_ib_pool_init(struct radeon_device *rdev) 202int radeon_ib_pool_init(struct radeon_device *rdev)
206{ 203{
207 void *ptr; 204 int i, r;
208 uint64_t gpu_addr;
209 int i;
210 int r = 0;
211 205
212 if (rdev->ib_pool.robj) 206 mutex_lock(&rdev->ib_pool.mutex);
207 if (rdev->ib_pool.ready) {
208 mutex_unlock(&rdev->ib_pool.mutex);
213 return 0; 209 return 0;
214 INIT_LIST_HEAD(&rdev->ib_pool.bogus_ib);
215 /* Allocate 1M object buffer */
216 r = radeon_bo_create(rdev, RADEON_IB_POOL_SIZE*64*1024,
217 PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
218 &rdev->ib_pool.robj);
219 if (r) {
220 DRM_ERROR("radeon: failed to ib pool (%d).\n", r);
221 return r;
222 } 210 }
223 r = radeon_bo_reserve(rdev->ib_pool.robj, false); 211
224 if (unlikely(r != 0)) 212 r = radeon_sa_bo_manager_init(rdev, &rdev->ib_pool.sa_manager,
225 return r; 213 RADEON_IB_POOL_SIZE*64*1024,
226 r = radeon_bo_pin(rdev->ib_pool.robj, RADEON_GEM_DOMAIN_GTT, &gpu_addr); 214 RADEON_GEM_DOMAIN_GTT);
227 if (r) {
228 radeon_bo_unreserve(rdev->ib_pool.robj);
229 DRM_ERROR("radeon: failed to pin ib pool (%d).\n", r);
230 return r;
231 }
232 r = radeon_bo_kmap(rdev->ib_pool.robj, &ptr);
233 radeon_bo_unreserve(rdev->ib_pool.robj);
234 if (r) { 215 if (r) {
235 DRM_ERROR("radeon: failed to map ib pool (%d).\n", r); 216 mutex_unlock(&rdev->ib_pool.mutex);
236 return r; 217 return r;
237 } 218 }
238 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
239 unsigned offset;
240 219
241 offset = i * 64 * 1024; 220 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
242 rdev->ib_pool.ibs[i].gpu_addr = gpu_addr + offset; 221 rdev->ib_pool.ibs[i].fence = NULL;
243 rdev->ib_pool.ibs[i].ptr = ptr + offset;
244 rdev->ib_pool.ibs[i].idx = i; 222 rdev->ib_pool.ibs[i].idx = i;
245 rdev->ib_pool.ibs[i].length_dw = 0; 223 rdev->ib_pool.ibs[i].length_dw = 0;
246 rdev->ib_pool.ibs[i].free = true; 224 INIT_LIST_HEAD(&rdev->ib_pool.ibs[i].sa_bo.list);
247 } 225 }
248 rdev->ib_pool.head_id = 0; 226 rdev->ib_pool.head_id = 0;
249 rdev->ib_pool.ready = true; 227 rdev->ib_pool.ready = true;
250 DRM_INFO("radeon: ib pool ready.\n"); 228 DRM_INFO("radeon: ib pool ready.\n");
229
251 if (radeon_debugfs_ib_init(rdev)) { 230 if (radeon_debugfs_ib_init(rdev)) {
252 DRM_ERROR("Failed to register debugfs file for IB !\n"); 231 DRM_ERROR("Failed to register debugfs file for IB !\n");
253 } 232 }
254 return r; 233 if (radeon_debugfs_ring_init(rdev)) {
234 DRM_ERROR("Failed to register debugfs file for rings !\n");
235 }
236 mutex_unlock(&rdev->ib_pool.mutex);
237 return 0;
255} 238}
256 239
257void radeon_ib_pool_fini(struct radeon_device *rdev) 240void radeon_ib_pool_fini(struct radeon_device *rdev)
258{ 241{
259 int r; 242 unsigned i;
260 struct radeon_bo *robj;
261 243
262 if (!rdev->ib_pool.ready) {
263 return;
264 }
265 mutex_lock(&rdev->ib_pool.mutex); 244 mutex_lock(&rdev->ib_pool.mutex);
266 radeon_ib_bogus_cleanup(rdev); 245 if (rdev->ib_pool.ready) {
267 robj = rdev->ib_pool.robj; 246 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
268 rdev->ib_pool.robj = NULL; 247 radeon_sa_bo_free(rdev, &rdev->ib_pool.ibs[i].sa_bo);
269 mutex_unlock(&rdev->ib_pool.mutex); 248 radeon_fence_unref(&rdev->ib_pool.ibs[i].fence);
270
271 if (robj) {
272 r = radeon_bo_reserve(robj, false);
273 if (likely(r == 0)) {
274 radeon_bo_kunmap(robj);
275 radeon_bo_unpin(robj);
276 radeon_bo_unreserve(robj);
277 } 249 }
278 radeon_bo_unref(&robj); 250 radeon_sa_bo_manager_fini(rdev, &rdev->ib_pool.sa_manager);
251 rdev->ib_pool.ready = false;
279 } 252 }
253 mutex_unlock(&rdev->ib_pool.mutex);
280} 254}
281 255
256int radeon_ib_pool_start(struct radeon_device *rdev)
257{
258 return radeon_sa_bo_manager_start(rdev, &rdev->ib_pool.sa_manager);
259}
260
261int radeon_ib_pool_suspend(struct radeon_device *rdev)
262{
263 return radeon_sa_bo_manager_suspend(rdev, &rdev->ib_pool.sa_manager);
264}
282 265
283/* 266/*
284 * Ring. 267 * Ring.
285 */ 268 */
286void radeon_ring_free_size(struct radeon_device *rdev) 269int radeon_ring_index(struct radeon_device *rdev, struct radeon_ring *ring)
287{ 270{
288 if (rdev->wb.enabled) 271 /* r1xx-r5xx only has CP ring */
289 rdev->cp.rptr = le32_to_cpu(rdev->wb.wb[RADEON_WB_CP_RPTR_OFFSET/4]); 272 if (rdev->family < CHIP_R600)
290 else { 273 return RADEON_RING_TYPE_GFX_INDEX;
291 if (rdev->family >= CHIP_R600) 274
292 rdev->cp.rptr = RREG32(R600_CP_RB_RPTR); 275 if (rdev->family >= CHIP_CAYMAN) {
293 else 276 if (ring == &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX])
294 rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR); 277 return CAYMAN_RING_TYPE_CP1_INDEX;
278 else if (ring == &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX])
279 return CAYMAN_RING_TYPE_CP2_INDEX;
295 } 280 }
281 return RADEON_RING_TYPE_GFX_INDEX;
282}
283
284void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
285{
286 u32 rptr;
287
288 if (rdev->wb.enabled)
289 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
290 else
291 rptr = RREG32(ring->rptr_reg);
292 ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
296 /* This works because ring_size is a power of 2 */ 293 /* This works because ring_size is a power of 2 */
297 rdev->cp.ring_free_dw = (rdev->cp.rptr + (rdev->cp.ring_size / 4)); 294 ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
298 rdev->cp.ring_free_dw -= rdev->cp.wptr; 295 ring->ring_free_dw -= ring->wptr;
299 rdev->cp.ring_free_dw &= rdev->cp.ptr_mask; 296 ring->ring_free_dw &= ring->ptr_mask;
300 if (!rdev->cp.ring_free_dw) { 297 if (!ring->ring_free_dw) {
301 rdev->cp.ring_free_dw = rdev->cp.ring_size / 4; 298 ring->ring_free_dw = ring->ring_size / 4;
302 } 299 }
303} 300}
304 301
305int radeon_ring_alloc(struct radeon_device *rdev, unsigned ndw) 302
303int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
306{ 304{
307 int r; 305 int r;
308 306
309 /* Align requested size with padding so unlock_commit can 307 /* Align requested size with padding so unlock_commit can
310 * pad safely */ 308 * pad safely */
311 ndw = (ndw + rdev->cp.align_mask) & ~rdev->cp.align_mask; 309 ndw = (ndw + ring->align_mask) & ~ring->align_mask;
312 while (ndw > (rdev->cp.ring_free_dw - 1)) { 310 while (ndw > (ring->ring_free_dw - 1)) {
313 radeon_ring_free_size(rdev); 311 radeon_ring_free_size(rdev, ring);
314 if (ndw < rdev->cp.ring_free_dw) { 312 if (ndw < ring->ring_free_dw) {
315 break; 313 break;
316 } 314 }
317 r = radeon_fence_wait_next(rdev); 315 r = radeon_fence_wait_next(rdev, radeon_ring_index(rdev, ring));
318 if (r) 316 if (r)
319 return r; 317 return r;
320 } 318 }
321 rdev->cp.count_dw = ndw; 319 ring->count_dw = ndw;
322 rdev->cp.wptr_old = rdev->cp.wptr; 320 ring->wptr_old = ring->wptr;
323 return 0; 321 return 0;
324} 322}
325 323
326int radeon_ring_lock(struct radeon_device *rdev, unsigned ndw) 324int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
327{ 325{
328 int r; 326 int r;
329 327
330 mutex_lock(&rdev->cp.mutex); 328 mutex_lock(&ring->mutex);
331 r = radeon_ring_alloc(rdev, ndw); 329 r = radeon_ring_alloc(rdev, ring, ndw);
332 if (r) { 330 if (r) {
333 mutex_unlock(&rdev->cp.mutex); 331 mutex_unlock(&ring->mutex);
334 return r; 332 return r;
335 } 333 }
336 return 0; 334 return 0;
337} 335}
338 336
339void radeon_ring_commit(struct radeon_device *rdev) 337void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
340{ 338{
341 unsigned count_dw_pad; 339 unsigned count_dw_pad;
342 unsigned i; 340 unsigned i;
343 341
344 /* We pad to match fetch size */ 342 /* We pad to match fetch size */
345 count_dw_pad = (rdev->cp.align_mask + 1) - 343 count_dw_pad = (ring->align_mask + 1) -
346 (rdev->cp.wptr & rdev->cp.align_mask); 344 (ring->wptr & ring->align_mask);
347 for (i = 0; i < count_dw_pad; i++) { 345 for (i = 0; i < count_dw_pad; i++) {
348 radeon_ring_write(rdev, 2 << 30); 346 radeon_ring_write(ring, ring->nop);
349 } 347 }
350 DRM_MEMORYBARRIER(); 348 DRM_MEMORYBARRIER();
351 radeon_cp_commit(rdev); 349 WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
350 (void)RREG32(ring->wptr_reg);
352} 351}
353 352
354void radeon_ring_unlock_commit(struct radeon_device *rdev) 353void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring)
355{ 354{
356 radeon_ring_commit(rdev); 355 radeon_ring_commit(rdev, ring);
357 mutex_unlock(&rdev->cp.mutex); 356 mutex_unlock(&ring->mutex);
358} 357}
359 358
360void radeon_ring_unlock_undo(struct radeon_device *rdev) 359void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
361{ 360{
362 rdev->cp.wptr = rdev->cp.wptr_old; 361 ring->wptr = ring->wptr_old;
363 mutex_unlock(&rdev->cp.mutex); 362 mutex_unlock(&ring->mutex);
364} 363}
365 364
366int radeon_ring_init(struct radeon_device *rdev, unsigned ring_size) 365int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
366 unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg,
367 u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop)
367{ 368{
368 int r; 369 int r;
369 370
370 rdev->cp.ring_size = ring_size; 371 ring->ring_size = ring_size;
372 ring->rptr_offs = rptr_offs;
373 ring->rptr_reg = rptr_reg;
374 ring->wptr_reg = wptr_reg;
375 ring->ptr_reg_shift = ptr_reg_shift;
376 ring->ptr_reg_mask = ptr_reg_mask;
377 ring->nop = nop;
371 /* Allocate ring buffer */ 378 /* Allocate ring buffer */
372 if (rdev->cp.ring_obj == NULL) { 379 if (ring->ring_obj == NULL) {
373 r = radeon_bo_create(rdev, rdev->cp.ring_size, PAGE_SIZE, true, 380 r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
374 RADEON_GEM_DOMAIN_GTT, 381 RADEON_GEM_DOMAIN_GTT,
375 &rdev->cp.ring_obj); 382 &ring->ring_obj);
376 if (r) { 383 if (r) {
377 dev_err(rdev->dev, "(%d) ring create failed\n", r); 384 dev_err(rdev->dev, "(%d) ring create failed\n", r);
378 return r; 385 return r;
379 } 386 }
380 r = radeon_bo_reserve(rdev->cp.ring_obj, false); 387 r = radeon_bo_reserve(ring->ring_obj, false);
381 if (unlikely(r != 0)) 388 if (unlikely(r != 0))
382 return r; 389 return r;
383 r = radeon_bo_pin(rdev->cp.ring_obj, RADEON_GEM_DOMAIN_GTT, 390 r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
384 &rdev->cp.gpu_addr); 391 &ring->gpu_addr);
385 if (r) { 392 if (r) {
386 radeon_bo_unreserve(rdev->cp.ring_obj); 393 radeon_bo_unreserve(ring->ring_obj);
387 dev_err(rdev->dev, "(%d) ring pin failed\n", r); 394 dev_err(rdev->dev, "(%d) ring pin failed\n", r);
388 return r; 395 return r;
389 } 396 }
390 r = radeon_bo_kmap(rdev->cp.ring_obj, 397 r = radeon_bo_kmap(ring->ring_obj,
391 (void **)&rdev->cp.ring); 398 (void **)&ring->ring);
392 radeon_bo_unreserve(rdev->cp.ring_obj); 399 radeon_bo_unreserve(ring->ring_obj);
393 if (r) { 400 if (r) {
394 dev_err(rdev->dev, "(%d) ring map failed\n", r); 401 dev_err(rdev->dev, "(%d) ring map failed\n", r);
395 return r; 402 return r;
396 } 403 }
397 } 404 }
398 rdev->cp.ptr_mask = (rdev->cp.ring_size / 4) - 1; 405 ring->ptr_mask = (ring->ring_size / 4) - 1;
399 rdev->cp.ring_free_dw = rdev->cp.ring_size / 4; 406 ring->ring_free_dw = ring->ring_size / 4;
400 return 0; 407 return 0;
401} 408}
402 409
403void radeon_ring_fini(struct radeon_device *rdev) 410void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
404{ 411{
405 int r; 412 int r;
406 struct radeon_bo *ring_obj; 413 struct radeon_bo *ring_obj;
407 414
408 mutex_lock(&rdev->cp.mutex); 415 mutex_lock(&ring->mutex);
409 ring_obj = rdev->cp.ring_obj; 416 ring_obj = ring->ring_obj;
410 rdev->cp.ring = NULL; 417 ring->ring = NULL;
411 rdev->cp.ring_obj = NULL; 418 ring->ring_obj = NULL;
412 mutex_unlock(&rdev->cp.mutex); 419 mutex_unlock(&ring->mutex);
413 420
414 if (ring_obj) { 421 if (ring_obj) {
415 r = radeon_bo_reserve(ring_obj, false); 422 r = radeon_bo_reserve(ring_obj, false);
@@ -422,72 +429,83 @@ void radeon_ring_fini(struct radeon_device *rdev)
422 } 429 }
423} 430}
424 431
425
426/* 432/*
427 * Debugfs info 433 * Debugfs info
428 */ 434 */
429#if defined(CONFIG_DEBUG_FS) 435#if defined(CONFIG_DEBUG_FS)
430static int radeon_debugfs_ib_info(struct seq_file *m, void *data) 436
437static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
431{ 438{
432 struct drm_info_node *node = (struct drm_info_node *) m->private; 439 struct drm_info_node *node = (struct drm_info_node *) m->private;
433 struct radeon_ib *ib = node->info_ent->data; 440 struct drm_device *dev = node->minor->dev;
434 unsigned i; 441 struct radeon_device *rdev = dev->dev_private;
435 442 int ridx = *(int*)node->info_ent->data;
436 if (ib == NULL) { 443 struct radeon_ring *ring = &rdev->ring[ridx];
437 return 0; 444 unsigned count, i, j;
438 } 445
439 seq_printf(m, "IB %04u\n", ib->idx); 446 radeon_ring_free_size(rdev, ring);
440 seq_printf(m, "IB fence %p\n", ib->fence); 447 count = (ring->ring_size / 4) - ring->ring_free_dw;
441 seq_printf(m, "IB size %05u dwords\n", ib->length_dw); 448 seq_printf(m, "wptr(0x%04x): 0x%08x\n", ring->wptr_reg, RREG32(ring->wptr_reg));
442 for (i = 0; i < ib->length_dw; i++) { 449 seq_printf(m, "rptr(0x%04x): 0x%08x\n", ring->rptr_reg, RREG32(ring->rptr_reg));
443 seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]); 450 seq_printf(m, "driver's copy of the wptr: 0x%08x\n", ring->wptr);
451 seq_printf(m, "driver's copy of the rptr: 0x%08x\n", ring->rptr);
452 seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
453 seq_printf(m, "%u dwords in ring\n", count);
454 i = ring->rptr;
455 for (j = 0; j <= count; j++) {
456 seq_printf(m, "r[%04d]=0x%08x\n", i, ring->ring[i]);
457 i = (i + 1) & ring->ptr_mask;
444 } 458 }
445 return 0; 459 return 0;
446} 460}
447 461
448static int radeon_debugfs_ib_bogus_info(struct seq_file *m, void *data) 462static int radeon_ring_type_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
463static int cayman_ring_type_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
464static int cayman_ring_type_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
465
466static struct drm_info_list radeon_debugfs_ring_info_list[] = {
467 {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_ring_type_gfx_index},
468 {"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp1_index},
469 {"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp2_index},
470};
471
472static int radeon_debugfs_ib_info(struct seq_file *m, void *data)
449{ 473{
450 struct drm_info_node *node = (struct drm_info_node *) m->private; 474 struct drm_info_node *node = (struct drm_info_node *) m->private;
451 struct radeon_device *rdev = node->info_ent->data; 475 struct radeon_ib *ib = node->info_ent->data;
452 struct radeon_ib *ib;
453 unsigned i; 476 unsigned i;
454 477
455 mutex_lock(&rdev->ib_pool.mutex); 478 if (ib == NULL) {
456 if (list_empty(&rdev->ib_pool.bogus_ib)) {
457 mutex_unlock(&rdev->ib_pool.mutex);
458 seq_printf(m, "no bogus IB recorded\n");
459 return 0; 479 return 0;
460 } 480 }
461 ib = list_first_entry(&rdev->ib_pool.bogus_ib, struct radeon_ib, list); 481 seq_printf(m, "IB %04u\n", ib->idx);
462 list_del_init(&ib->list); 482 seq_printf(m, "IB fence %p\n", ib->fence);
463 mutex_unlock(&rdev->ib_pool.mutex);
464 seq_printf(m, "IB size %05u dwords\n", ib->length_dw); 483 seq_printf(m, "IB size %05u dwords\n", ib->length_dw);
465 for (i = 0; i < ib->length_dw; i++) { 484 for (i = 0; i < ib->length_dw; i++) {
466 seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]); 485 seq_printf(m, "[%05u]=0x%08X\n", i, ib->ptr[i]);
467 } 486 }
468 vfree(ib->ptr);
469 kfree(ib);
470 return 0; 487 return 0;
471} 488}
472 489
473static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE]; 490static struct drm_info_list radeon_debugfs_ib_list[RADEON_IB_POOL_SIZE];
474static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32]; 491static char radeon_debugfs_ib_names[RADEON_IB_POOL_SIZE][32];
492#endif
475 493
476static struct drm_info_list radeon_debugfs_ib_bogus_info_list[] = { 494int radeon_debugfs_ring_init(struct radeon_device *rdev)
477 {"radeon_ib_bogus", radeon_debugfs_ib_bogus_info, 0, NULL}, 495{
478}; 496#if defined(CONFIG_DEBUG_FS)
497 return radeon_debugfs_add_files(rdev, radeon_debugfs_ring_info_list,
498 ARRAY_SIZE(radeon_debugfs_ring_info_list));
499#else
500 return 0;
479#endif 501#endif
502}
480 503
481int radeon_debugfs_ib_init(struct radeon_device *rdev) 504int radeon_debugfs_ib_init(struct radeon_device *rdev)
482{ 505{
483#if defined(CONFIG_DEBUG_FS) 506#if defined(CONFIG_DEBUG_FS)
484 unsigned i; 507 unsigned i;
485 int r;
486 508
487 radeon_debugfs_ib_bogus_info_list[0].data = rdev;
488 r = radeon_debugfs_add_files(rdev, radeon_debugfs_ib_bogus_info_list, 1);
489 if (r)
490 return r;
491 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { 509 for (i = 0; i < RADEON_IB_POOL_SIZE; i++) {
492 sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i); 510 sprintf(radeon_debugfs_ib_names[i], "radeon_ib_%04u", i);
493 radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i]; 511 radeon_debugfs_ib_list[i].name = radeon_debugfs_ib_names[i];
diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c
new file mode 100644
index 00000000000..4cce47e7dc0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_sa.c
@@ -0,0 +1,189 @@
1/*
2 * Copyright 2011 Red Hat Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 */
30#include "drmP.h"
31#include "drm.h"
32#include "radeon.h"
33
34int radeon_sa_bo_manager_init(struct radeon_device *rdev,
35 struct radeon_sa_manager *sa_manager,
36 unsigned size, u32 domain)
37{
38 int r;
39
40 sa_manager->bo = NULL;
41 sa_manager->size = size;
42 sa_manager->domain = domain;
43 INIT_LIST_HEAD(&sa_manager->sa_bo);
44
45 r = radeon_bo_create(rdev, size, RADEON_GPU_PAGE_SIZE, true,
46 RADEON_GEM_DOMAIN_CPU, &sa_manager->bo);
47 if (r) {
48 dev_err(rdev->dev, "(%d) failed to allocate bo for manager\n", r);
49 return r;
50 }
51
52 return r;
53}
54
55void radeon_sa_bo_manager_fini(struct radeon_device *rdev,
56 struct radeon_sa_manager *sa_manager)
57{
58 struct radeon_sa_bo *sa_bo, *tmp;
59
60 if (!list_empty(&sa_manager->sa_bo)) {
61 dev_err(rdev->dev, "sa_manager is not empty, clearing anyway\n");
62 }
63 list_for_each_entry_safe(sa_bo, tmp, &sa_manager->sa_bo, list) {
64 list_del_init(&sa_bo->list);
65 }
66 radeon_bo_unref(&sa_manager->bo);
67 sa_manager->size = 0;
68}
69
70int radeon_sa_bo_manager_start(struct radeon_device *rdev,
71 struct radeon_sa_manager *sa_manager)
72{
73 int r;
74
75 if (sa_manager->bo == NULL) {
76 dev_err(rdev->dev, "no bo for sa manager\n");
77 return -EINVAL;
78 }
79
80 /* map the buffer */
81 r = radeon_bo_reserve(sa_manager->bo, false);
82 if (r) {
83 dev_err(rdev->dev, "(%d) failed to reserve manager bo\n", r);
84 return r;
85 }
86 r = radeon_bo_pin(sa_manager->bo, sa_manager->domain, &sa_manager->gpu_addr);
87 if (r) {
88 radeon_bo_unreserve(sa_manager->bo);
89 dev_err(rdev->dev, "(%d) failed to pin manager bo\n", r);
90 return r;
91 }
92 r = radeon_bo_kmap(sa_manager->bo, &sa_manager->cpu_ptr);
93 radeon_bo_unreserve(sa_manager->bo);
94 return r;
95}
96
97int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
98 struct radeon_sa_manager *sa_manager)
99{
100 int r;
101
102 if (sa_manager->bo == NULL) {
103 dev_err(rdev->dev, "no bo for sa manager\n");
104 return -EINVAL;
105 }
106
107 r = radeon_bo_reserve(sa_manager->bo, false);
108 if (!r) {
109 radeon_bo_kunmap(sa_manager->bo);
110 radeon_bo_unpin(sa_manager->bo);
111 radeon_bo_unreserve(sa_manager->bo);
112 }
113 return r;
114}
115
116/*
117 * Principe is simple, we keep a list of sub allocation in offset
118 * order (first entry has offset == 0, last entry has the highest
119 * offset).
120 *
121 * When allocating new object we first check if there is room at
122 * the end total_size - (last_object_offset + last_object_size) >=
123 * alloc_size. If so we allocate new object there.
124 *
125 * When there is not enough room at the end, we start waiting for
126 * each sub object until we reach object_offset+object_size >=
127 * alloc_size, this object then become the sub object we return.
128 *
129 * Alignment can't be bigger than page size
130 */
131int radeon_sa_bo_new(struct radeon_device *rdev,
132 struct radeon_sa_manager *sa_manager,
133 struct radeon_sa_bo *sa_bo,
134 unsigned size, unsigned align)
135{
136 struct radeon_sa_bo *tmp;
137 struct list_head *head;
138 unsigned offset = 0, wasted = 0;
139
140 BUG_ON(align > RADEON_GPU_PAGE_SIZE);
141 BUG_ON(size > sa_manager->size);
142
143 /* no one ? */
144 head = sa_manager->sa_bo.prev;
145 if (list_empty(&sa_manager->sa_bo)) {
146 goto out;
147 }
148
149 /* look for a hole big enough */
150 offset = 0;
151 list_for_each_entry(tmp, &sa_manager->sa_bo, list) {
152 /* room before this object ? */
153 if ((tmp->offset - offset) >= size) {
154 head = tmp->list.prev;
155 goto out;
156 }
157 offset = tmp->offset + tmp->size;
158 wasted = offset % align;
159 if (wasted) {
160 wasted = align - wasted;
161 }
162 offset += wasted;
163 }
164 /* room at the end ? */
165 head = sa_manager->sa_bo.prev;
166 tmp = list_entry(head, struct radeon_sa_bo, list);
167 offset = tmp->offset + tmp->size;
168 wasted = offset % align;
169 if (wasted) {
170 wasted = align - wasted;
171 }
172 offset += wasted;
173 if ((sa_manager->size - offset) < size) {
174 /* failed to find somethings big enough */
175 return -ENOMEM;
176 }
177
178out:
179 sa_bo->manager = sa_manager;
180 sa_bo->offset = offset;
181 sa_bo->size = size;
182 list_add(&sa_bo->list, head);
183 return 0;
184}
185
186void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo *sa_bo)
187{
188 list_del_init(&sa_bo->list);
189}
diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c
new file mode 100644
index 00000000000..f8cf04499a5
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_semaphore.c
@@ -0,0 +1,161 @@
1/*
2 * Copyright 2011 Christian König.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Christian König <deathsimple@vodafone.de>
29 */
30#include "drmP.h"
31#include "drm.h"
32#include "radeon.h"
33
34static int allocate_semaphores(struct radeon_device *rdev)
35{
36 const unsigned long bo_size = PAGE_SIZE * 4;
37
38 struct radeon_bo *bo;
39 struct list_head new_entrys;
40 unsigned long irq_flags;
41 uint64_t gpu_addr;
42 void *map;
43 int i, r;
44
45 r = radeon_bo_create(rdev, bo_size, RADEON_GPU_PAGE_SIZE, true,
46 RADEON_GEM_DOMAIN_GTT, &bo);
47 if (r) {
48 dev_err(rdev->dev, "(%d) failed to allocate semaphore bo\n", r);
49 return r;
50 }
51
52 r = radeon_bo_reserve(bo, false);
53 if (r) {
54 radeon_bo_unref(&bo);
55 dev_err(rdev->dev, "(%d) failed to reserve semaphore bo\n", r);
56 return r;
57 }
58
59 r = radeon_bo_kmap(bo, &map);
60 if (r) {
61 radeon_bo_unreserve(bo);
62 radeon_bo_unref(&bo);
63 dev_err(rdev->dev, "(%d) semaphore map failed\n", r);
64 return r;
65 }
66 memset(map, 0, bo_size);
67 radeon_bo_kunmap(bo);
68
69 r = radeon_bo_pin(bo, RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
70 if (r) {
71 radeon_bo_unreserve(bo);
72 radeon_bo_unref(&bo);
73 dev_err(rdev->dev, "(%d) semaphore pin failed\n", r);
74 return r;
75 }
76
77 INIT_LIST_HEAD(&new_entrys);
78 for (i = 0; i < bo_size/8; ++i) {
79 struct radeon_semaphore *sem = kmalloc(sizeof(struct radeon_semaphore), GFP_KERNEL);
80 ttm_bo_reference(&bo->tbo);
81 sem->robj = bo;
82 sem->gpu_addr = gpu_addr;
83 gpu_addr += 8;
84 list_add_tail(&sem->list, &new_entrys);
85 }
86
87 radeon_bo_unreserve(bo);
88 radeon_bo_unref(&bo);
89
90 write_lock_irqsave(&rdev->semaphore_drv.lock, irq_flags);
91 list_splice_tail(&new_entrys, &rdev->semaphore_drv.free);
92 write_unlock_irqrestore(&rdev->semaphore_drv.lock, irq_flags);
93
94 DRM_INFO("%d new semaphores allocated\n", (int)(bo_size/8));
95
96 return 0;
97}
98
99int radeon_semaphore_create(struct radeon_device *rdev,
100 struct radeon_semaphore **semaphore)
101{
102 unsigned long irq_flags;
103
104 write_lock_irqsave(&rdev->semaphore_drv.lock, irq_flags);
105 if (list_empty(&rdev->semaphore_drv.free)) {
106 int r;
107 write_unlock_irqrestore(&rdev->semaphore_drv.lock, irq_flags);
108 r = allocate_semaphores(rdev);
109 if (r)
110 return r;
111 write_lock_irqsave(&rdev->semaphore_drv.lock, irq_flags);
112 }
113
114 *semaphore = list_first_entry(&rdev->semaphore_drv.free, struct radeon_semaphore, list);
115 list_del(&(*semaphore)->list);
116
117 write_unlock_irqrestore(&rdev->semaphore_drv.lock, irq_flags);
118 return 0;
119}
120
121void radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
122 struct radeon_semaphore *semaphore)
123{
124 radeon_semaphore_ring_emit(rdev, ring, &rdev->ring[ring], semaphore, false);
125}
126
127void radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
128 struct radeon_semaphore *semaphore)
129{
130 radeon_semaphore_ring_emit(rdev, ring, &rdev->ring[ring], semaphore, true);
131}
132
133void radeon_semaphore_free(struct radeon_device *rdev,
134 struct radeon_semaphore *semaphore)
135{
136 unsigned long irq_flags;
137
138 write_lock_irqsave(&rdev->semaphore_drv.lock, irq_flags);
139 list_add_tail(&semaphore->list, &rdev->semaphore_drv.free);
140 write_unlock_irqrestore(&rdev->semaphore_drv.lock, irq_flags);
141}
142
143void radeon_semaphore_driver_fini(struct radeon_device *rdev)
144{
145 struct radeon_semaphore *i, *n;
146 struct list_head entrys;
147 unsigned long irq_flags;
148
149 INIT_LIST_HEAD(&entrys);
150 write_lock_irqsave(&rdev->semaphore_drv.lock, irq_flags);
151 if (!list_empty(&rdev->semaphore_drv.free)) {
152 list_splice(&rdev->semaphore_drv.free, &entrys);
153 }
154 INIT_LIST_HEAD(&rdev->semaphore_drv.free);
155 write_unlock_irqrestore(&rdev->semaphore_drv.lock, irq_flags);
156
157 list_for_each_entry_safe(i, n, &entrys, list) {
158 radeon_bo_unref(&i->robj);
159 kfree(i);
160 }
161}
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 602fa3541c4..210d99f9193 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -42,7 +42,9 @@ void radeon_test_moves(struct radeon_device *rdev)
42 /* Number of tests = 42 /* Number of tests =
43 * (Total GTT - IB pool - writeback page - ring buffers) / test size 43 * (Total GTT - IB pool - writeback page - ring buffers) / test size
44 */ 44 */
45 n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024 - rdev->cp.ring_size; 45 n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024;
46 for (i = 0; i < RADEON_NUM_RINGS; ++i)
47 n -= rdev->ring[i].ring_size;
46 if (rdev->wb.wb_obj) 48 if (rdev->wb.wb_obj)
47 n -= RADEON_GPU_PAGE_SIZE; 49 n -= RADEON_GPU_PAGE_SIZE;
48 if (rdev->ih.ring_obj) 50 if (rdev->ih.ring_obj)
@@ -104,7 +106,7 @@ void radeon_test_moves(struct radeon_device *rdev)
104 106
105 radeon_bo_kunmap(gtt_obj[i]); 107 radeon_bo_kunmap(gtt_obj[i]);
106 108
107 r = radeon_fence_create(rdev, &fence); 109 r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
108 if (r) { 110 if (r) {
109 DRM_ERROR("Failed to create GTT->VRAM fence %d\n", i); 111 DRM_ERROR("Failed to create GTT->VRAM fence %d\n", i);
110 goto out_cleanup; 112 goto out_cleanup;
@@ -153,7 +155,7 @@ void radeon_test_moves(struct radeon_device *rdev)
153 155
154 radeon_bo_kunmap(vram_obj); 156 radeon_bo_kunmap(vram_obj);
155 157
156 r = radeon_fence_create(rdev, &fence); 158 r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
157 if (r) { 159 if (r) {
158 DRM_ERROR("Failed to create VRAM->GTT fence %d\n", i); 160 DRM_ERROR("Failed to create VRAM->GTT fence %d\n", i);
159 goto out_cleanup; 161 goto out_cleanup;
@@ -232,3 +234,262 @@ out_cleanup:
232 printk(KERN_WARNING "Error while testing BO move.\n"); 234 printk(KERN_WARNING "Error while testing BO move.\n");
233 } 235 }
234} 236}
237
238void radeon_test_ring_sync(struct radeon_device *rdev,
239 struct radeon_ring *ringA,
240 struct radeon_ring *ringB)
241{
242 struct radeon_fence *fence1 = NULL, *fence2 = NULL;
243 struct radeon_semaphore *semaphore = NULL;
244 int ridxA = radeon_ring_index(rdev, ringA);
245 int ridxB = radeon_ring_index(rdev, ringB);
246 int r;
247
248 r = radeon_fence_create(rdev, &fence1, ridxA);
249 if (r) {
250 DRM_ERROR("Failed to create sync fence 1\n");
251 goto out_cleanup;
252 }
253 r = radeon_fence_create(rdev, &fence2, ridxA);
254 if (r) {
255 DRM_ERROR("Failed to create sync fence 2\n");
256 goto out_cleanup;
257 }
258
259 r = radeon_semaphore_create(rdev, &semaphore);
260 if (r) {
261 DRM_ERROR("Failed to create semaphore\n");
262 goto out_cleanup;
263 }
264
265 r = radeon_ring_lock(rdev, ringA, 64);
266 if (r) {
267 DRM_ERROR("Failed to lock ring A %d\n", ridxA);
268 goto out_cleanup;
269 }
270 radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
271 radeon_fence_emit(rdev, fence1);
272 radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
273 radeon_fence_emit(rdev, fence2);
274 radeon_ring_unlock_commit(rdev, ringA);
275
276 mdelay(1000);
277
278 if (radeon_fence_signaled(fence1)) {
279 DRM_ERROR("Fence 1 signaled without waiting for semaphore.\n");
280 goto out_cleanup;
281 }
282
283 r = radeon_ring_lock(rdev, ringB, 64);
284 if (r) {
285 DRM_ERROR("Failed to lock ring B %p\n", ringB);
286 goto out_cleanup;
287 }
288 radeon_semaphore_emit_signal(rdev, ridxB, semaphore);
289 radeon_ring_unlock_commit(rdev, ringB);
290
291 r = radeon_fence_wait(fence1, false);
292 if (r) {
293 DRM_ERROR("Failed to wait for sync fence 1\n");
294 goto out_cleanup;
295 }
296
297 mdelay(1000);
298
299 if (radeon_fence_signaled(fence2)) {
300 DRM_ERROR("Fence 2 signaled without waiting for semaphore.\n");
301 goto out_cleanup;
302 }
303
304 r = radeon_ring_lock(rdev, ringB, 64);
305 if (r) {
306 DRM_ERROR("Failed to lock ring B %p\n", ringB);
307 goto out_cleanup;
308 }
309 radeon_semaphore_emit_signal(rdev, ridxB, semaphore);
310 radeon_ring_unlock_commit(rdev, ringB);
311
312 r = radeon_fence_wait(fence2, false);
313 if (r) {
314 DRM_ERROR("Failed to wait for sync fence 1\n");
315 goto out_cleanup;
316 }
317
318out_cleanup:
319 if (semaphore)
320 radeon_semaphore_free(rdev, semaphore);
321
322 if (fence1)
323 radeon_fence_unref(&fence1);
324
325 if (fence2)
326 radeon_fence_unref(&fence2);
327
328 if (r)
329 printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
330}
331
332void radeon_test_ring_sync2(struct radeon_device *rdev,
333 struct radeon_ring *ringA,
334 struct radeon_ring *ringB,
335 struct radeon_ring *ringC)
336{
337 struct radeon_fence *fenceA = NULL, *fenceB = NULL;
338 struct radeon_semaphore *semaphore = NULL;
339 int ridxA = radeon_ring_index(rdev, ringA);
340 int ridxB = radeon_ring_index(rdev, ringB);
341 int ridxC = radeon_ring_index(rdev, ringC);
342 bool sigA, sigB;
343 int i, r;
344
345 r = radeon_fence_create(rdev, &fenceA, ridxA);
346 if (r) {
347 DRM_ERROR("Failed to create sync fence 1\n");
348 goto out_cleanup;
349 }
350 r = radeon_fence_create(rdev, &fenceB, ridxB);
351 if (r) {
352 DRM_ERROR("Failed to create sync fence 2\n");
353 goto out_cleanup;
354 }
355
356 r = radeon_semaphore_create(rdev, &semaphore);
357 if (r) {
358 DRM_ERROR("Failed to create semaphore\n");
359 goto out_cleanup;
360 }
361
362 r = radeon_ring_lock(rdev, ringA, 64);
363 if (r) {
364 DRM_ERROR("Failed to lock ring A %d\n", ridxA);
365 goto out_cleanup;
366 }
367 radeon_semaphore_emit_wait(rdev, ridxA, semaphore);
368 radeon_fence_emit(rdev, fenceA);
369 radeon_ring_unlock_commit(rdev, ringA);
370
371 r = radeon_ring_lock(rdev, ringB, 64);
372 if (r) {
373 DRM_ERROR("Failed to lock ring B %d\n", ridxB);
374 goto out_cleanup;
375 }
376 radeon_semaphore_emit_wait(rdev, ridxB, semaphore);
377 radeon_fence_emit(rdev, fenceB);
378 radeon_ring_unlock_commit(rdev, ringB);
379
380 mdelay(1000);
381
382 if (radeon_fence_signaled(fenceA)) {
383 DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
384 goto out_cleanup;
385 }
386 if (radeon_fence_signaled(fenceB)) {
387 DRM_ERROR("Fence A signaled without waiting for semaphore.\n");
388 goto out_cleanup;
389 }
390
391 r = radeon_ring_lock(rdev, ringC, 64);
392 if (r) {
393 DRM_ERROR("Failed to lock ring B %p\n", ringC);
394 goto out_cleanup;
395 }
396 radeon_semaphore_emit_signal(rdev, ridxC, semaphore);
397 radeon_ring_unlock_commit(rdev, ringC);
398
399 for (i = 0; i < 30; ++i) {
400 mdelay(100);
401 sigA = radeon_fence_signaled(fenceA);
402 sigB = radeon_fence_signaled(fenceB);
403 if (sigA || sigB)
404 break;
405 }
406
407 if (!sigA && !sigB) {
408 DRM_ERROR("Neither fence A nor B has been signaled\n");
409 goto out_cleanup;
410 } else if (sigA && sigB) {
411 DRM_ERROR("Both fence A and B has been signaled\n");
412 goto out_cleanup;
413 }
414
415 DRM_INFO("Fence %c was first signaled\n", sigA ? 'A' : 'B');
416
417 r = radeon_ring_lock(rdev, ringC, 64);
418 if (r) {
419 DRM_ERROR("Failed to lock ring B %p\n", ringC);
420 goto out_cleanup;
421 }
422 radeon_semaphore_emit_signal(rdev, ridxC, semaphore);
423 radeon_ring_unlock_commit(rdev, ringC);
424
425 mdelay(1000);
426
427 r = radeon_fence_wait(fenceA, false);
428 if (r) {
429 DRM_ERROR("Failed to wait for sync fence A\n");
430 goto out_cleanup;
431 }
432 r = radeon_fence_wait(fenceB, false);
433 if (r) {
434 DRM_ERROR("Failed to wait for sync fence B\n");
435 goto out_cleanup;
436 }
437
438out_cleanup:
439 if (semaphore)
440 radeon_semaphore_free(rdev, semaphore);
441
442 if (fenceA)
443 radeon_fence_unref(&fenceA);
444
445 if (fenceB)
446 radeon_fence_unref(&fenceB);
447
448 if (r)
449 printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
450}
451
452void radeon_test_syncing(struct radeon_device *rdev)
453{
454 int i, j, k;
455
456 for (i = 1; i < RADEON_NUM_RINGS; ++i) {
457 struct radeon_ring *ringA = &rdev->ring[i];
458 if (!ringA->ready)
459 continue;
460
461 for (j = 0; j < i; ++j) {
462 struct radeon_ring *ringB = &rdev->ring[j];
463 if (!ringB->ready)
464 continue;
465
466 DRM_INFO("Testing syncing between rings %d and %d...\n", i, j);
467 radeon_test_ring_sync(rdev, ringA, ringB);
468
469 DRM_INFO("Testing syncing between rings %d and %d...\n", j, i);
470 radeon_test_ring_sync(rdev, ringB, ringA);
471
472 for (k = 0; k < j; ++k) {
473 struct radeon_ring *ringC = &rdev->ring[k];
474
475 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", i, j, k);
476 radeon_test_ring_sync2(rdev, ringA, ringB, ringC);
477
478 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", i, k, j);
479 radeon_test_ring_sync2(rdev, ringA, ringC, ringB);
480
481 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", j, i, k);
482 radeon_test_ring_sync2(rdev, ringB, ringA, ringC);
483
484 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", j, k, i);
485 radeon_test_ring_sync2(rdev, ringB, ringC, ringA);
486
487 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", k, i, j);
488 radeon_test_ring_sync2(rdev, ringC, ringA, ringB);
489
490 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", k, j, i);
491 radeon_test_ring_sync2(rdev, ringC, ringB, ringA);
492 }
493 }
494 }
495}
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index e111a381243..b0ebaf893ac 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -188,7 +188,7 @@ static void radeon_evict_flags(struct ttm_buffer_object *bo,
188 rbo = container_of(bo, struct radeon_bo, tbo); 188 rbo = container_of(bo, struct radeon_bo, tbo);
189 switch (bo->mem.mem_type) { 189 switch (bo->mem.mem_type) {
190 case TTM_PL_VRAM: 190 case TTM_PL_VRAM:
191 if (rbo->rdev->cp.ready == false) 191 if (rbo->rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready == false)
192 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 192 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU);
193 else 193 else
194 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 194 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
@@ -226,7 +226,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
226 int r; 226 int r;
227 227
228 rdev = radeon_get_rdev(bo->bdev); 228 rdev = radeon_get_rdev(bo->bdev);
229 r = radeon_fence_create(rdev, &fence); 229 r = radeon_fence_create(rdev, &fence, RADEON_RING_TYPE_GFX_INDEX);
230 if (unlikely(r)) { 230 if (unlikely(r)) {
231 return r; 231 return r;
232 } 232 }
@@ -255,7 +255,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
255 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 255 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
256 return -EINVAL; 256 return -EINVAL;
257 } 257 }
258 if (!rdev->cp.ready) { 258 if (!rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready) {
259 DRM_ERROR("Trying to move memory with CP turned off.\n"); 259 DRM_ERROR("Trying to move memory with CP turned off.\n");
260 return -EINVAL; 260 return -EINVAL;
261 } 261 }
@@ -380,7 +380,7 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
380 radeon_move_null(bo, new_mem); 380 radeon_move_null(bo, new_mem);
381 return 0; 381 return 0;
382 } 382 }
383 if (!rdev->cp.ready || rdev->asic->copy == NULL) { 383 if (!rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready || rdev->asic->copy == NULL) {
384 /* use memcpy */ 384 /* use memcpy */
385 goto memcpy; 385 goto memcpy;
386 } 386 }
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index 06b90c87f8f..b0ce84a20a6 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -410,6 +410,12 @@ static int rs400_startup(struct radeon_device *rdev)
410 if (r) 410 if (r)
411 return r; 411 return r;
412 412
413 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
414 if (r) {
415 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
416 return r;
417 }
418
413 /* Enable IRQ */ 419 /* Enable IRQ */
414 r100_irq_set(rdev); 420 r100_irq_set(rdev);
415 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 421 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -419,11 +425,18 @@ static int rs400_startup(struct radeon_device *rdev)
419 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 425 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
420 return r; 426 return r;
421 } 427 }
422 r = r100_ib_init(rdev); 428
429 r = radeon_ib_pool_start(rdev);
430 if (r)
431 return r;
432
433 r = r100_ib_test(rdev);
423 if (r) { 434 if (r) {
424 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 435 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
436 rdev->accel_working = false;
425 return r; 437 return r;
426 } 438 }
439
427 return 0; 440 return 0;
428} 441}
429 442
@@ -447,11 +460,14 @@ int rs400_resume(struct radeon_device *rdev)
447 r300_clock_startup(rdev); 460 r300_clock_startup(rdev);
448 /* Initialize surface registers */ 461 /* Initialize surface registers */
449 radeon_surface_init(rdev); 462 radeon_surface_init(rdev);
463
464 rdev->accel_working = true;
450 return rs400_startup(rdev); 465 return rs400_startup(rdev);
451} 466}
452 467
453int rs400_suspend(struct radeon_device *rdev) 468int rs400_suspend(struct radeon_device *rdev)
454{ 469{
470 radeon_ib_pool_suspend(rdev);
455 r100_cp_disable(rdev); 471 r100_cp_disable(rdev);
456 radeon_wb_disable(rdev); 472 radeon_wb_disable(rdev);
457 r100_irq_disable(rdev); 473 r100_irq_disable(rdev);
@@ -530,7 +546,14 @@ int rs400_init(struct radeon_device *rdev)
530 if (r) 546 if (r)
531 return r; 547 return r;
532 r300_set_reg_safe(rdev); 548 r300_set_reg_safe(rdev);
549
550 r = radeon_ib_pool_init(rdev);
533 rdev->accel_working = true; 551 rdev->accel_working = true;
552 if (r) {
553 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
554 rdev->accel_working = false;
555 }
556
534 r = rs400_startup(rdev); 557 r = rs400_startup(rdev);
535 if (r) { 558 if (r) {
536 /* Somethings want wront with the accel init stop accel */ 559 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index b1053d64042..ca6d5b6eaaa 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -549,7 +549,7 @@ int rs600_irq_set(struct radeon_device *rdev)
549 WREG32(R_000040_GEN_INT_CNTL, 0); 549 WREG32(R_000040_GEN_INT_CNTL, 0);
550 return -EINVAL; 550 return -EINVAL;
551 } 551 }
552 if (rdev->irq.sw_int) { 552 if (rdev->irq.sw_int[RADEON_RING_TYPE_GFX_INDEX]) {
553 tmp |= S_000040_SW_INT_EN(1); 553 tmp |= S_000040_SW_INT_EN(1);
554 } 554 }
555 if (rdev->irq.gui_idle) { 555 if (rdev->irq.gui_idle) {
@@ -642,7 +642,7 @@ int rs600_irq_process(struct radeon_device *rdev)
642 while (status || rdev->irq.stat_regs.r500.disp_int) { 642 while (status || rdev->irq.stat_regs.r500.disp_int) {
643 /* SW interrupt */ 643 /* SW interrupt */
644 if (G_000044_SW_INT(status)) { 644 if (G_000044_SW_INT(status)) {
645 radeon_fence_process(rdev); 645 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
646 } 646 }
647 /* GUI idle */ 647 /* GUI idle */
648 if (G_000040_GUI_IDLE(status)) { 648 if (G_000040_GUI_IDLE(status)) {
@@ -849,6 +849,12 @@ static int rs600_startup(struct radeon_device *rdev)
849 if (r) 849 if (r)
850 return r; 850 return r;
851 851
852 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
853 if (r) {
854 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
855 return r;
856 }
857
852 /* Enable IRQ */ 858 /* Enable IRQ */
853 rs600_irq_set(rdev); 859 rs600_irq_set(rdev);
854 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 860 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -858,15 +864,21 @@ static int rs600_startup(struct radeon_device *rdev)
858 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 864 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
859 return r; 865 return r;
860 } 866 }
861 r = r100_ib_init(rdev); 867
868 r = r600_audio_init(rdev);
862 if (r) { 869 if (r) {
863 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 870 dev_err(rdev->dev, "failed initializing audio\n");
864 return r; 871 return r;
865 } 872 }
866 873
867 r = r600_audio_init(rdev); 874 r = radeon_ib_pool_start(rdev);
875 if (r)
876 return r;
877
878 r = r100_ib_test(rdev);
868 if (r) { 879 if (r) {
869 dev_err(rdev->dev, "failed initializing audio\n"); 880 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
881 rdev->accel_working = false;
870 return r; 882 return r;
871 } 883 }
872 884
@@ -891,11 +903,14 @@ int rs600_resume(struct radeon_device *rdev)
891 rv515_clock_startup(rdev); 903 rv515_clock_startup(rdev);
892 /* Initialize surface registers */ 904 /* Initialize surface registers */
893 radeon_surface_init(rdev); 905 radeon_surface_init(rdev);
906
907 rdev->accel_working = true;
894 return rs600_startup(rdev); 908 return rs600_startup(rdev);
895} 909}
896 910
897int rs600_suspend(struct radeon_device *rdev) 911int rs600_suspend(struct radeon_device *rdev)
898{ 912{
913 radeon_ib_pool_suspend(rdev);
899 r600_audio_fini(rdev); 914 r600_audio_fini(rdev);
900 r100_cp_disable(rdev); 915 r100_cp_disable(rdev);
901 radeon_wb_disable(rdev); 916 radeon_wb_disable(rdev);
@@ -976,7 +991,14 @@ int rs600_init(struct radeon_device *rdev)
976 if (r) 991 if (r)
977 return r; 992 return r;
978 rs600_set_safe_registers(rdev); 993 rs600_set_safe_registers(rdev);
994
995 r = radeon_ib_pool_init(rdev);
979 rdev->accel_working = true; 996 rdev->accel_working = true;
997 if (r) {
998 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
999 rdev->accel_working = false;
1000 }
1001
980 r = rs600_startup(rdev); 1002 r = rs600_startup(rdev);
981 if (r) { 1003 if (r) {
982 /* Somethings want wront with the accel init stop accel */ 1004 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index a9049ed1a51..4f24a0fa8c8 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -621,6 +621,12 @@ static int rs690_startup(struct radeon_device *rdev)
621 if (r) 621 if (r)
622 return r; 622 return r;
623 623
624 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
625 if (r) {
626 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
627 return r;
628 }
629
624 /* Enable IRQ */ 630 /* Enable IRQ */
625 rs600_irq_set(rdev); 631 rs600_irq_set(rdev);
626 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 632 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -630,15 +636,21 @@ static int rs690_startup(struct radeon_device *rdev)
630 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 636 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
631 return r; 637 return r;
632 } 638 }
633 r = r100_ib_init(rdev); 639
640 r = r600_audio_init(rdev);
634 if (r) { 641 if (r) {
635 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 642 dev_err(rdev->dev, "failed initializing audio\n");
636 return r; 643 return r;
637 } 644 }
638 645
639 r = r600_audio_init(rdev); 646 r = radeon_ib_pool_start(rdev);
647 if (r)
648 return r;
649
650 r = r100_ib_test(rdev);
640 if (r) { 651 if (r) {
641 dev_err(rdev->dev, "failed initializing audio\n"); 652 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
653 rdev->accel_working = false;
642 return r; 654 return r;
643 } 655 }
644 656
@@ -663,11 +675,14 @@ int rs690_resume(struct radeon_device *rdev)
663 rv515_clock_startup(rdev); 675 rv515_clock_startup(rdev);
664 /* Initialize surface registers */ 676 /* Initialize surface registers */
665 radeon_surface_init(rdev); 677 radeon_surface_init(rdev);
678
679 rdev->accel_working = true;
666 return rs690_startup(rdev); 680 return rs690_startup(rdev);
667} 681}
668 682
669int rs690_suspend(struct radeon_device *rdev) 683int rs690_suspend(struct radeon_device *rdev)
670{ 684{
685 radeon_ib_pool_suspend(rdev);
671 r600_audio_fini(rdev); 686 r600_audio_fini(rdev);
672 r100_cp_disable(rdev); 687 r100_cp_disable(rdev);
673 radeon_wb_disable(rdev); 688 radeon_wb_disable(rdev);
@@ -749,7 +764,14 @@ int rs690_init(struct radeon_device *rdev)
749 if (r) 764 if (r)
750 return r; 765 return r;
751 rs600_set_safe_registers(rdev); 766 rs600_set_safe_registers(rdev);
767
768 r = radeon_ib_pool_init(rdev);
752 rdev->accel_working = true; 769 rdev->accel_working = true;
770 if (r) {
771 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
772 rdev->accel_working = false;
773 }
774
753 r = rs690_startup(rdev); 775 r = rs690_startup(rdev);
754 if (r) { 776 if (r) {
755 /* Somethings want wront with the accel init stop accel */ 777 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 6613ee9ecca..880637fd194 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -55,44 +55,45 @@ void rv515_debugfs(struct radeon_device *rdev)
55 55
56void rv515_ring_start(struct radeon_device *rdev) 56void rv515_ring_start(struct radeon_device *rdev)
57{ 57{
58 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
58 int r; 59 int r;
59 60
60 r = radeon_ring_lock(rdev, 64); 61 r = radeon_ring_lock(rdev, ring, 64);
61 if (r) { 62 if (r) {
62 return; 63 return;
63 } 64 }
64 radeon_ring_write(rdev, PACKET0(ISYNC_CNTL, 0)); 65 radeon_ring_write(ring, PACKET0(ISYNC_CNTL, 0));
65 radeon_ring_write(rdev, 66 radeon_ring_write(ring,
66 ISYNC_ANY2D_IDLE3D | 67 ISYNC_ANY2D_IDLE3D |
67 ISYNC_ANY3D_IDLE2D | 68 ISYNC_ANY3D_IDLE2D |
68 ISYNC_WAIT_IDLEGUI | 69 ISYNC_WAIT_IDLEGUI |
69 ISYNC_CPSCRATCH_IDLEGUI); 70 ISYNC_CPSCRATCH_IDLEGUI);
70 radeon_ring_write(rdev, PACKET0(WAIT_UNTIL, 0)); 71 radeon_ring_write(ring, PACKET0(WAIT_UNTIL, 0));
71 radeon_ring_write(rdev, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN); 72 radeon_ring_write(ring, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN);
72 radeon_ring_write(rdev, PACKET0(R300_DST_PIPE_CONFIG, 0)); 73 radeon_ring_write(ring, PACKET0(R300_DST_PIPE_CONFIG, 0));
73 radeon_ring_write(rdev, R300_PIPE_AUTO_CONFIG); 74 radeon_ring_write(ring, R300_PIPE_AUTO_CONFIG);
74 radeon_ring_write(rdev, PACKET0(GB_SELECT, 0)); 75 radeon_ring_write(ring, PACKET0(GB_SELECT, 0));
75 radeon_ring_write(rdev, 0); 76 radeon_ring_write(ring, 0);
76 radeon_ring_write(rdev, PACKET0(GB_ENABLE, 0)); 77 radeon_ring_write(ring, PACKET0(GB_ENABLE, 0));
77 radeon_ring_write(rdev, 0); 78 radeon_ring_write(ring, 0);
78 radeon_ring_write(rdev, PACKET0(R500_SU_REG_DEST, 0)); 79 radeon_ring_write(ring, PACKET0(R500_SU_REG_DEST, 0));
79 radeon_ring_write(rdev, (1 << rdev->num_gb_pipes) - 1); 80 radeon_ring_write(ring, (1 << rdev->num_gb_pipes) - 1);
80 radeon_ring_write(rdev, PACKET0(VAP_INDEX_OFFSET, 0)); 81 radeon_ring_write(ring, PACKET0(VAP_INDEX_OFFSET, 0));
81 radeon_ring_write(rdev, 0); 82 radeon_ring_write(ring, 0);
82 radeon_ring_write(rdev, PACKET0(RB3D_DSTCACHE_CTLSTAT, 0)); 83 radeon_ring_write(ring, PACKET0(RB3D_DSTCACHE_CTLSTAT, 0));
83 radeon_ring_write(rdev, RB3D_DC_FLUSH | RB3D_DC_FREE); 84 radeon_ring_write(ring, RB3D_DC_FLUSH | RB3D_DC_FREE);
84 radeon_ring_write(rdev, PACKET0(ZB_ZCACHE_CTLSTAT, 0)); 85 radeon_ring_write(ring, PACKET0(ZB_ZCACHE_CTLSTAT, 0));
85 radeon_ring_write(rdev, ZC_FLUSH | ZC_FREE); 86 radeon_ring_write(ring, ZC_FLUSH | ZC_FREE);
86 radeon_ring_write(rdev, PACKET0(WAIT_UNTIL, 0)); 87 radeon_ring_write(ring, PACKET0(WAIT_UNTIL, 0));
87 radeon_ring_write(rdev, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN); 88 radeon_ring_write(ring, WAIT_2D_IDLECLEAN | WAIT_3D_IDLECLEAN);
88 radeon_ring_write(rdev, PACKET0(GB_AA_CONFIG, 0)); 89 radeon_ring_write(ring, PACKET0(GB_AA_CONFIG, 0));
89 radeon_ring_write(rdev, 0); 90 radeon_ring_write(ring, 0);
90 radeon_ring_write(rdev, PACKET0(RB3D_DSTCACHE_CTLSTAT, 0)); 91 radeon_ring_write(ring, PACKET0(RB3D_DSTCACHE_CTLSTAT, 0));
91 radeon_ring_write(rdev, RB3D_DC_FLUSH | RB3D_DC_FREE); 92 radeon_ring_write(ring, RB3D_DC_FLUSH | RB3D_DC_FREE);
92 radeon_ring_write(rdev, PACKET0(ZB_ZCACHE_CTLSTAT, 0)); 93 radeon_ring_write(ring, PACKET0(ZB_ZCACHE_CTLSTAT, 0));
93 radeon_ring_write(rdev, ZC_FLUSH | ZC_FREE); 94 radeon_ring_write(ring, ZC_FLUSH | ZC_FREE);
94 radeon_ring_write(rdev, PACKET0(GB_MSPOS0, 0)); 95 radeon_ring_write(ring, PACKET0(GB_MSPOS0, 0));
95 radeon_ring_write(rdev, 96 radeon_ring_write(ring,
96 ((6 << MS_X0_SHIFT) | 97 ((6 << MS_X0_SHIFT) |
97 (6 << MS_Y0_SHIFT) | 98 (6 << MS_Y0_SHIFT) |
98 (6 << MS_X1_SHIFT) | 99 (6 << MS_X1_SHIFT) |
@@ -101,8 +102,8 @@ void rv515_ring_start(struct radeon_device *rdev)
101 (6 << MS_Y2_SHIFT) | 102 (6 << MS_Y2_SHIFT) |
102 (6 << MSBD0_Y_SHIFT) | 103 (6 << MSBD0_Y_SHIFT) |
103 (6 << MSBD0_X_SHIFT))); 104 (6 << MSBD0_X_SHIFT)));
104 radeon_ring_write(rdev, PACKET0(GB_MSPOS1, 0)); 105 radeon_ring_write(ring, PACKET0(GB_MSPOS1, 0));
105 radeon_ring_write(rdev, 106 radeon_ring_write(ring,
106 ((6 << MS_X3_SHIFT) | 107 ((6 << MS_X3_SHIFT) |
107 (6 << MS_Y3_SHIFT) | 108 (6 << MS_Y3_SHIFT) |
108 (6 << MS_X4_SHIFT) | 109 (6 << MS_X4_SHIFT) |
@@ -110,15 +111,15 @@ void rv515_ring_start(struct radeon_device *rdev)
110 (6 << MS_X5_SHIFT) | 111 (6 << MS_X5_SHIFT) |
111 (6 << MS_Y5_SHIFT) | 112 (6 << MS_Y5_SHIFT) |
112 (6 << MSBD1_SHIFT))); 113 (6 << MSBD1_SHIFT)));
113 radeon_ring_write(rdev, PACKET0(GA_ENHANCE, 0)); 114 radeon_ring_write(ring, PACKET0(GA_ENHANCE, 0));
114 radeon_ring_write(rdev, GA_DEADLOCK_CNTL | GA_FASTSYNC_CNTL); 115 radeon_ring_write(ring, GA_DEADLOCK_CNTL | GA_FASTSYNC_CNTL);
115 radeon_ring_write(rdev, PACKET0(GA_POLY_MODE, 0)); 116 radeon_ring_write(ring, PACKET0(GA_POLY_MODE, 0));
116 radeon_ring_write(rdev, FRONT_PTYPE_TRIANGE | BACK_PTYPE_TRIANGE); 117 radeon_ring_write(ring, FRONT_PTYPE_TRIANGE | BACK_PTYPE_TRIANGE);
117 radeon_ring_write(rdev, PACKET0(GA_ROUND_MODE, 0)); 118 radeon_ring_write(ring, PACKET0(GA_ROUND_MODE, 0));
118 radeon_ring_write(rdev, GEOMETRY_ROUND_NEAREST | COLOR_ROUND_NEAREST); 119 radeon_ring_write(ring, GEOMETRY_ROUND_NEAREST | COLOR_ROUND_NEAREST);
119 radeon_ring_write(rdev, PACKET0(0x20C8, 0)); 120 radeon_ring_write(ring, PACKET0(0x20C8, 0));
120 radeon_ring_write(rdev, 0); 121 radeon_ring_write(ring, 0);
121 radeon_ring_unlock_commit(rdev); 122 radeon_ring_unlock_commit(rdev, ring);
122} 123}
123 124
124int rv515_mc_wait_for_idle(struct radeon_device *rdev) 125int rv515_mc_wait_for_idle(struct radeon_device *rdev)
@@ -392,6 +393,12 @@ static int rv515_startup(struct radeon_device *rdev)
392 if (r) 393 if (r)
393 return r; 394 return r;
394 395
396 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
397 if (r) {
398 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
399 return r;
400 }
401
395 /* Enable IRQ */ 402 /* Enable IRQ */
396 rs600_irq_set(rdev); 403 rs600_irq_set(rdev);
397 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL); 404 rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
@@ -401,9 +408,15 @@ static int rv515_startup(struct radeon_device *rdev)
401 dev_err(rdev->dev, "failed initializing CP (%d).\n", r); 408 dev_err(rdev->dev, "failed initializing CP (%d).\n", r);
402 return r; 409 return r;
403 } 410 }
404 r = r100_ib_init(rdev); 411
412 r = radeon_ib_pool_start(rdev);
413 if (r)
414 return r;
415
416 r = r100_ib_test(rdev);
405 if (r) { 417 if (r) {
406 dev_err(rdev->dev, "failed initializing IB (%d).\n", r); 418 dev_err(rdev->dev, "failed testing IB (%d).\n", r);
419 rdev->accel_working = false;
407 return r; 420 return r;
408 } 421 }
409 return 0; 422 return 0;
@@ -428,6 +441,8 @@ int rv515_resume(struct radeon_device *rdev)
428 rv515_clock_startup(rdev); 441 rv515_clock_startup(rdev);
429 /* Initialize surface registers */ 442 /* Initialize surface registers */
430 radeon_surface_init(rdev); 443 radeon_surface_init(rdev);
444
445 rdev->accel_working = true;
431 return rv515_startup(rdev); 446 return rv515_startup(rdev);
432} 447}
433 448
@@ -524,7 +539,14 @@ int rv515_init(struct radeon_device *rdev)
524 if (r) 539 if (r)
525 return r; 540 return r;
526 rv515_set_safe_registers(rdev); 541 rv515_set_safe_registers(rdev);
542
543 r = radeon_ib_pool_init(rdev);
527 rdev->accel_working = true; 544 rdev->accel_working = true;
545 if (r) {
546 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
547 rdev->accel_working = false;
548 }
549
528 r = rv515_startup(rdev); 550 r = rv515_startup(rdev);
529 if (r) { 551 if (r) {
530 /* Somethings want wront with the accel init stop accel */ 552 /* Somethings want wront with the accel init stop accel */
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 23ae1c60ab3..a1668b659dd 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -357,7 +357,7 @@ static int rv770_cp_load_microcode(struct radeon_device *rdev)
357void r700_cp_fini(struct radeon_device *rdev) 357void r700_cp_fini(struct radeon_device *rdev)
358{ 358{
359 r700_cp_stop(rdev); 359 r700_cp_stop(rdev);
360 radeon_ring_fini(rdev); 360 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
361} 361}
362 362
363/* 363/*
@@ -1043,6 +1043,7 @@ int rv770_mc_init(struct radeon_device *rdev)
1043 1043
1044static int rv770_startup(struct radeon_device *rdev) 1044static int rv770_startup(struct radeon_device *rdev)
1045{ 1045{
1046 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1046 int r; 1047 int r;
1047 1048
1048 /* enable pcie gen2 link */ 1049 /* enable pcie gen2 link */
@@ -1082,6 +1083,12 @@ static int rv770_startup(struct radeon_device *rdev)
1082 if (r) 1083 if (r)
1083 return r; 1084 return r;
1084 1085
1086 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1087 if (r) {
1088 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1089 return r;
1090 }
1091
1085 /* Enable IRQ */ 1092 /* Enable IRQ */
1086 r = r600_irq_init(rdev); 1093 r = r600_irq_init(rdev);
1087 if (r) { 1094 if (r) {
@@ -1091,7 +1098,9 @@ static int rv770_startup(struct radeon_device *rdev)
1091 } 1098 }
1092 r600_irq_set(rdev); 1099 r600_irq_set(rdev);
1093 1100
1094 r = radeon_ring_init(rdev, rdev->cp.ring_size); 1101 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1102 R600_CP_RB_RPTR, R600_CP_RB_WPTR,
1103 0, 0xfffff, RADEON_CP_PACKET2);
1095 if (r) 1104 if (r)
1096 return r; 1105 return r;
1097 r = rv770_cp_load_microcode(rdev); 1106 r = rv770_cp_load_microcode(rdev);
@@ -1101,6 +1110,17 @@ static int rv770_startup(struct radeon_device *rdev)
1101 if (r) 1110 if (r)
1102 return r; 1111 return r;
1103 1112
1113 r = radeon_ib_pool_start(rdev);
1114 if (r)
1115 return r;
1116
1117 r = r600_ib_test(rdev, RADEON_RING_TYPE_GFX_INDEX);
1118 if (r) {
1119 dev_err(rdev->dev, "IB test failed (%d).\n", r);
1120 rdev->accel_working = false;
1121 return r;
1122 }
1123
1104 return 0; 1124 return 0;
1105} 1125}
1106 1126
@@ -1115,18 +1135,13 @@ int rv770_resume(struct radeon_device *rdev)
1115 /* post card */ 1135 /* post card */
1116 atom_asic_init(rdev->mode_info.atom_context); 1136 atom_asic_init(rdev->mode_info.atom_context);
1117 1137
1138 rdev->accel_working = true;
1118 r = rv770_startup(rdev); 1139 r = rv770_startup(rdev);
1119 if (r) { 1140 if (r) {
1120 DRM_ERROR("r600 startup failed on resume\n"); 1141 DRM_ERROR("r600 startup failed on resume\n");
1121 return r; 1142 return r;
1122 } 1143 }
1123 1144
1124 r = r600_ib_test(rdev);
1125 if (r) {
1126 DRM_ERROR("radeon: failed testing IB (%d).\n", r);
1127 return r;
1128 }
1129
1130 r = r600_audio_init(rdev); 1145 r = r600_audio_init(rdev);
1131 if (r) { 1146 if (r) {
1132 dev_err(rdev->dev, "radeon: audio init failed\n"); 1147 dev_err(rdev->dev, "radeon: audio init failed\n");
@@ -1140,13 +1155,14 @@ int rv770_resume(struct radeon_device *rdev)
1140int rv770_suspend(struct radeon_device *rdev) 1155int rv770_suspend(struct radeon_device *rdev)
1141{ 1156{
1142 r600_audio_fini(rdev); 1157 r600_audio_fini(rdev);
1158 radeon_ib_pool_suspend(rdev);
1159 r600_blit_suspend(rdev);
1143 /* FIXME: we should wait for ring to be empty */ 1160 /* FIXME: we should wait for ring to be empty */
1144 r700_cp_stop(rdev); 1161 r700_cp_stop(rdev);
1145 rdev->cp.ready = false; 1162 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1146 r600_irq_suspend(rdev); 1163 r600_irq_suspend(rdev);
1147 radeon_wb_disable(rdev); 1164 radeon_wb_disable(rdev);
1148 rv770_pcie_gart_disable(rdev); 1165 rv770_pcie_gart_disable(rdev);
1149 r600_blit_suspend(rdev);
1150 1166
1151 return 0; 1167 return 0;
1152} 1168}
@@ -1215,8 +1231,8 @@ int rv770_init(struct radeon_device *rdev)
1215 if (r) 1231 if (r)
1216 return r; 1232 return r;
1217 1233
1218 rdev->cp.ring_obj = NULL; 1234 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1219 r600_ring_init(rdev, 1024 * 1024); 1235 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1220 1236
1221 rdev->ih.ring_obj = NULL; 1237 rdev->ih.ring_obj = NULL;
1222 r600_ih_ring_init(rdev, 64 * 1024); 1238 r600_ih_ring_init(rdev, 64 * 1024);
@@ -1225,30 +1241,24 @@ int rv770_init(struct radeon_device *rdev)
1225 if (r) 1241 if (r)
1226 return r; 1242 return r;
1227 1243
1244 r = radeon_ib_pool_init(rdev);
1228 rdev->accel_working = true; 1245 rdev->accel_working = true;
1246 if (r) {
1247 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1248 rdev->accel_working = false;
1249 }
1250
1229 r = rv770_startup(rdev); 1251 r = rv770_startup(rdev);
1230 if (r) { 1252 if (r) {
1231 dev_err(rdev->dev, "disabling GPU acceleration\n"); 1253 dev_err(rdev->dev, "disabling GPU acceleration\n");
1232 r700_cp_fini(rdev); 1254 r700_cp_fini(rdev);
1233 r600_irq_fini(rdev); 1255 r600_irq_fini(rdev);
1234 radeon_wb_fini(rdev); 1256 radeon_wb_fini(rdev);
1257 r100_ib_fini(rdev);
1235 radeon_irq_kms_fini(rdev); 1258 radeon_irq_kms_fini(rdev);
1236 rv770_pcie_gart_fini(rdev); 1259 rv770_pcie_gart_fini(rdev);
1237 rdev->accel_working = false; 1260 rdev->accel_working = false;
1238 } 1261 }
1239 if (rdev->accel_working) {
1240 r = radeon_ib_pool_init(rdev);
1241 if (r) {
1242 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1243 rdev->accel_working = false;
1244 } else {
1245 r = r600_ib_test(rdev);
1246 if (r) {
1247 dev_err(rdev->dev, "IB test failed (%d).\n", r);
1248 rdev->accel_working = false;
1249 }
1250 }
1251 }
1252 1262
1253 r = r600_audio_init(rdev); 1263 r = r600_audio_init(rdev);
1254 if (r) { 1264 if (r) {
@@ -1265,11 +1275,12 @@ void rv770_fini(struct radeon_device *rdev)
1265 r700_cp_fini(rdev); 1275 r700_cp_fini(rdev);
1266 r600_irq_fini(rdev); 1276 r600_irq_fini(rdev);
1267 radeon_wb_fini(rdev); 1277 radeon_wb_fini(rdev);
1268 radeon_ib_pool_fini(rdev); 1278 r100_ib_fini(rdev);
1269 radeon_irq_kms_fini(rdev); 1279 radeon_irq_kms_fini(rdev);
1270 rv770_pcie_gart_fini(rdev); 1280 rv770_pcie_gart_fini(rdev);
1271 r600_vram_scratch_fini(rdev); 1281 r600_vram_scratch_fini(rdev);
1272 radeon_gem_fini(rdev); 1282 radeon_gem_fini(rdev);
1283 radeon_semaphore_driver_fini(rdev);
1273 radeon_fence_driver_fini(rdev); 1284 radeon_fence_driver_fini(rdev);
1274 radeon_agp_fini(rdev); 1285 radeon_agp_fini(rdev);
1275 radeon_bo_fini(rdev); 1286 radeon_bo_fini(rdev);