From da26bae051b8c4a13c31f2275a4ec4b560c172e6 Mon Sep 17 00:00:00 2001 From: Rex Zhu Date: Thu, 21 Jan 2016 19:33:56 +0800 Subject: [PATCH 01/50] drm/amd/powerplay: refine the dmesg info. this do not mean driver error. Change-Id: If2080eb4b79fc6389280b7c75cb7998d77090739 Signed-off-by: Rex Zhu Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c | 3 ++- drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c index ef1daf1251c7..7fdaf8c3f7a9 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c @@ -726,8 +726,9 @@ static int cz_tf_update_sclk_limit(struct pp_hwmgr *hwmgr, cz_hwmgr->sclk_dpm.soft_max_clk = table->entries[table->count - 1].clk; clock = hwmgr->display_config.min_core_set_clock; +; if (clock == 0) - printk(KERN_ERR "[ powerplay ] min_core_set_clock not set\n"); + printk(KERN_INFO "[ powerplay ] min_core_set_clock not set\n"); if (cz_hwmgr->sclk_dpm.hard_min_clk != clock) { cz_hwmgr->sclk_dpm.hard_min_clk = clock; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c index 9deadabbc81c..72cfecc4f9f7 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c @@ -34,6 +34,11 @@ static int phm_run_table(struct pp_hwmgr *hwmgr, int result = 0; phm_table_function *function; + if (rt_table->function_list == NULL) { + printk(KERN_INFO "[ powerplay ] this function not implement!\n"); + return 0; + } + for (function = rt_table->function_list; NULL != *function; function++) { int tmp = (*function)(hwmgr, input, output, temp_storage, result); @@ -57,9 +62,9 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr, int result = 0; void *temp_storage = NULL; - if (hwmgr == NULL || rt_table == NULL || rt_table->function_list == NULL) { + if (hwmgr == NULL || rt_table == NULL) { printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n"); - return 0; /*temp return ture because some function not implement on some asic */ + return -EINVAL; } if (0 != rt_table->storage_size) { From 225ac139285a3ebde698694a4957a9de907b97ee Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Thu, 3 Mar 2016 11:28:24 -0500 Subject: [PATCH 02/50] drm/amdgpu/ci: drop some old thermal setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was leftover from a long time ago and is not longer needed since the thermal controller setup code was added. Additional mucking with the thermal interrupts can cause spurious thermal events which can lead to unnecessary state changes. Reviewed-by: Christian König Reviewed-by: Ken Wang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/ci_dpm.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c index 8b4731d4e10e..7226a418ae4b 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c @@ -5394,30 +5394,6 @@ static int ci_dpm_enable(struct amdgpu_device *adev) ci_update_current_ps(adev, boot_ps); - if (adev->irq.installed && - amdgpu_is_internal_thermal_sensor(adev->pm.int_thermal_type)) { -#if 0 - PPSMC_Result result; -#endif - ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN, - CISLANDS_TEMP_RANGE_MAX); - if (ret) { - DRM_ERROR("ci_thermal_set_temperature_range failed\n"); - return ret; - } - amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_LOW_TO_HIGH); - amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq, - AMDGPU_THERMAL_IRQ_HIGH_TO_LOW); - -#if 0 - result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableThermalInterrupt); - - if (result != PPSMC_Result_OK) - DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); -#endif - } - return 0; } From 12424e55281beb97a6fbfb1ba31a9d90c45e6102 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Thu, 3 Mar 2016 12:28:37 -0500 Subject: [PATCH 03/50] drm/amdgpu/ci: remove redundant pcie setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Looks like this got leftover by accident. Reviewed-by: Christian König Reviewed-by: Ken Wang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/ci_dpm.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c index 7226a418ae4b..9826022e9082 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c @@ -4380,26 +4380,6 @@ static int ci_dpm_force_performance_level(struct amdgpu_device *adev, } } } - if ((!pi->pcie_dpm_key_disabled) && - pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { - levels = 0; - tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask; - while (tmp >>= 1) - levels++; - if (levels) { - ret = ci_dpm_force_state_pcie(adev, level); - if (ret) - return ret; - for (i = 0; i < adev->usec_timeout; i++) { - tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) & - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >> - TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) { if ((!pi->sclk_dpm_key_disabled) && pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { From 4223cc3de2c4de880a945f265d9e24cbd01b2293 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Thu, 3 Mar 2016 12:27:46 -0500 Subject: [PATCH 04/50] drm/amdgpu/ci: sync up with dpm changes from radeon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Looks like radeon commit: d3052b8ce8a308d2086519fa5f7c4966257ea184 was missed. Reviewed-by: Christian König Reviewed-by: Ken Wang Reviewed-by: Rex Zhu Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/ci_dpm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c index 9826022e9082..061ba07d07d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c @@ -3016,7 +3016,6 @@ static int ci_populate_single_memory_level(struct amdgpu_device *adev, &memory_level->MinVddcPhases); memory_level->EnabledForThrottle = 1; - memory_level->EnabledForActivity = 1; memory_level->UpH = 0; memory_level->DownH = 100; memory_level->VoltageDownH = 0; @@ -3375,7 +3374,6 @@ static int ci_populate_single_graphic_level(struct amdgpu_device *adev, graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2); graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm); graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1); - graphic_level->EnabledForActivity = 1; return 0; } @@ -3406,6 +3404,7 @@ static int ci_populate_all_graphic_levels(struct amdgpu_device *adev) pi->smc_state_table.GraphicsLevel[i].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH; } + pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1; pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count; pi->dpm_level_enable_mask.sclk_dpm_enable_mask = @@ -3449,6 +3448,8 @@ static int ci_populate_all_memory_levels(struct amdgpu_device *adev) return ret; } + pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1; + if ((dpm_table->mclk_table.count >= 2) && ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) { pi->smc_state_table.MemoryLevel[1].MinVddc = From 0bf5df3b58e152ba2a50f9918f31a9c15b479b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20Wallm=C3=A9nius?= Date: Sat, 5 Mar 2016 06:59:51 +0100 Subject: [PATCH 05/50] drm/amdgpu: delete set-but-not-read member has_uvd from amdgpu_device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean up leftover from radeon code. Signed-off-by: Nils Wallménius Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 - drivers/gpu/drm/amd/amdgpu/cik.c | 2 -- drivers/gpu/drm/amd/amdgpu/vi.c | 4 ---- 3 files changed, 7 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 0c42a85ca5a5..cfd35b0fce06 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -2053,7 +2053,6 @@ struct amdgpu_device { struct amdgpu_sdma sdma; /* uvd */ - bool has_uvd; struct amdgpu_uvd uvd; /* vce */ diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 6b1f0539ce9d..5da14a3005b4 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -2025,8 +2025,6 @@ static int cik_common_early_init(void *handle) adev->asic_funcs = &cik_asic_funcs; - adev->has_uvd = true; - adev->rev_id = cik_get_rev_id(adev); adev->external_rev_id = 0xFF; switch (adev->asic_type) { diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 125003517544..9163f5968a2f 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -1071,26 +1071,22 @@ static int vi_common_early_init(void *handle) adev->external_rev_id = 0xFF; switch (adev->asic_type) { case CHIP_TOPAZ: - adev->has_uvd = false; adev->cg_flags = 0; adev->pg_flags = 0; adev->external_rev_id = 0x1; break; case CHIP_FIJI: - adev->has_uvd = true; adev->cg_flags = 0; adev->pg_flags = 0; adev->external_rev_id = adev->rev_id + 0x3c; break; case CHIP_TONGA: - adev->has_uvd = true; adev->cg_flags = 0; adev->pg_flags = 0; adev->external_rev_id = adev->rev_id + 0x14; break; case CHIP_CARRIZO: case CHIP_STONEY: - adev->has_uvd = true; adev->cg_flags = 0; /* Disable UVD pg */ adev->pg_flags = /* AMDGPU_PG_SUPPORT_UVD | */AMDGPU_PG_SUPPORT_VCE; From 777e3cbc791f131806d9bf24b3325637c7fc228d Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Thu, 21 Jan 2016 11:08:57 +0100 Subject: [PATCH 06/50] drm/radeon: Switch to drm_vblank_on/off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These should be functionally equivalent to the older per/post modeset functions, except that they block out drm_vblank_get right away. There's only the clock adjusting code (outside of pageflips) in readone which uses drm_vblank_get. But that code doesn't synchronize against concurrent modesets and instead handles any such races by waiting for the right vblank to arrive with a short timetout. The longer-term plan here is to switch all kms drivers to drm_vblank_on/off so that common code like pending event cleanup can be done there, while drm_vblank_pre/post_modeset will be purely drm internal for the old UMS ioctl. Note that with this patch Michel uncovered a bug in the dri3 implementation of the DDX (it does vblank waits when the pipe is off), which had to be fixed first. Cc: Michel Dänzer Acked-and-tested-by: Michel Dänzer Signed-off-by: Daniel Vetter Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/atombios_crtc.c | 4 ++-- drivers/gpu/drm/radeon/radeon_legacy_crtc.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c index 801dd60ac192..e187beca38f7 100644 --- a/drivers/gpu/drm/radeon/atombios_crtc.c +++ b/drivers/gpu/drm/radeon/atombios_crtc.c @@ -275,13 +275,13 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode) if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev)) atombios_enable_crtc_memreq(crtc, ATOM_ENABLE); atombios_blank_crtc(crtc, ATOM_DISABLE); - drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); + drm_vblank_on(dev, radeon_crtc->crtc_id); radeon_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id); + drm_vblank_off(dev, radeon_crtc->crtc_id); if (radeon_crtc->enabled) atombios_blank_crtc(crtc, ATOM_ENABLE); if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev)) diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c index 32b338ff436b..24152dfef199 100644 --- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c +++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c @@ -331,13 +331,13 @@ static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode) RADEON_CRTC_DISP_REQ_EN_B)); WREG32_P(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl, ~(mask | crtc_ext_cntl)); } - drm_vblank_post_modeset(dev, radeon_crtc->crtc_id); + drm_vblank_on(dev, radeon_crtc->crtc_id); radeon_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_pre_modeset(dev, radeon_crtc->crtc_id); + drm_vblank_off(dev, radeon_crtc->crtc_id); if (radeon_crtc->crtc_id) WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~(RADEON_CRTC2_EN | mask)); else { From b6bae58d8b6c21357198255cdb42a3929e7f7357 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Thu, 21 Jan 2016 11:08:58 +0100 Subject: [PATCH 07/50] drm/amdgpu: Switch to drm_vblank_on/off (v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Equivalent change to the radeon driver. Note that with radeon this caught a bug in the dri3 DDX implementation, which asked for vblank interrupts when the pipe is off. That bug needs to be fixed before we can merge this patch (if amdgpu is affected too). Michel discovered this one. v2: agd: switch dce8 as well. Acked-by: Michel Dänzer Cc: Michel Dänzer Signed-off-by: Daniel Vetter Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index e3ff809a0cae..b8cb411918ab 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -2700,13 +2700,13 @@ static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_on(dev, amdgpu_crtc->crtc_id); dce_v10_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_off(dev, amdgpu_crtc->crtc_id); if (amdgpu_crtc->enabled) { dce_v10_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 6b6c9b6879ae..6e8d74da183f 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -2691,13 +2691,13 @@ static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_on(dev, amdgpu_crtc->crtc_id); dce_v11_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_off(dev, amdgpu_crtc->crtc_id); if (amdgpu_crtc->enabled) { dce_v11_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 56bea36a6b18..b059853d93bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -2612,13 +2612,13 @@ static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_on(dev, amdgpu_crtc->crtc_id); dce_v8_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id); + drm_vblank_off(dev, amdgpu_crtc->crtc_id); if (amdgpu_crtc->enabled) { dce_v8_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); From 5c55db83b759b079ead93aa0767fc97048b1f899 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 2 Mar 2016 11:30:31 +0800 Subject: [PATCH 08/50] drm/amdgpu: wait engine idle before vm flush for sdma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 14 ++++++++++++++ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 15 +++++++++++++++ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 15 +++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index e4e4b2ac77b7..6cf45b0ff91b 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -829,6 +829,20 @@ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, { u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ + uint32_t seq = ring->fence_drv.sync_seq; + uint64_t addr = ring->fence_drv.gpu_addr; + + /* wait for idle */ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, + SDMA_POLL_REG_MEM_EXTRA_OP(0) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */ + SDMA_POLL_REG_MEM_EXTRA_M)); + amdgpu_ring_write(ring, addr & 0xfffffffc); + amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); + amdgpu_ring_write(ring, seq); /* reference */ + amdgpu_ring_write(ring, 0xfffffff); /* mask */ + amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); if (vm_id < 8) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index dddb8d6a81f3..0e246ff6734c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -885,6 +885,21 @@ static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr) { + uint32_t seq = ring->fence_drv.sync_seq; + uint64_t addr = ring->fence_drv.gpu_addr; + + /* wait for idle */ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | + SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | + SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */ + SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1)); + amdgpu_ring_write(ring, addr & 0xfffffffc); + amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); + amdgpu_ring_write(ring, seq); /* reference */ + amdgpu_ring_write(ring, 0xfffffff); /* mask */ + amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | + SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); if (vm_id < 8) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 19e02f7a06f3..67441f8844bb 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -1035,6 +1035,21 @@ static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr) { + uint32_t seq = ring->fence_drv.sync_seq; + uint64_t addr = ring->fence_drv.gpu_addr; + + /* wait for idle */ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) | + SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | + SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */ + SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1)); + amdgpu_ring_write(ring, addr & 0xfffffffc); + amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); + amdgpu_ring_write(ring, seq); /* reference */ + amdgpu_ring_write(ring, 0xfffffff); /* mask */ + amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | + SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); if (vm_id < 8) { From 11afbde85eeba1ccb0a459c49444aaf9032e9755 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 11:38:48 +0800 Subject: [PATCH 09/50] drm/amdgpu: add hdp_invalidate function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's called after emitting ibs. Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index cfd35b0fce06..497ca587cc35 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -329,6 +329,7 @@ struct amdgpu_ring_funcs { void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr); void (*emit_hdp_flush)(struct amdgpu_ring *ring); + void (*emit_hdp_invalidate)(struct amdgpu_ring *ring); void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid, uint32_t gds_base, uint32_t gds_size, uint32_t gws_base, uint32_t gws_size, @@ -2230,6 +2231,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as)) #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r)) +#define amdgpu_ring_emit_hdp_invalidate(r) (r)->funcs->emit_hdp_invalidate((r)) #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib))) #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev)) #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index db14a7bbb8f4..4e978e7aa1b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -178,6 +178,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, ring->current_ctx = ctx; } + if (vm) { + if (ring->funcs->emit_hdp_invalidate) + amdgpu_ring_emit_hdp_invalidate(ring); + } + r = amdgpu_fence_emit(ring, owner, &ib->fence); if (r) { dev_err(adev->dev, "failed to emit fence (%d)\n", r); From 0955860b87183d4bf506eb861760ea178de027bf Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 11:46:40 +0800 Subject: [PATCH 10/50] drm/amdgpu: add hdp invalidation for gfx7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 8fb7ebf3be3e..1b713009c5d2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -1924,6 +1924,25 @@ static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x20); /* poll interval */ } +/** + * gfx_v7_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp + * + * @adev: amdgpu_device pointer + * @ridx: amdgpu ring index + * + * Emits an hdp invalidate on the cp. + */ +static void gfx_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | + WRITE_DATA_DST_SEL(0) | + WR_CONFIRM)); + amdgpu_ring_write(ring, mmHDP_DEBUG0); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, 1); +} + /** * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring * @@ -5132,6 +5151,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = { .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = gfx_v7_0_ring_emit_hdp_invalidate, .test_ring = gfx_v7_0_ring_test_ring, .test_ib = gfx_v7_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, @@ -5148,6 +5168,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = { .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = gfx_v7_0_ring_emit_hdp_invalidate, .test_ring = gfx_v7_0_ring_test_ring, .test_ib = gfx_v7_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, From d35db5617a75b3c77a61008e8fbf249ab8c24e5e Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 11:55:13 +0800 Subject: [PATCH 11/50] drm/amdgpu: add hdp invalidation for gfx8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index e37378fe1edc..aeb35f504999 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4589,6 +4589,18 @@ static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x20); /* poll interval */ } +static void gfx_v8_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | + WRITE_DATA_DST_SEL(0) | + WR_CONFIRM)); + amdgpu_ring_write(ring, mmHDP_DEBUG0); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, 1); + +} + static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, struct amdgpu_ib *ib) { @@ -5030,6 +5042,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate, .test_ring = gfx_v8_0_ring_test_ring, .test_ib = gfx_v8_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, @@ -5046,6 +5059,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate, .test_ring = gfx_v8_0_ring_test_ring, .test_ib = gfx_v8_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, From 498dd97ddbc47fdb8a5e3bf4df8a77893b8b07a2 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 12:05:44 +0800 Subject: [PATCH 12/50] drm/amdgpu: add hdp invalidation for cik sdma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 6cf45b0ff91b..c4101ebd2b3f 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -261,6 +261,13 @@ static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring) amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */ } +static void cik_sdma_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); + amdgpu_ring_write(ring, mmHDP_DEBUG0); + amdgpu_ring_write(ring, 1); +} + /** * cik_sdma_ring_emit_fence - emit a fence on the DMA ring * @@ -1286,6 +1293,7 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = { .emit_fence = cik_sdma_ring_emit_fence, .emit_vm_flush = cik_sdma_ring_emit_vm_flush, .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush, + .emit_hdp_invalidate = cik_sdma_ring_emit_hdp_invalidate, .test_ring = cik_sdma_ring_test_ring, .test_ib = cik_sdma_ring_test_ib, .insert_nop = cik_sdma_ring_insert_nop, From 6ad550c35d8c5327a548bc59bba406ef2db51f1b Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 12:06:34 +0800 Subject: [PATCH 13/50] drm/amdgpu: add hdp invalidation for sdma v2_4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 0e246ff6734c..134d5d0bb77f 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -300,6 +300,13 @@ static void sdma_v2_4_ring_emit_hdp_flush(struct amdgpu_ring *ring) SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ } +static void sdma_v2_4_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | + SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); + amdgpu_ring_write(ring, mmHDP_DEBUG0); + amdgpu_ring_write(ring, 1); +} /** * sdma_v2_4_ring_emit_fence - emit a fence on the DMA ring * @@ -1291,6 +1298,7 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = { .emit_fence = sdma_v2_4_ring_emit_fence, .emit_vm_flush = sdma_v2_4_ring_emit_vm_flush, .emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush, + .emit_hdp_invalidate = sdma_v2_4_ring_emit_hdp_invalidate, .test_ring = sdma_v2_4_ring_test_ring, .test_ib = sdma_v2_4_ring_test_ib, .insert_nop = sdma_v2_4_ring_insert_nop, From cc958e67aa47372e8e1e875c653cd9328f215cb4 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 12:06:45 +0800 Subject: [PATCH 14/50] drm/amdgpu: add hdp invalidation for sdma v3_0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 67441f8844bb..ca12d6faa8c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -410,6 +410,14 @@ static void sdma_v3_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */ } +static void sdma_v3_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | + SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); + amdgpu_ring_write(ring, mmHDP_DEBUG0); + amdgpu_ring_write(ring, 1); +} + /** * sdma_v3_0_ring_emit_fence - emit a fence on the DMA ring * @@ -1558,6 +1566,7 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = { .emit_fence = sdma_v3_0_ring_emit_fence, .emit_vm_flush = sdma_v3_0_ring_emit_vm_flush, .emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = sdma_v3_0_ring_emit_hdp_invalidate, .test_ring = sdma_v3_0_ring_test_ring, .test_ib = sdma_v3_0_ring_test_ib, .insert_nop = sdma_v3_0_ring_insert_nop, From 13459bd0fb4d178bcbd20cff155f1849313cd3bf Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 3 Mar 2016 14:47:54 +0800 Subject: [PATCH 15/50] drm/amdgpu: Clear HDP_MISC_CNTL.HDP_FLUSH_INVALIDATE_CACHE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that we can invalidate and flush the HDP independently Signed-off-by: Chunming Zhou Reviewed-by: Alex Deucher Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 68ee66b38e5c..45cde4351027 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -339,7 +339,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev) WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); tmp = RREG32(mmHDP_MISC_CNTL); - tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); + tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0); WREG32(mmHDP_MISC_CNTL, tmp); tmp = RREG32(mmHDP_HOST_PATH_CNTL); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 757803ae7c4a..29bd7b57dc91 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -386,7 +386,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev) WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK); tmp = RREG32(mmHDP_MISC_CNTL); - tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); + tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0); WREG32(mmHDP_MISC_CNTL, tmp); tmp = RREG32(mmHDP_HOST_PATH_CNTL); From a8bd1bec7c68dd5082012bcdd1d765e65b3ee05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 3 Mar 2016 10:50:01 +0100 Subject: [PATCH 16/50] drm/amdgpu: sync to the active user on reusing a VMID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König Reviewed-by: Chunming Zhou Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index d9dc8bea5e98..364db7c45c67 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -188,6 +188,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, if (!is_later && owner == (long)id && pd_addr == id->pd_gpu_addr) { + r = amdgpu_sync_fence(ring->adev, sync, + id->mgr_id->active); + if (r) { + mutex_unlock(&adev->vm_manager.lock); + return r; + } + fence_put(id->mgr_id->active); id->mgr_id->active = fence_get(fence); From 211dff5518d938d0c11205af3113d2c7a292dd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 22 Feb 2016 15:40:59 +0100 Subject: [PATCH 17/50] drm/amdgpu: group userptr in the BO list v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need them together with the next patch. v2: Don't take bo reference twice Signed-off-by: Christian König Signed-off-by: Felix Kuehling Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c | 24 +++++++++++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 3 ++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 497ca587cc35..11a9f5899995 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1048,7 +1048,7 @@ struct amdgpu_bo_list { struct amdgpu_bo *gds_obj; struct amdgpu_bo *gws_obj; struct amdgpu_bo *oa_obj; - bool has_userptr; + unsigned first_userptr; unsigned num_entries; struct amdgpu_bo_list_entry *array; }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 4792f9d0b7d4..9763e52886fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -91,7 +91,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo; struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo; - bool has_userptr = false; + unsigned last_entry = 0, first_userptr = num_entries; unsigned i; int r; @@ -101,8 +101,9 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); for (i = 0; i < num_entries; ++i) { - struct amdgpu_bo_list_entry *entry = &array[i]; + struct amdgpu_bo_list_entry *entry; struct drm_gem_object *gobj; + struct amdgpu_bo *bo; struct mm_struct *usermm; gobj = drm_gem_object_lookup(adev->ddev, filp, info[i].bo_handle); @@ -111,19 +112,24 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, goto error_free; } - entry->robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); + bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); drm_gem_object_unreference_unlocked(gobj); - entry->priority = min(info[i].bo_priority, - AMDGPU_BO_LIST_MAX_PRIORITY); - usermm = amdgpu_ttm_tt_get_usermm(entry->robj->tbo.ttm); + + usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); if (usermm) { if (usermm != current->mm) { - amdgpu_bo_unref(&entry->robj); + amdgpu_bo_unref(&bo); r = -EPERM; goto error_free; } - has_userptr = true; + entry = &array[--first_userptr]; + } else { + entry = &array[last_entry++]; } + + entry->robj = bo; + entry->priority = min(info[i].bo_priority, + AMDGPU_BO_LIST_MAX_PRIORITY); entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; @@ -145,7 +151,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, list->gds_obj = gds_obj; list->gws_obj = gws_obj; list->oa_obj = oa_obj; - list->has_userptr = has_userptr; + list->first_userptr = first_userptr; list->array = array; list->num_entries = num_entries; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 52c3eb96b199..7833dfb1ff6e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -350,7 +350,8 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); if (p->bo_list) { - need_mmap_lock = p->bo_list->has_userptr; + need_mmap_lock = p->bo_list->first_userptr != + p->bo_list->num_entries; amdgpu_bo_list_get_list(p->bo_list, &p->validated); } From 637dd3b5ca9e0d38b046f04e97a1d2f7523e7e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 3 Mar 2016 14:24:57 +0100 Subject: [PATCH 18/50] drm/amdgpu: prevent get_user_pages recursion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remember the tasks which are inside get_user_pages() and ignore MMU callbacks from there. Signed-off-by: Christian König Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 48 +++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index e52fc641edfb..53e627970798 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -494,13 +494,20 @@ static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re /* * TTM backend functions. */ +struct amdgpu_ttm_gup_task_list { + struct list_head list; + struct task_struct *task; +}; + struct amdgpu_ttm_tt { - struct ttm_dma_tt ttm; - struct amdgpu_device *adev; - u64 offset; - uint64_t userptr; - struct mm_struct *usermm; - uint32_t userflags; + struct ttm_dma_tt ttm; + struct amdgpu_device *adev; + u64 offset; + uint64_t userptr; + struct mm_struct *usermm; + uint32_t userflags; + spinlock_t guptasklock; + struct list_head guptasks; }; /* prepare the sg table with the user pages */ @@ -530,9 +537,20 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) unsigned num_pages = ttm->num_pages - pinned; uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; struct page **pages = ttm->pages + pinned; + struct amdgpu_ttm_gup_task_list guptask; + + guptask.task = current; + spin_lock(>t->guptasklock); + list_add(&guptask.list, >t->guptasks); + spin_unlock(>t->guptasklock); r = get_user_pages(current, current->mm, userptr, num_pages, write, 0, pages, NULL); + + spin_lock(>t->guptasklock); + list_del(&guptask.list); + spin_unlock(>t->guptasklock); + if (r < 0) goto release_pages; @@ -783,6 +801,9 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, gtt->userptr = addr; gtt->usermm = current->mm; gtt->userflags = flags; + spin_lock_init(>t->guptasklock); + INIT_LIST_HEAD(>t->guptasks); + return 0; } @@ -800,18 +821,25 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, unsigned long end) { struct amdgpu_ttm_tt *gtt = (void *)ttm; + struct amdgpu_ttm_gup_task_list *entry; unsigned long size; - if (gtt == NULL) - return false; - - if (gtt->ttm.ttm.state != tt_bound || !gtt->userptr) + if (gtt == NULL || !gtt->userptr) return false; size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE; if (gtt->userptr > end || gtt->userptr + size <= start) return false; + spin_lock(>t->guptasklock); + list_for_each_entry(entry, >t->guptasks, list) { + if (entry->task == current) { + spin_unlock(>t->guptasklock); + return false; + } + } + spin_unlock(>t->guptasklock); + return true; } From bcc634f4a8f10a851dd3d8429ea28998351ca843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 16 Feb 2016 16:23:02 +0100 Subject: [PATCH 19/50] drm/amdgpu: cleanup the sync code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need for two functions doing the same, remove one and add comments what those functions actually do. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 48 +++++++++++++++--------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index c15be00de904..87690cc57206 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -50,6 +50,14 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) sync->last_vm_update = NULL; } +/** + * amdgpu_sync_same_dev - test if fence belong to us + * + * @adev: amdgpu device to use for the test + * @f: fence to test + * + * Test if the fence was issued by us. + */ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) { struct amdgpu_fence *a_fence = to_amdgpu_fence(f); @@ -68,17 +76,33 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) return false; } -static bool amdgpu_sync_test_owner(struct fence *f, void *owner) +/** + * amdgpu_sync_get_owner - extract the owner of a fence + * + * @fence: fence get the owner from + * + * Extract who originally created the fence. + */ +static void *amdgpu_sync_get_owner(struct fence *f) { struct amdgpu_fence *a_fence = to_amdgpu_fence(f); struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + if (s_fence) - return s_fence->owner == owner; - if (a_fence) - return a_fence->owner == owner; - return false; + return s_fence->owner; + else if (a_fence) + return a_fence->owner; + return AMDGPU_FENCE_OWNER_UNDEFINED; } +/** + * amdgpu_sync_keep_later - Keep the later fence + * + * @keep: existing fence to test + * @fence: new fence + * + * Either keep the existing fence or the new one, depending which one is later. + */ static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence) { if (*keep && fence_is_later(*keep, fence)) @@ -104,7 +128,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, return 0; if (amdgpu_sync_same_dev(adev, f) && - amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) + amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM) amdgpu_sync_keep_later(&sync->last_vm_update, f); hash_for_each_possible(sync->fences, e, node, f->context) { @@ -124,18 +148,6 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, return 0; } -static void *amdgpu_sync_get_owner(struct fence *f) -{ - struct amdgpu_fence *a_fence = to_amdgpu_fence(f); - struct amd_sched_fence *s_fence = to_amd_sched_fence(f); - - if (s_fence) - return s_fence->owner; - else if (a_fence) - return a_fence->owner; - return AMDGPU_FENCE_OWNER_UNDEFINED; -} - /** * amdgpu_sync_resv - sync to a reservation object * From 336d1f5efe93db3d997a6d105760dd613d7ecdce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 16 Feb 2016 10:57:10 +0100 Subject: [PATCH 20/50] drm/amdgpu: remove HW fence owner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not used any more since we now always use the sheduler. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 9 ++------- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 6 +----- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 6 ++---- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 8 +------- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 3 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 7 ++----- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 3 +-- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 3 +-- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 6 ++---- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 3 +-- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 3 +-- 12 files changed, 16 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 11a9f5899995..fab6ddb26b5b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -409,9 +409,6 @@ struct amdgpu_fence { struct amdgpu_ring *ring; uint64_t seq; - /* filp or special value for fence creator */ - void *owner; - wait_queue_t fence_wake; }; @@ -432,8 +429,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, unsigned irq_type); void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); -int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, - struct amdgpu_fence **fence); +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence **fence); void amdgpu_fence_process(struct amdgpu_ring *ring); int amdgpu_fence_wait_next(struct amdgpu_ring *ring); int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); @@ -1177,8 +1173,7 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned size, struct amdgpu_ib *ib); void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib); int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, - struct amdgpu_ib *ib, void *owner, - struct fence *last_vm_update, + struct amdgpu_ib *ib, struct fence *last_vm_update, struct fence **f); int amdgpu_ib_pool_init(struct amdgpu_device *adev); void amdgpu_ib_pool_fini(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 97db196dc6f8..d94b13ac290f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -91,25 +91,21 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring) * amdgpu_fence_emit - emit a fence on the requested ring * * @ring: ring the fence is associated with - * @owner: creator of the fence * @fence: amdgpu fence object * * Emits a fence command on the requested ring (all asics). * Returns 0 on success, -ENOMEM on failure. */ -int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, - struct amdgpu_fence **fence) +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence **fence) { struct amdgpu_device *adev = ring->adev; - /* we are protected by the ring emission mutex */ *fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); if ((*fence) == NULL) { return -ENOMEM; } (*fence)->seq = ++ring->fence_drv.sync_seq; (*fence)->ring = ring; - (*fence)->owner = owner; fence_init(&(*fence)->base, &amdgpu_fence_ops, &ring->fence_drv.fence_queue.lock, adev->fence_context + ring->idx, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 4e978e7aa1b8..9550247b030d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -101,7 +101,6 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) * @adev: amdgpu_device pointer * @num_ibs: number of IBs to schedule * @ibs: IB objects to schedule - * @owner: owner for creating the fences * @f: fence created during this submission * * Schedule an IB on the associated ring (all asics). @@ -118,8 +117,7 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) * to SI there was just a DE IB. */ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, - struct amdgpu_ib *ibs, void *owner, - struct fence *last_vm_update, + struct amdgpu_ib *ibs, struct fence *last_vm_update, struct fence **f) { struct amdgpu_device *adev = ring->adev; @@ -183,7 +181,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, amdgpu_ring_emit_hdp_invalidate(ring); } - r = amdgpu_fence_emit(ring, owner, &ib->fence); + r = amdgpu_fence_emit(ring, &ib->fence); if (r) { dev_err(adev->dev, "failed to emit fence (%d)\n", r); ring->current_ctx = old_ctx; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 90e52f7e17a0..692b45560d0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -148,7 +148,7 @@ static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job) } trace_amdgpu_sched_run_job(job); - r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job->owner, + r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job->sync.last_vm_update, &fence); if (r) { DRM_ERROR("Error scheduling IBs (%d)\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 87690cc57206..e3673422aac8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -60,12 +60,8 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) */ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) { - struct amdgpu_fence *a_fence = to_amdgpu_fence(f); struct amd_sched_fence *s_fence = to_amd_sched_fence(f); - if (a_fence) - return a_fence->ring->adev == adev; - if (s_fence) { struct amdgpu_ring *ring; @@ -85,13 +81,11 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) */ static void *amdgpu_sync_get_owner(struct fence *f) { - struct amdgpu_fence *a_fence = to_amdgpu_fence(f); struct amd_sched_fence *s_fence = to_amd_sched_fence(f); if (s_fence) return s_fence->owner; - else if (a_fence) - return a_fence->owner; + return AMDGPU_FENCE_OWNER_UNDEFINED; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 1de82bf4fc79..e5f0a5e29551 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -886,8 +886,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, ib->length_dw = 16; if (direct) { - r = amdgpu_ib_schedule(ring, 1, ib, - AMDGPU_FENCE_OWNER_UNDEFINED, NULL, &f); + r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f); if (r) goto err_free; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 39c3aa60381a..6d191fb40b38 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -425,8 +425,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, for (i = ib->length_dw; i < ib_size_dw; ++i) ib->ptr[i] = 0x0; - r = amdgpu_ib_schedule(ring, 1, ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f); if (r) goto err; @@ -487,9 +486,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, ib->ptr[i] = 0x0; if (direct) { - r = amdgpu_ib_schedule(ring, 1, ib, - AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, ib, NULL, &f); if (r) goto err; diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index c4101ebd2b3f..b5b4220a6141 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -643,8 +643,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[3] = 1; ib.ptr[4] = 0xDEADBEEF; ib.length_dw = 5; - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) goto err1; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 1b713009c5d2..46c2436d74bd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2136,8 +2136,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[2] = 0xDEADBEEF; ib.length_dw = 3; - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) goto err2; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index aeb35f504999..e0b64de9b5af 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -706,8 +706,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[2] = 0xDEADBEEF; ib.length_dw = 3; - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) goto err2; @@ -1262,8 +1261,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); /* shedule the ib on the ring */ - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) { DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); goto fail; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 134d5d0bb77f..747ef558033a 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -701,8 +701,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP); ib.length_dw = 8; - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) goto err1; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index ca12d6faa8c2..11a544fbf197 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -853,8 +853,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); ib.length_dw = 8; - r = amdgpu_ib_schedule(ring, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED, - NULL, &f); + r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); if (r) goto err1; From 257bf15a4b9795f8b352beb6e72a7e3e5aab8d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 16 Feb 2016 11:24:58 +0100 Subject: [PATCH 21/50] drm/amdgpu: add slap cache for sync objects as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need them all the time. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 36 +++++++++++++++++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index fab6ddb26b5b..3e4ec56919c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -634,6 +634,8 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); int amdgpu_sync_wait(struct amdgpu_sync *sync); void amdgpu_sync_free(struct amdgpu_sync *sync); +int amdgpu_sync_init(void); +void amdgpu_sync_fini(void); /* * GART structures, functions & helpers diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index ce79a8b605a0..875333bb4e45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -539,6 +539,7 @@ static struct pci_driver amdgpu_kms_pci_driver = { static int __init amdgpu_init(void) { + amdgpu_sync_init(); #ifdef CONFIG_VGA_CONSOLE if (vgacon_text_force()) { DRM_ERROR("VGACON disables amdgpu kernel modesetting.\n"); @@ -563,6 +564,7 @@ static void __exit amdgpu_exit(void) amdgpu_amdkfd_fini(); drm_pci_exit(driver, pdriver); amdgpu_unregister_atpx_handler(); + amdgpu_sync_fini(); } module_init(amdgpu_init); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index e3673422aac8..c48b4fce5e57 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -37,6 +37,8 @@ struct amdgpu_sync_entry { struct fence *fence; }; +static struct kmem_cache *amdgpu_sync_slab; + /** * amdgpu_sync_create - zero init sync object * @@ -133,7 +135,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, return 0; } - e = kmalloc(sizeof(struct amdgpu_sync_entry), GFP_KERNEL); + e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); if (!e) return -ENOMEM; @@ -214,7 +216,7 @@ struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) f = e->fence; hash_del(&e->node); - kfree(e); + kmem_cache_free(amdgpu_sync_slab, e); if (!fence_is_signaled(f)) return f; @@ -237,7 +239,7 @@ int amdgpu_sync_wait(struct amdgpu_sync *sync) hash_del(&e->node); fence_put(e->fence); - kfree(e); + kmem_cache_free(amdgpu_sync_slab, e); } return 0; @@ -259,8 +261,34 @@ void amdgpu_sync_free(struct amdgpu_sync *sync) hash_for_each_safe(sync->fences, i, tmp, e, node) { hash_del(&e->node); fence_put(e->fence); - kfree(e); + kmem_cache_free(amdgpu_sync_slab, e); } fence_put(sync->last_vm_update); } + +/** + * amdgpu_sync_init - init sync object subsystem + * + * Allocate the slab allocator. + */ +int amdgpu_sync_init(void) +{ + amdgpu_sync_slab = kmem_cache_create( + "amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!amdgpu_sync_slab) + return -ENOMEM; + + return 0; +} + +/** + * amdgpu_sync_fini - fini sync object subsystem + * + * Free the slab allocator. + */ +void amdgpu_sync_fini(void) +{ + kmem_cache_destroy(amdgpu_sync_slab); +} From 364beb2cc45247e980a097e53d0932e143873333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 16 Feb 2016 17:39:39 +0100 Subject: [PATCH 22/50] drm/amdgpu: return the common fence from amdgpu_fence_emit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try to avoid using the hardware specific fences even more. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 27 ++++++++++++----------- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 7 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 3e4ec56919c7..128eba604f97 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -429,7 +429,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, unsigned irq_type); void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); void amdgpu_fence_driver_resume(struct amdgpu_device *adev); -int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence **fence); +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **fence); void amdgpu_fence_process(struct amdgpu_ring *ring); int amdgpu_fence_wait_next(struct amdgpu_ring *ring); int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); @@ -766,7 +766,7 @@ struct amdgpu_ib { uint32_t length_dw; uint64_t gpu_addr; uint32_t *ptr; - struct amdgpu_fence *fence; + struct fence *fence; struct amdgpu_user_fence *user; struct amdgpu_vm *vm; unsigned vm_id; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index d94b13ac290f..83599f2a0387 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -91,28 +91,29 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring) * amdgpu_fence_emit - emit a fence on the requested ring * * @ring: ring the fence is associated with - * @fence: amdgpu fence object + * @f: resulting fence object * * Emits a fence command on the requested ring (all asics). * Returns 0 on success, -ENOMEM on failure. */ -int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence **fence) +int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f) { struct amdgpu_device *adev = ring->adev; + struct amdgpu_fence *fence; - *fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); - if ((*fence) == NULL) { + fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL); + if (fence == NULL) return -ENOMEM; - } - (*fence)->seq = ++ring->fence_drv.sync_seq; - (*fence)->ring = ring; - fence_init(&(*fence)->base, &amdgpu_fence_ops, - &ring->fence_drv.fence_queue.lock, - adev->fence_context + ring->idx, - (*fence)->seq); + + fence->seq = ++ring->fence_drv.sync_seq; + fence->ring = ring; + fence_init(&fence->base, &amdgpu_fence_ops, + &ring->fence_drv.fence_queue.lock, + adev->fence_context + ring->idx, + fence->seq); amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, - (*fence)->seq, - AMDGPU_FENCE_FLAG_INT); + fence->seq, AMDGPU_FENCE_FLAG_INT); + *f = &fence->base; return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 9550247b030d..979c445f8096 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -90,9 +90,8 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, */ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) { - amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base); - if (ib->fence) - fence_put(&ib->fence->base); + amdgpu_sa_bo_free(adev, &ib->sa_bo, ib->fence); + fence_put(ib->fence); } /** @@ -198,7 +197,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, } if (f) - *f = fence_get(&ib->fence->base); + *f = fence_get(ib->fence); amdgpu_ring_commit(ring); return 0; From cffadc83c7016ba68721affe5da537ead279e2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 1 Mar 2016 13:34:49 +0100 Subject: [PATCH 23/50] drm/amdgpu: move the GDS switch into vm flush as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After all it's an operation on the VMID. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 6 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 11 ++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 18 +++++++++++++----- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 128eba604f97..b6fae4b301d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -957,8 +957,10 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct fence *fence, unsigned *vm_id, uint64_t *vm_pd_addr); void amdgpu_vm_flush(struct amdgpu_ring *ring, - unsigned vmid, - uint64_t pd_addr); + unsigned vm_id, uint64_t pd_addr, + uint32_t gds_base, uint32_t gds_size, + uint32_t gws_base, uint32_t gws_size, + uint32_t oa_base, uint32_t oa_size); uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr); int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, struct amdgpu_vm *vm); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 979c445f8096..e63e57e51db7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -150,13 +150,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if (vm) { /* do context switch */ - amdgpu_vm_flush(ring, ib->vm_id, ib->vm_pd_addr); - - if (ring->funcs->emit_gds_switch) - amdgpu_ring_emit_gds_switch(ring, ib->vm_id, - ib->gds_base, ib->gds_size, - ib->gws_base, ib->gws_size, - ib->oa_base, ib->oa_size); + amdgpu_vm_flush(ring, ib->vm_id, ib->vm_pd_addr, + ib->gds_base, ib->gds_size, + ib->gws_base, ib->gws_size, + ib->oa_base, ib->oa_size); if (ring->funcs->emit_hdp_flush) amdgpu_ring_emit_hdp_flush(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 364db7c45c67..5fab5b25b935 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -241,19 +241,27 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, * amdgpu_vm_flush - hardware flush the vm * * @ring: ring to use for flush - * @vmid: vmid number to use + * @vm_id: vmid number to use * @pd_addr: address of the page directory * * Emit a VM flush when it is necessary. */ void amdgpu_vm_flush(struct amdgpu_ring *ring, - unsigned vmid, - uint64_t pd_addr) + unsigned vm_id, uint64_t pd_addr, + uint32_t gds_base, uint32_t gds_size, + uint32_t gws_base, uint32_t gws_size, + uint32_t oa_base, uint32_t oa_size) { if (pd_addr != AMDGPU_VM_NO_FLUSH) { - trace_amdgpu_vm_flush(pd_addr, ring->idx, vmid); - amdgpu_ring_emit_vm_flush(ring, vmid, pd_addr); + trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id); + amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr); } + + if (ring->funcs->emit_gds_switch) + amdgpu_ring_emit_gds_switch(ring, vm_id, + gds_base, gds_size, + gws_base, gws_size, + oa_base, oa_size); } /** From 971fe9a9414b2ccabc11ff6a5ff6be0d6f2dabda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 1 Mar 2016 15:09:25 +0100 Subject: [PATCH 24/50] drm/amdgpu: switch the GDS only on demand v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switching the GDS space to often seems to be problematic. This patch together with the following can avoid VM faults on context switch. v2: extend commit message a bit Signed-off-by: Christian König Reviewed-by: Alex Deucher (v1) Reviewed-by: Chunming Zhou (v1) --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 4 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 43 ++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index b6fae4b301d7..866b5fa298e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -922,6 +922,13 @@ struct amdgpu_vm_manager_id { struct list_head list; struct fence *active; atomic_long_t owner; + + uint32_t gds_base; + uint32_t gds_size; + uint32_t gws_base; + uint32_t gws_size; + uint32_t oa_base; + uint32_t oa_size; }; struct amdgpu_vm_manager { @@ -961,6 +968,7 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, uint32_t gds_base, uint32_t gds_size, uint32_t gws_base, uint32_t gws_size, uint32_t oa_base, uint32_t oa_size); +void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id); uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr); int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, struct amdgpu_vm *vm); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index e63e57e51db7..798d46626820 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -165,6 +165,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if (ib->ctx != ctx || ib->vm != vm) { ring->current_ctx = old_ctx; + if (ib->vm_id) + amdgpu_vm_reset_id(adev, ib->vm_id); amdgpu_ring_undo(ring); return -EINVAL; } @@ -181,6 +183,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if (r) { dev_err(adev->dev, "failed to emit fence (%d)\n", r); ring->current_ctx = old_ctx; + if (ib->vm_id) + amdgpu_vm_reset_id(adev, ib->vm_id); amdgpu_ring_undo(ring); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 5fab5b25b935..8642a1ccd6c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -252,16 +252,53 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, uint32_t gws_base, uint32_t gws_size, uint32_t oa_base, uint32_t oa_size) { + struct amdgpu_device *adev = ring->adev; + struct amdgpu_vm_manager_id *mgr_id = &adev->vm_manager.ids[vm_id]; + if (pd_addr != AMDGPU_VM_NO_FLUSH) { trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id); amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr); } - if (ring->funcs->emit_gds_switch) + if (ring->funcs->emit_gds_switch && ( + mgr_id->gds_base != gds_base || + mgr_id->gds_size != gds_size || + mgr_id->gws_base != gws_base || + mgr_id->gws_size != gws_size || + mgr_id->oa_base != oa_base || + mgr_id->oa_size != oa_size)) { + + mgr_id->gds_base = gds_base; + mgr_id->gds_size = gds_size; + mgr_id->gws_base = gws_base; + mgr_id->gws_size = gws_size; + mgr_id->oa_base = oa_base; + mgr_id->oa_size = oa_size; amdgpu_ring_emit_gds_switch(ring, vm_id, gds_base, gds_size, gws_base, gws_size, oa_base, oa_size); + } +} + +/** + * amdgpu_vm_reset_id - reset VMID to zero + * + * @adev: amdgpu device structure + * @vm_id: vmid number to use + * + * Reset saved GDW, GWS and OA to force switch on next flush. + */ +void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id) +{ + struct amdgpu_vm_manager_id *mgr_id = &adev->vm_manager.ids[vm_id]; + + mgr_id->gds_base = 0; + mgr_id->gds_size = 0; + mgr_id->gws_base = 0; + mgr_id->gws_size = 0; + mgr_id->oa_base = 0; + mgr_id->oa_size = 0; } /** @@ -1425,9 +1462,11 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) INIT_LIST_HEAD(&adev->vm_manager.ids_lru); /* skip over VMID 0, since it is the system VM */ - for (i = 1; i < adev->vm_manager.num_ids; ++i) + for (i = 1; i < adev->vm_manager.num_ids; ++i) { + amdgpu_vm_reset_id(adev, i); list_add_tail(&adev->vm_manager.ids[i].list, &adev->vm_manager.ids_lru); + } atomic_set(&adev->vm_manager.vm_pte_next_ring, 0); } From b8c7b39ec1db98199df5bee33ca145ed9c2c62e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 1 Mar 2016 15:42:52 +0100 Subject: [PATCH 25/50] drm/amdgpu: split pipeline sync and vm flush MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to use the pipeline sync for other tasks as well. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 ++ drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 29 +++++++++++++++++++------- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 11 ++++++++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 866b5fa298e7..e0727de9c2b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -326,6 +326,7 @@ struct amdgpu_ring_funcs { struct amdgpu_ib *ib); void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr, uint64_t seq, unsigned flags); + void (*emit_pipeline_sync)(struct amdgpu_ring *ring); void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr); void (*emit_hdp_flush)(struct amdgpu_ring *ring); @@ -2234,6 +2235,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) #define amdgpu_ring_emit_ib(r, ib) (r)->funcs->emit_ib((r), (ib)) +#define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r)) #define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr)) #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as)) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 8642a1ccd6c3..9a9abbab9b7c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -257,6 +257,8 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, if (pd_addr != AMDGPU_VM_NO_FLUSH) { trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id); + if (ring->funcs->emit_pipeline_sync) + amdgpu_ring_emit_pipeline_sync(ring); amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 46c2436d74bd..a4913ebb673c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -3041,6 +3041,26 @@ static int gfx_v7_0_cp_resume(struct amdgpu_device *adev) return 0; } +/** + * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP + * + * @ring: the ring to emmit the commands to + * + * Sync the command pipeline with the PFP. E.g. wait for everything + * to be completed. + */ +static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) +{ + int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); + if (usepfp) { + /* synce CE with ME to prevent CE fetch CEIB before context switch done */ + amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + amdgpu_ring_write(ring, 0); + } +} + /* * vm * VMID 0 is the physical GPU addresses as used by the kernel. @@ -3059,13 +3079,6 @@ static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr) { int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); - if (usepfp) { - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -5147,6 +5160,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = { .parse_cs = NULL, .emit_ib = gfx_v7_0_ring_emit_ib_gfx, .emit_fence = gfx_v7_0_ring_emit_fence_gfx, + .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, @@ -5164,6 +5178,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = { .parse_cs = NULL, .emit_ib = gfx_v7_0_ring_emit_ib_compute, .emit_fence = gfx_v7_0_ring_emit_fence_compute, + .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index e0b64de9b5af..828e205a7326 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4692,8 +4692,7 @@ static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, } -static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr) +static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); uint32_t seq = ring->fence_drv.sync_seq; @@ -4715,6 +4714,12 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); amdgpu_ring_write(ring, 0); } +} + +static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -5037,6 +5042,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { .parse_cs = NULL, .emit_ib = gfx_v8_0_ring_emit_ib_gfx, .emit_fence = gfx_v8_0_ring_emit_fence_gfx, + .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, @@ -5054,6 +5060,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { .parse_cs = NULL, .emit_ib = gfx_v8_0_ring_emit_ib_compute, .emit_fence = gfx_v8_0_ring_emit_fence_compute, + .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, From d564a06e1c9c285bab1c1579c18c811aa1271884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 1 Mar 2016 15:51:53 +0100 Subject: [PATCH 26/50] drm/amdgpu: if a GDS switch is needed emit a pipeline sync as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise we might change the GDS settings while they are still in use. Signed-off-by: Christian König Reviewed-by: Alex Deucher Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 9a9abbab9b7c..a0896c761108 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -254,22 +254,24 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, { struct amdgpu_device *adev = ring->adev; struct amdgpu_vm_manager_id *mgr_id = &adev->vm_manager.ids[vm_id]; + bool gds_switch_needed = ring->funcs->emit_gds_switch && ( + mgr_id->gds_base != gds_base || + mgr_id->gds_size != gds_size || + mgr_id->gws_base != gws_base || + mgr_id->gws_size != gws_size || + mgr_id->oa_base != oa_base || + mgr_id->oa_size != oa_size); + + if (ring->funcs->emit_pipeline_sync && ( + pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed)) + amdgpu_ring_emit_pipeline_sync(ring); if (pd_addr != AMDGPU_VM_NO_FLUSH) { trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id); - if (ring->funcs->emit_pipeline_sync) - amdgpu_ring_emit_pipeline_sync(ring); amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr); } - if (ring->funcs->emit_gds_switch && ( - mgr_id->gds_base != gds_base || - mgr_id->gds_size != gds_size || - mgr_id->gws_base != gws_base || - mgr_id->gws_size != gws_size || - mgr_id->oa_base != oa_base || - mgr_id->oa_size != oa_size)) { - + if (gds_switch_needed) { mgr_id->gds_base = gds_base; mgr_id->gds_size = gds_size; mgr_id->gws_base = gws_base; From 2f568dbd6b944c2e8c0c54b53c2211c23995e6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 23 Feb 2016 12:36:59 +0100 Subject: [PATCH 27/50] drm/amdgpu: move get_user_pages out of amdgpu_ttm_tt_pin_userptr v6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That avoids lock inversion between the BO reservation lock and the anon_vma lock. v2: * Changed amdgpu_bo_list_entry.user_pages to an array of pointers * Lock mmap_sem only for get_user_pages * Added invalidation of unbound userpointer BOs * Fixed memory leak and page reference leak v3 (chk): * Revert locking mmap_sem only for_get user_pages * Revert adding invalidation of unbound userpointer BOs * Sanitize and fix error handling v4 (chk): * Init userpages pointer everywhere. * Fix error handling when get_user_pages() fails. * Add invalidation of unbound userpointer BOs again. v5 (chk): * Add maximum number of tries. v6 (chk): * Fix error handling when we run out of tries. Signed-off-by: Christian König Signed-off-by: Felix Kuehling Reviewed-by: Felix Kuehling (v4) Acked-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 5 + drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 114 +++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 23 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 53 ++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 + 6 files changed, 174 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index e0727de9c2b2..e32ab13227d4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -471,6 +471,8 @@ struct amdgpu_bo_list_entry { struct ttm_validate_buffer tv; struct amdgpu_bo_va *bo_va; uint32_t priority; + struct page **user_pages; + int user_invalidated; }; struct amdgpu_bo_va_mapping { @@ -2365,11 +2367,14 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, struct amdgpu_ring **out_ring); void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *rbo, u32 domain); bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); +int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages); int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, uint32_t flags); struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm); bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, unsigned long end); +bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm, + int *last_invalidated); bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm); uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, struct ttm_mem_reg *mem); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 9763e52886fe..eacd810fc09b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -200,6 +200,7 @@ void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list, list_add_tail(&list->array[i].tv.head, &bucket[priority]); + list->array[i].user_pages = NULL; } /* Connect the sorted buckets in the output list. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 7833dfb1ff6e..4f5ef4149e87 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -111,6 +111,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, p->uf_entry.priority = 0; p->uf_entry.tv.bo = &p->uf_entry.robj->tbo; p->uf_entry.tv.shared = true; + p->uf_entry.user_pages = NULL; drm_gem_object_unreference_unlocked(gobj); return 0; @@ -297,6 +298,7 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, list_for_each_entry(lobj, validated, tv.head) { struct amdgpu_bo *bo = lobj->robj; + bool binding_userptr = false; struct mm_struct *usermm; uint32_t domain; @@ -304,6 +306,15 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, if (usermm && usermm != current->mm) return -EPERM; + /* Check if we have user pages and nobody bound the BO already */ + if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) { + size_t size = sizeof(struct page *); + + size *= bo->tbo.ttm->num_pages; + memcpy(bo->tbo.ttm->pages, lobj->user_pages, size); + binding_userptr = true; + } + if (bo->pin_count) continue; @@ -334,6 +345,11 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, } return r; } + + if (binding_userptr) { + drm_free_large(lobj->user_pages); + lobj->user_pages = NULL; + } } return 0; } @@ -342,8 +358,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_bo_list_entry *e; struct list_head duplicates; bool need_mmap_lock = false; + unsigned i, tries = 10; int r; INIT_LIST_HEAD(&p->validated); @@ -364,9 +382,81 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, if (need_mmap_lock) down_read(¤t->mm->mmap_sem); - r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); - if (unlikely(r != 0)) - goto error_reserve; + while (1) { + struct list_head need_pages; + unsigned i; + + r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, + &duplicates); + if (unlikely(r != 0)) + goto error_free_pages; + + /* Without a BO list we don't have userptr BOs */ + if (!p->bo_list) + break; + + INIT_LIST_HEAD(&need_pages); + for (i = p->bo_list->first_userptr; + i < p->bo_list->num_entries; ++i) { + + e = &p->bo_list->array[i]; + + if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm, + &e->user_invalidated) && e->user_pages) { + + /* We acquired a page array, but somebody + * invalidated it. Free it an try again + */ + release_pages(e->user_pages, + e->robj->tbo.ttm->num_pages, + false); + drm_free_large(e->user_pages); + e->user_pages = NULL; + } + + if (e->robj->tbo.ttm->state != tt_bound && + !e->user_pages) { + list_del(&e->tv.head); + list_add(&e->tv.head, &need_pages); + + amdgpu_bo_unreserve(e->robj); + } + } + + if (list_empty(&need_pages)) + break; + + /* Unreserve everything again. */ + ttm_eu_backoff_reservation(&p->ticket, &p->validated); + + /* We tried to often, just abort */ + if (!--tries) { + r = -EDEADLK; + goto error_free_pages; + } + + /* Fill the page arrays for all useptrs. */ + list_for_each_entry(e, &need_pages, tv.head) { + struct ttm_tt *ttm = e->robj->tbo.ttm; + + e->user_pages = drm_calloc_large(ttm->num_pages, + sizeof(struct page*)); + if (!e->user_pages) { + r = -ENOMEM; + goto error_free_pages; + } + + r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages); + if (r) { + drm_free_large(e->user_pages); + e->user_pages = NULL; + goto error_free_pages; + } + } + + /* And try again. */ + list_splice(&need_pages, &p->validated); + } amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates); @@ -398,10 +488,26 @@ error_validate: ttm_eu_backoff_reservation(&p->ticket, &p->validated); } -error_reserve: +error_free_pages: + if (need_mmap_lock) up_read(¤t->mm->mmap_sem); + if (p->bo_list) { + for (i = p->bo_list->first_userptr; + i < p->bo_list->num_entries; ++i) { + e = &p->bo_list->array[i]; + + if (!e->user_pages) + continue; + + release_pages(e->user_pages, + e->robj->tbo.ttm->num_pages, + false); + drm_free_large(e->user_pages); + } + } + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 2e26a517f2d6..cb27754ec07f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -274,18 +274,23 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) { down_read(¤t->mm->mmap_sem); + + r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm, + bo->tbo.ttm->pages); + if (r) + goto unlock_mmap_sem; + r = amdgpu_bo_reserve(bo, true); - if (r) { - up_read(¤t->mm->mmap_sem); - goto release_object; - } + if (r) + goto free_pages; amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); amdgpu_bo_unreserve(bo); - up_read(¤t->mm->mmap_sem); if (r) - goto release_object; + goto free_pages; + + up_read(¤t->mm->mmap_sem); } r = drm_gem_handle_create(filp, gobj, &handle); @@ -297,6 +302,12 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, args->handle = handle; return 0; +free_pages: + release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages, false); + +unlock_mmap_sem: + up_read(¤t->mm->mmap_sem); + release_object: drm_gem_object_unreference_unlocked(gobj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 53e627970798..ed0d2e7cc57d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -508,22 +508,18 @@ struct amdgpu_ttm_tt { uint32_t userflags; spinlock_t guptasklock; struct list_head guptasks; + atomic_t mmu_invalidations; }; -/* prepare the sg table with the user pages */ -static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) +int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) { - struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev); struct amdgpu_ttm_tt *gtt = (void *)ttm; - unsigned pinned = 0, nents; + int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); + unsigned pinned = 0; int r; - int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); - enum dma_data_direction direction = write ? - DMA_BIDIRECTIONAL : DMA_TO_DEVICE; - if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) { - /* check that we only pin down anonymous memory + /* check that we only use anonymous memory to prevent problems with writeback */ unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; struct vm_area_struct *vma; @@ -536,7 +532,7 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) do { unsigned num_pages = ttm->num_pages - pinned; uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; - struct page **pages = ttm->pages + pinned; + struct page **p = pages + pinned; struct amdgpu_ttm_gup_task_list guptask; guptask.task = current; @@ -545,7 +541,7 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) spin_unlock(>t->guptasklock); r = get_user_pages(current, current->mm, userptr, num_pages, - write, 0, pages, NULL); + write, 0, p, NULL); spin_lock(>t->guptasklock); list_del(&guptask.list); @@ -558,6 +554,25 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) } while (pinned < ttm->num_pages); + return 0; + +release_pages: + release_pages(pages, pinned, 0); + return r; +} + +/* prepare the sg table with the user pages */ +static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) +{ + struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev); + struct amdgpu_ttm_tt *gtt = (void *)ttm; + unsigned nents; + int r; + + int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); + enum dma_data_direction direction = write ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, ttm->num_pages << PAGE_SHIFT, GFP_KERNEL); @@ -576,9 +591,6 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) release_sg: kfree(ttm->sg); - -release_pages: - release_pages(ttm->pages, pinned, 0); return r; } @@ -803,6 +815,7 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, gtt->userflags = flags; spin_lock_init(>t->guptasklock); INIT_LIST_HEAD(>t->guptasks); + atomic_set(>t->mmu_invalidations, 0); return 0; } @@ -840,9 +853,21 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, } spin_unlock(>t->guptasklock); + atomic_inc(>t->mmu_invalidations); + return true; } +bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm, + int *last_invalidated) +{ + struct amdgpu_ttm_tt *gtt = (void *)ttm; + int prev_invalidated = *last_invalidated; + + *last_invalidated = atomic_read(>t->mmu_invalidations); + return prev_invalidated != *last_invalidated; +} + bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) { struct amdgpu_ttm_tt *gtt = (void *)ttm; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index a0896c761108..cc967b44f104 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -95,6 +95,7 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, entry->priority = 0; entry->tv.bo = &vm->page_directory->tbo; entry->tv.shared = true; + entry->user_pages = NULL; list_add(&entry->tv.head, validated); } @@ -1186,6 +1187,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, entry->priority = 0; entry->tv.bo = &entry->robj->tbo; entry->tv.shared = true; + entry->user_pages = NULL; vm->page_tables[pt_idx].addr = 0; } From 29f646dfb2269fa55d48a8d8c91a42ee64e5e4fc Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Wed, 2 Mar 2016 08:58:07 -0500 Subject: [PATCH 28/50] drm/amd/amdgpu: Don't proceed in audio_fini in DCEv11 if disabled If amdgpu_audio is disabled then the audio structure is not initialized so we shouldn't read it in the fini function. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 6e8d74da183f..4501ed0e97c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -1658,6 +1658,9 @@ static void dce_v11_0_audio_fini(struct amdgpu_device *adev) { int i; + if (!amdgpu_audio) + return; + if (!adev->mode_info.audio.enabled) return; From bcc71beb2a101699426820d5cd3f28dcb9877181 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Wed, 2 Mar 2016 09:10:50 -0500 Subject: [PATCH 29/50] drm/amd/amdgpu: Whitespace typo fix in sw_init (DCEv11) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 4501ed0e97c7..75b5366b9a49 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -2964,7 +2964,7 @@ static int dce_v11_0_sw_init(void *handle) for (i = 0; i < adev->mode_info.num_crtc; i++) { r = amdgpu_irq_add_id(adev, i + 1, &adev->crtc_irq); if (r) - return r; + return r; } for (i = 8; i < 20; i += 2) { @@ -2976,7 +2976,7 @@ static int dce_v11_0_sw_init(void *handle) /* HPD hotplug */ r = amdgpu_irq_add_id(adev, 42, &adev->hpd_irq); if (r) - return r; + return r; adev->mode_info.mode_config_initialized = true; From c437b9d6c093c83c485a40c0f9e0d2cd98b36d38 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Wed, 2 Mar 2016 09:14:40 -0500 Subject: [PATCH 30/50] drm/amd/amdgpu: Move init flag to after init in sw_init() (DCEv11) Don't set config_init to true until all config statements pass. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 75b5366b9a49..ef7290c552b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -2978,8 +2978,6 @@ static int dce_v11_0_sw_init(void *handle) if (r) return r; - adev->mode_info.mode_config_initialized = true; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; adev->ddev->mode_config.max_width = 16384; @@ -2997,6 +2995,7 @@ static int dce_v11_0_sw_init(void *handle) adev->ddev->mode_config.max_width = 16384; adev->ddev->mode_config.max_height = 16384; + /* allocate crtcs */ for (i = 0; i < adev->mode_info.num_crtc; i++) { r = dce_v11_0_crtc_init(adev, i); @@ -3018,7 +3017,8 @@ static int dce_v11_0_sw_init(void *handle) drm_kms_helper_poll_init(adev->ddev); - return r; + adev->mode_info.mode_config_initialized = true; + return 0; } static int dce_v11_0_sw_fini(void *handle) From 041ab0a49482502bd138203fd1b4d529eab97fe4 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Wed, 2 Mar 2016 09:19:57 -0500 Subject: [PATCH 31/50] drm/amd/amdgpu: Make afmt_init() cleanup if alloc fails (DCEv11) Updated DCEv11 afmt_init to cleanup if any kzalloc fails and then return an error code. Don't continue initializing the audio stack in that case. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index ef7290c552b0..45c96c9b7ff1 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -1966,7 +1966,7 @@ static void dce_v11_0_afmt_enable(struct drm_encoder *encoder, bool enable) enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); } -static void dce_v11_0_afmt_init(struct amdgpu_device *adev) +static int dce_v11_0_afmt_init(struct amdgpu_device *adev) { int i; @@ -1979,8 +1979,16 @@ static void dce_v11_0_afmt_init(struct amdgpu_device *adev) if (adev->mode_info.afmt[i]) { adev->mode_info.afmt[i]->offset = dig_offsets[i]; adev->mode_info.afmt[i]->id = i; + } else { + int j; + for (j = 0; j < i; j++) { + kfree(adev->mode_info.afmt[j]); + adev->mode_info.afmt[j] = NULL; + } + return -ENOMEM; } } + return 0; } static void dce_v11_0_afmt_fini(struct amdgpu_device *adev) @@ -3009,7 +3017,9 @@ static int dce_v11_0_sw_init(void *handle) return -EINVAL; /* setup afmt */ - dce_v11_0_afmt_init(adev); + r = dce_v11_0_afmt_init(adev); + if (r) + return r; r = dce_v11_0_audio_init(adev); if (r) From e484f8d47965d1fecac3c46977bf1fd3e688b432 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Wed, 2 Mar 2016 12:07:02 -0500 Subject: [PATCH 32/50] drm/amd/amdgpu: Fix indentation in dce_v11_0_crtc_do_set_base() Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 45c96c9b7ff1..e9ccc6b787f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -2065,8 +2065,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc, if (atomic) { amdgpu_fb = to_amdgpu_framebuffer(fb); target_fb = fb; - } - else { + } else { amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); target_fb = crtc->primary->fb; } @@ -2080,9 +2079,9 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc, if (unlikely(r != 0)) return r; - if (atomic) + if (atomic) { fb_location = amdgpu_bo_gpu_offset(rbo); - else { + } else { r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); if (unlikely(r != 0)) { amdgpu_bo_unreserve(rbo); From 441ce96f14075f7c5e2995f07e7f037cc43c4a08 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:20:10 -0500 Subject: [PATCH 33/50] drm/amd/amdgpu: Don't proceed in audio_fini if disabled (DCEv10) If audio is disabled we shouldn't proceed into the fini function. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index b8cb411918ab..199b95fff046 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -1668,6 +1668,9 @@ static void dce_v10_0_audio_fini(struct amdgpu_device *adev) { int i; + if (!amdgpu_audio) + return; + if (!adev->mode_info.audio.enabled) return; From 98822a2f748cf400a1fa08eabd0ccf6baa146a92 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:21:44 -0500 Subject: [PATCH 34/50] drm/amd/amdgpu: Move initialized flag to bottom of sw_init (DCEv10) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 199b95fff046..6555b4a0c34a 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -2983,8 +2983,6 @@ static int dce_v10_0_sw_init(void *handle) if (r) return r; - adev->mode_info.mode_config_initialized = true; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; adev->ddev->mode_config.max_width = 16384; @@ -3023,7 +3021,8 @@ static int dce_v10_0_sw_init(void *handle) drm_kms_helper_poll_init(adev->ddev); - return r; + adev->mode_info.mode_config_initialized = true; + return 0; } static int dce_v10_0_sw_fini(void *handle) From 720a6ce3cedf664834df436c343ea2ccc26272fa Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:23:48 -0500 Subject: [PATCH 35/50] drm/amd/amdgpu: Make afmt_init cleanup if alloc fails (DCEv10) Make the function free memory and return an error code if the allocation fails. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 6555b4a0c34a..dc29e061c179 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -1976,7 +1976,7 @@ static void dce_v10_0_afmt_enable(struct drm_encoder *encoder, bool enable) enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); } -static void dce_v10_0_afmt_init(struct amdgpu_device *adev) +static int dce_v10_0_afmt_init(struct amdgpu_device *adev) { int i; @@ -1989,8 +1989,16 @@ static void dce_v10_0_afmt_init(struct amdgpu_device *adev) if (adev->mode_info.afmt[i]) { adev->mode_info.afmt[i]->offset = dig_offsets[i]; adev->mode_info.afmt[i]->id = i; + } else { + int j; + for (j = 0; j < i; j++) { + kfree(adev->mode_info.afmt[j]); + adev->mode_info.afmt[j] = NULL; + } + return -ENOMEM; } } + return 0; } static void dce_v10_0_afmt_fini(struct amdgpu_device *adev) @@ -3013,7 +3021,9 @@ static int dce_v10_0_sw_init(void *handle) return -EINVAL; /* setup afmt */ - dce_v10_0_afmt_init(adev); + r = dce_v10_0_afmt_init(adev); + if (r) + return r; r = dce_v10_0_audio_init(adev); if (r) From 849dc32b20a496a45ad8d00fefd8ade37a272aba Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:25:05 -0500 Subject: [PATCH 36/50] drm/amd/amdgpu: Fix identation in do_set_base() (DCEv10) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index dc29e061c179..6de2ce535e37 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -2075,8 +2075,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc, if (atomic) { amdgpu_fb = to_amdgpu_framebuffer(fb); target_fb = fb; - } - else { + } else { amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); target_fb = crtc->primary->fb; } @@ -2090,9 +2089,9 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc, if (unlikely(r != 0)) return r; - if (atomic) + if (atomic) { fb_location = amdgpu_bo_gpu_offset(rbo); - else { + } else { r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); if (unlikely(r != 0)) { amdgpu_bo_unreserve(rbo); From 84cffef127cce662e73f5f887c590ac582dcc845 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:31:49 -0500 Subject: [PATCH 37/50] drm/amd/amdgpu: Don't proceed into audio_fini if audio is disabled (DCEv8) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index b059853d93bf..ab536d3ed965 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -1639,6 +1639,9 @@ static void dce_v8_0_audio_fini(struct amdgpu_device *adev) { int i; + if (!amdgpu_audio) + return; + if (!adev->mode_info.audio.enabled) return; From 74c1e842793fcc470413b1f7d14e2e74e1e8240b Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:42:43 -0500 Subject: [PATCH 38/50] drm/amd/amdgpu: Move config init flag to bottom of sw_init (DCEv8) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index ab536d3ed965..a7d871105b2a 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -2893,8 +2893,6 @@ static int dce_v8_0_sw_init(void *handle) if (r) return r; - adev->mode_info.mode_config_initialized = true; - adev->ddev->mode_config.funcs = &amdgpu_mode_funcs; adev->ddev->mode_config.max_width = 16384; @@ -2933,7 +2931,8 @@ static int dce_v8_0_sw_init(void *handle) drm_kms_helper_poll_init(adev->ddev); - return r; + adev->mode_info.mode_config_initialized = true; + return 0; } static int dce_v8_0_sw_fini(void *handle) From ff923479ee9b6ba9244398c7e97878fa37b5343c Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:45:29 -0500 Subject: [PATCH 39/50] drm/amd/amdgpu: make afmt_init cleanup if alloc fails (DCEv8) If the allocation fails free memory and return error code. Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index a7d871105b2a..35b4f7752ab5 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -1913,7 +1913,7 @@ static void dce_v8_0_afmt_enable(struct drm_encoder *encoder, bool enable) enable ? "En" : "Dis", dig->afmt->offset, amdgpu_encoder->encoder_id); } -static void dce_v8_0_afmt_init(struct amdgpu_device *adev) +static int dce_v8_0_afmt_init(struct amdgpu_device *adev) { int i; @@ -1926,8 +1926,16 @@ static void dce_v8_0_afmt_init(struct amdgpu_device *adev) if (adev->mode_info.afmt[i]) { adev->mode_info.afmt[i]->offset = dig_offsets[i]; adev->mode_info.afmt[i]->id = i; + } else { + int j; + for (j = 0; j < i; j++) { + kfree(adev->mode_info.afmt[j]); + adev->mode_info.afmt[j] = NULL; + } + return -ENOMEM; } } + return 0; } static void dce_v8_0_afmt_fini(struct amdgpu_device *adev) @@ -2923,7 +2931,9 @@ static int dce_v8_0_sw_init(void *handle) return -EINVAL; /* setup afmt */ - dce_v8_0_afmt_init(adev); + r = dce_v8_0_afmt_init(adev); + if (r) + return r; r = dce_v8_0_audio_init(adev); if (r) From 92821c261d1f0ca250b7cff4e4bbcc58d8530f90 Mon Sep 17 00:00:00 2001 From: Tom St Denis Date: Thu, 3 Mar 2016 09:46:41 -0500 Subject: [PATCH 40/50] drm/amd/amdgpu: Fix indentation in do_set_base() (DCEv8) Signed-off-by: Tom St Denis Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 35b4f7752ab5..e56b55d8c280 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -2012,8 +2012,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc, if (atomic) { amdgpu_fb = to_amdgpu_framebuffer(fb); target_fb = fb; - } - else { + } else { amdgpu_fb = to_amdgpu_framebuffer(crtc->primary->fb); target_fb = crtc->primary->fb; } @@ -2027,9 +2026,9 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc, if (unlikely(r != 0)) return r; - if (atomic) + if (atomic) { fb_location = amdgpu_bo_gpu_offset(rbo); - else { + } else { r = amdgpu_bo_pin(rbo, AMDGPU_GEM_DOMAIN_VRAM, &fb_location); if (unlikely(r != 0)) { amdgpu_bo_unreserve(rbo); From e64c952efb8e0c15ae82cec8e455ab4910690ef1 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 2 Mar 2016 11:47:29 -0500 Subject: [PATCH 41/50] drm/radeon: disable runtime pm on PX laptops without dGPU power control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some PX laptops don't provide an ACPI method to control dGPU power. On those systems, the driver is responsible for handling the dGPU power state. Disable runtime PM on them until support for this is implemented. Reviewed-by: Michel Dänzer Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/radeon/radeon_atpx_handler.c | 8 ++++---- drivers/gpu/drm/radeon/radeon_device.c | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c index 56482e35d43e..fd8c4d317e60 100644 --- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c +++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c @@ -62,6 +62,10 @@ bool radeon_has_atpx(void) { return radeon_atpx_priv.atpx_detected; } +bool radeon_has_atpx_dgpu_power_cntl(void) { + return radeon_atpx_priv.atpx.functions.power_cntl; +} + /** * radeon_atpx_call - call an ATPX method * @@ -141,10 +145,6 @@ static void radeon_atpx_parse_functions(struct radeon_atpx_functions *f, u32 mas */ static int radeon_atpx_validate(struct radeon_atpx *atpx) { - /* make sure required functions are enabled */ - /* dGPU power control is required */ - atpx->functions.power_cntl = true; - if (atpx->functions.px_params) { union acpi_object *info; struct atpx_px_params output; diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 902b59cebac5..4de23ae3a64b 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -103,6 +103,12 @@ static const char radeon_family_name[][16] = { "LAST", }; +#if defined(CONFIG_VGA_SWITCHEROO) +bool radeon_has_atpx_dgpu_power_cntl(void); +#else +static inline bool radeon_has_atpx_dgpu_power_cntl(void) { return false; } +#endif + #define RADEON_PX_QUIRK_DISABLE_PX (1 << 0) #define RADEON_PX_QUIRK_LONG_WAKEUP (1 << 1) @@ -1433,7 +1439,7 @@ int radeon_device_init(struct radeon_device *rdev, * ignore it */ vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode); - if (rdev->flags & RADEON_IS_PX) + if ((rdev->flags & RADEON_IS_PX) && radeon_has_atpx_dgpu_power_cntl()) runtime = true; vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime); if (runtime) From bedf2a65c1aa8fb29ba8527fd00c0f68ec1f55f1 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 2 Mar 2016 12:10:20 -0500 Subject: [PATCH 42/50] drm/amdgpu: disable runtime pm on PX laptops without dGPU power control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some PX laptops don't provide an ACPI method to control dGPU power. On those systems, the driver is responsible for handling the dGPU power state. Disable runtime PM on them until support for this is implemented. Reviewed-by: Michel Dänzer Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c | 8 ++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c index fa948dcbdd5d..0020a0ea43ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c @@ -63,6 +63,10 @@ bool amdgpu_has_atpx(void) { return amdgpu_atpx_priv.atpx_detected; } +bool amdgpu_has_atpx_dgpu_power_cntl(void) { + return amdgpu_atpx_priv.atpx.functions.power_cntl; +} + /** * amdgpu_atpx_call - call an ATPX method * @@ -142,10 +146,6 @@ static void amdgpu_atpx_parse_functions(struct amdgpu_atpx_functions *f, u32 mas */ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx) { - /* make sure required functions are enabled */ - /* dGPU power control is required */ - atpx->functions.power_cntl = true; - if (atpx->functions.px_params) { union acpi_object *info; struct atpx_px_params output; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index db20d2783def..462fec0ab594 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -62,6 +62,12 @@ static const char *amdgpu_asic_name[] = { "LAST", }; +#if defined(CONFIG_VGA_SWITCHEROO) +bool amdgpu_has_atpx_dgpu_power_cntl(void); +#else +static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; } +#endif + bool amdgpu_device_is_px(struct drm_device *dev) { struct amdgpu_device *adev = dev->dev_private; @@ -1479,7 +1485,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (amdgpu_runtime_pm == 1) runtime = true; - if (amdgpu_device_is_px(ddev)) + if (amdgpu_device_is_px(ddev) && amdgpu_has_atpx_dgpu_power_cntl()) runtime = true; vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime); if (runtime) From c26a4017da144227bb4a02f354c9bb6079854201 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Thu, 3 Mar 2016 11:49:34 -0500 Subject: [PATCH 43/50] MAINTAINERS: update radeon entry to include amdgpu as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are maintained by same team. Reviewed-by: Christian König Acked-by: Thierry Reding Signed-off-by: Alex Deucher --- MAINTAINERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index c647c40e141d..20166fbd0bac 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3712,7 +3712,7 @@ F: drivers/gpu/vga/ F: include/drm/ F: include/uapi/drm/ -RADEON DRM DRIVERS +RADEON and AMDGPU DRM DRIVERS M: Alex Deucher M: Christian König L: dri-devel@lists.freedesktop.org @@ -3720,6 +3720,8 @@ T: git git://people.freedesktop.org/~agd5f/linux S: Supported F: drivers/gpu/drm/radeon/ F: include/uapi/drm/radeon* +F: drivers/gpu/drm/amd/ +F: include/uapi/drm/amdgpu* DRM PANEL DRIVERS M: Thierry Reding From 459ee1c3fd097ab56ababd8ff4bb7ef6a792de33 Mon Sep 17 00:00:00 2001 From: Mario Kleiner Date: Sun, 6 Mar 2016 02:39:53 +0100 Subject: [PATCH 44/50] drm/radeon: Don't drop DP 2.7 Ghz link setup on some cards. As observed on Apple iMac10,1, DCE-3.2, RV-730, link rate of 2.7 Ghz is not selected, because the args.v1.ucConfig flag setting for 2.7 Ghz gets overwritten by a following assignment of the transmitter to use. Move link rate setup a few lines down to fix this. In practice this didn't have any positive or negative effect on display setup on the tested iMac10,1 so i don't know if backporting to stable makes sense or not. Signed-off-by: Mario Kleiner Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/radeon/atombios_encoders.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c index 1603751b1164..edd05cdb0cd8 100644 --- a/drivers/gpu/drm/radeon/atombios_encoders.c +++ b/drivers/gpu/drm/radeon/atombios_encoders.c @@ -892,8 +892,6 @@ atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_m else args.v1.ucLaneNum = 4; - if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000)) - args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; switch (radeon_encoder->encoder_id) { case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1; @@ -910,6 +908,10 @@ atombios_dig_encoder_setup2(struct drm_encoder *encoder, int action, int panel_m args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKB; else args.v1.ucConfig |= ATOM_ENCODER_CONFIG_LINKA; + + if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000)) + args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ; + break; case 2: case 3: From fb29b57c34488b343db2e3f00e713901385f8c0d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 3 Mar 2016 19:13:20 -0500 Subject: [PATCH 45/50] drm/amdgpu: Fix two bugs in amdgpu_vm_bo_split_mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Off-by-one: last is inclusive, so the maximum is start + max_size - 1 Wrong unit: addr is in bytes, max_size is in pages Signed-off-by: Felix Kuehling Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index cc967b44f104..330e307c36ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -867,7 +867,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, while (start != mapping->it.last + 1) { uint64_t last; - last = min((uint64_t)mapping->it.last, start + max_size); + last = min((uint64_t)mapping->it.last, start + max_size - 1); r = amdgpu_vm_bo_update_mapping(adev, gtt, gtt_flags, vm, start, last, flags, addr, fence); @@ -875,7 +875,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, return r; start = last + 1; - addr += max_size; + addr += max_size * AMDGPU_GPU_PAGE_SIZE; } return 0; From b5a5ec55041bf129b767e7a2d282887226f4fae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 8 Mar 2016 17:47:46 +0100 Subject: [PATCH 46/50] drm/amdgpu: reserve the PD during unmap and remove MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We not only need to protect the mapping tree and freed list itself, but also the items on those list. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 34 +++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index cb27754ec07f..7f6b4d9841e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -140,25 +140,40 @@ int amdgpu_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_pri void amdgpu_gem_object_close(struct drm_gem_object *obj, struct drm_file *file_priv) { - struct amdgpu_bo *rbo = gem_to_amdgpu_bo(obj); - struct amdgpu_device *adev = rbo->adev; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = bo->adev; struct amdgpu_fpriv *fpriv = file_priv->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; + + struct amdgpu_bo_list_entry vm_pd; + struct list_head list, duplicates; + struct ttm_validate_buffer tv; + struct ww_acquire_ctx ticket; struct amdgpu_bo_va *bo_va; int r; - r = amdgpu_bo_reserve(rbo, true); + + INIT_LIST_HEAD(&list); + INIT_LIST_HEAD(&duplicates); + + tv.bo = &bo->tbo; + tv.shared = true; + list_add(&tv.head, &list); + + amdgpu_vm_get_pd_bo(vm, &list, &vm_pd); + + r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates); if (r) { dev_err(adev->dev, "leaking bo va because " "we fail to reserve bo (%d)\n", r); return; } - bo_va = amdgpu_vm_bo_find(vm, rbo); + bo_va = amdgpu_vm_bo_find(vm, bo); if (bo_va) { if (--bo_va->ref_count == 0) { amdgpu_vm_bo_rmv(adev, bo_va); } } - amdgpu_bo_unreserve(rbo); + ttm_eu_backoff_reservation(&ticket, &list); } static int amdgpu_gem_handle_lockup(struct amdgpu_device *adev, int r) @@ -580,11 +595,10 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, tv.shared = true; list_add(&tv.head, &list); - if (args->operation == AMDGPU_VA_OP_MAP) { - tv_pd.bo = &fpriv->vm.page_directory->tbo; - tv_pd.shared = true; - list_add(&tv_pd.head, &list); - } + tv_pd.bo = &fpriv->vm.page_directory->tbo; + tv_pd.shared = true; + list_add(&tv_pd.head, &list); + r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates); if (r) { drm_gem_object_unreference_unlocked(gobj); From e17841b97587adfbe96f48c488fd0873ddfcaff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 8 Mar 2016 17:52:01 +0100 Subject: [PATCH 47/50] drm/amdgpu: Revert "add spin lock to protect freed list in vm (v3)" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not needed any more because we need to protect the elements on the list anyway. This reverts commit dae6ecf9e6c9b677e577826c3ac665c6dd9c490b. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 330e307c36ed..7e73e54eadb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -971,22 +971,18 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping; int r; - spin_lock(&vm->freed_lock); while (!list_empty(&vm->freed)) { mapping = list_first_entry(&vm->freed, struct amdgpu_bo_va_mapping, list); list_del(&mapping->list); - spin_unlock(&vm->freed_lock); + r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, vm, mapping, 0, NULL); kfree(mapping); if (r) return r; - spin_lock(&vm->freed_lock); } - spin_unlock(&vm->freed_lock); - return 0; } @@ -1252,13 +1248,10 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, spin_unlock(&vm->it_lock); trace_amdgpu_vm_bo_unmap(bo_va, mapping); - if (valid) { - spin_lock(&vm->freed_lock); + if (valid) list_add(&mapping->list, &vm->freed); - spin_unlock(&vm->freed_lock); - } else { + else kfree(mapping); - } return 0; } @@ -1291,9 +1284,7 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, interval_tree_remove(&mapping->it, &vm->va); spin_unlock(&vm->it_lock); trace_amdgpu_vm_bo_unmap(bo_va, mapping); - spin_lock(&vm->freed_lock); list_add(&mapping->list, &vm->freed); - spin_unlock(&vm->freed_lock); } list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { list_del(&mapping->list); @@ -1357,7 +1348,6 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) INIT_LIST_HEAD(&vm->cleared); INIT_LIST_HEAD(&vm->freed); spin_lock_init(&vm->it_lock); - spin_lock_init(&vm->freed_lock); pd_size = amdgpu_vm_directory_size(adev); pd_entries = amdgpu_vm_num_pdes(adev); From 20250215821140801369b84f8cbe79a459a82ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 8 Mar 2016 17:58:35 +0100 Subject: [PATCH 48/50] drm/amdgpu: Revert "add lock for interval tree in vm" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not needed any more because we need to protect the elements on the list anyway. This reverts commit fe237ed7efec8ac147a4572fdf81173a7f8ddda7. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 16 ++-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index e32ab13227d4..28b4088b2530 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -888,7 +888,6 @@ struct amdgpu_vm_id { struct amdgpu_vm { /* tree of virtual addresses mapped */ - spinlock_t it_lock; struct rb_root va; /* protecting invalidated */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 7e73e54eadb9..0e6d0d1f4041 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1107,9 +1107,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, saddr /= AMDGPU_GPU_PAGE_SIZE; eaddr /= AMDGPU_GPU_PAGE_SIZE; - spin_lock(&vm->it_lock); it = interval_tree_iter_first(&vm->va, saddr, eaddr); - spin_unlock(&vm->it_lock); if (it) { struct amdgpu_bo_va_mapping *tmp; tmp = container_of(it, struct amdgpu_bo_va_mapping, it); @@ -1136,10 +1134,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, mutex_lock(&bo_va->mutex); list_add(&mapping->list, &bo_va->invalids); mutex_unlock(&bo_va->mutex); - spin_lock(&vm->it_lock); interval_tree_insert(&mapping->it, &vm->va); - spin_unlock(&vm->it_lock); - trace_amdgpu_vm_bo_map(bo_va, mapping); /* Make sure the page tables are allocated */ saddr >>= amdgpu_vm_block_size; @@ -1191,9 +1186,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, error_free: list_del(&mapping->list); - spin_lock(&vm->it_lock); interval_tree_remove(&mapping->it, &vm->va); - spin_unlock(&vm->it_lock); trace_amdgpu_vm_bo_unmap(bo_va, mapping); kfree(mapping); @@ -1243,9 +1236,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, } mutex_unlock(&bo_va->mutex); list_del(&mapping->list); - spin_lock(&vm->it_lock); interval_tree_remove(&mapping->it, &vm->va); - spin_unlock(&vm->it_lock); trace_amdgpu_vm_bo_unmap(bo_va, mapping); if (valid) @@ -1280,17 +1271,13 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_del(&mapping->list); - spin_lock(&vm->it_lock); interval_tree_remove(&mapping->it, &vm->va); - spin_unlock(&vm->it_lock); trace_amdgpu_vm_bo_unmap(bo_va, mapping); list_add(&mapping->list, &vm->freed); } list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { list_del(&mapping->list); - spin_lock(&vm->it_lock); interval_tree_remove(&mapping->it, &vm->va); - spin_unlock(&vm->it_lock); kfree(mapping); } fence_put(bo_va->last_pt_update); @@ -1347,7 +1334,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) INIT_LIST_HEAD(&vm->invalidated); INIT_LIST_HEAD(&vm->cleared); INIT_LIST_HEAD(&vm->freed); - spin_lock_init(&vm->it_lock); + pd_size = amdgpu_vm_directory_size(adev); pd_entries = amdgpu_vm_num_pdes(adev); @@ -1434,6 +1421,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_bo_unref(&vm->page_directory); fence_put(vm->page_directory_fence); + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_vm_id *id = &vm->ids[i]; From 32b41ac21fde8f7cea465d74c570fc7bd0089163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 8 Mar 2016 18:03:27 +0100 Subject: [PATCH 49/50] drm/amdgpu: Revert "add mutex for ba_va->valids/invalids" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not needed any more because we need to protect the elements on the list anyway. This reverts commit 38bf516c75b4ef0f5c716e05fa9baab7c52d6c39. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 17 ++++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 28b4088b2530..9a03d566bf6d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -484,7 +484,6 @@ struct amdgpu_bo_va_mapping { /* bo virtual addresses in a specific vm */ struct amdgpu_bo_va { - struct mutex mutex; /* protected by bo being reserved */ struct list_head bo_list; struct fence *last_pt_update; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 0e6d0d1f4041..b6c011b83641 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1009,9 +1009,8 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, vm_status); spin_unlock(&vm->status_lock); - mutex_lock(&bo_va->mutex); + r = amdgpu_vm_bo_update(adev, bo_va, NULL); - mutex_unlock(&bo_va->mutex); if (r) return r; @@ -1055,7 +1054,7 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, INIT_LIST_HEAD(&bo_va->valids); INIT_LIST_HEAD(&bo_va->invalids); INIT_LIST_HEAD(&bo_va->vm_status); - mutex_init(&bo_va->mutex); + list_add_tail(&bo_va->bo_list, &bo->va); return bo_va; @@ -1131,9 +1130,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, mapping->offset = offset; mapping->flags = flags; - mutex_lock(&bo_va->mutex); list_add(&mapping->list, &bo_va->invalids); - mutex_unlock(&bo_va->mutex); interval_tree_insert(&mapping->it, &vm->va); /* Make sure the page tables are allocated */ @@ -1215,7 +1212,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, bool valid = true; saddr /= AMDGPU_GPU_PAGE_SIZE; - mutex_lock(&bo_va->mutex); + list_for_each_entry(mapping, &bo_va->valids, list) { if (mapping->it.start == saddr) break; @@ -1229,12 +1226,10 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, break; } - if (&mapping->list == &bo_va->invalids) { - mutex_unlock(&bo_va->mutex); + if (&mapping->list == &bo_va->invalids) return -ENOENT; - } } - mutex_unlock(&bo_va->mutex); + list_del(&mapping->list); interval_tree_remove(&mapping->it, &vm->va); trace_amdgpu_vm_bo_unmap(bo_va, mapping); @@ -1280,8 +1275,8 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, interval_tree_remove(&mapping->it, &vm->va); kfree(mapping); } + fence_put(bo_va->last_pt_update); - mutex_destroy(&bo_va->mutex); kfree(bo_va); } From 00b7c4ff7d482d287a591f047e0963d494569b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 8 Mar 2016 14:11:00 +0100 Subject: [PATCH 50/50] drm/amdgpu: split pipeline sync out of SDMA vm_flush() as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code it similar to how we did it for the gfx and compute engines. Signed-off-by: Christian König Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 27 ++++++++++++++++++-------- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 23 ++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 23 ++++++++++++++++------ 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index b5b4220a6141..8663a27018a4 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -822,19 +822,14 @@ static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) } /** - * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA + * cik_sdma_ring_emit_pipeline_sync - sync the pipeline * * @ring: amdgpu_ring pointer - * @vm: amdgpu_vm pointer * - * Update the page table base and flush the VM TLB - * using sDMA (CIK). + * Make sure all previous operations are completed (CIK). */ -static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr) +static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { - u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | - SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -848,7 +843,22 @@ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, seq); /* reference */ amdgpu_ring_write(ring, 0xfffffff); /* mask */ amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */ +} +/** + * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA + * + * @ring: amdgpu_ring pointer + * @vm: amdgpu_vm pointer + * + * Update the page table base and flush the VM TLB + * using sDMA (CIK). + */ +static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ + u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) | + SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */ amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); if (vm_id < 8) { @@ -1290,6 +1300,7 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = { .parse_cs = NULL, .emit_ib = cik_sdma_ring_emit_ib, .emit_fence = cik_sdma_ring_emit_fence, + .emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync, .emit_vm_flush = cik_sdma_ring_emit_vm_flush, .emit_hdp_flush = cik_sdma_ring_emit_hdp_flush, .emit_hdp_invalidate = cik_sdma_ring_emit_hdp_invalidate, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 747ef558033a..ab9ff89a3096 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -880,16 +880,13 @@ static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib } /** - * sdma_v2_4_ring_emit_vm_flush - cik vm flush using sDMA + * sdma_v2_4_ring_emit_pipeline_sync - sync the pipeline * * @ring: amdgpu_ring pointer - * @vm: amdgpu_vm pointer * - * Update the page table base and flush the VM TLB - * using sDMA (VI). + * Make sure all previous operations are completed (CIK). */ -static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr) +static void sdma_v2_4_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -905,7 +902,20 @@ static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, 0xfffffff); /* mask */ amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ +} +/** + * sdma_v2_4_ring_emit_vm_flush - cik vm flush using sDMA + * + * @ring: amdgpu_ring pointer + * @vm: amdgpu_vm pointer + * + * Update the page table base and flush the VM TLB + * using sDMA (VI). + */ +static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); if (vm_id < 8) { @@ -1295,6 +1305,7 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = { .parse_cs = NULL, .emit_ib = sdma_v2_4_ring_emit_ib, .emit_fence = sdma_v2_4_ring_emit_fence, + .emit_pipeline_sync = sdma_v2_4_ring_emit_pipeline_sync, .emit_vm_flush = sdma_v2_4_ring_emit_vm_flush, .emit_hdp_flush = sdma_v2_4_ring_emit_hdp_flush, .emit_hdp_invalidate = sdma_v2_4_ring_emit_hdp_invalidate, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 11a544fbf197..4c24c371fec7 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -1031,16 +1031,13 @@ static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib } /** - * sdma_v3_0_ring_emit_vm_flush - cik vm flush using sDMA + * sdma_v3_0_ring_emit_pipeline_sync - sync the pipeline * * @ring: amdgpu_ring pointer - * @vm: amdgpu_vm pointer * - * Update the page table base and flush the VM TLB - * using sDMA (VI). + * Make sure all previous operations are completed (CIK). */ -static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr) +static void sdma_v3_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -1056,7 +1053,20 @@ static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, 0xfffffff); /* mask */ amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */ +} +/** + * sdma_v3_0_ring_emit_vm_flush - cik vm flush using sDMA + * + * @ring: amdgpu_ring pointer + * @vm: amdgpu_vm pointer + * + * Update the page table base and flush the VM TLB + * using sDMA (VI). + */ +static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vm_id, uint64_t pd_addr) +{ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) | SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); if (vm_id < 8) { @@ -1563,6 +1573,7 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = { .parse_cs = NULL, .emit_ib = sdma_v3_0_ring_emit_ib, .emit_fence = sdma_v3_0_ring_emit_fence, + .emit_pipeline_sync = sdma_v3_0_ring_emit_pipeline_sync, .emit_vm_flush = sdma_v3_0_ring_emit_vm_flush, .emit_hdp_flush = sdma_v3_0_ring_emit_hdp_flush, .emit_hdp_invalidate = sdma_v3_0_ring_emit_hdp_invalidate,