aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2019-04-23 20:12:34 -0400
committerDave Airlie <airlied@redhat.com>2019-04-23 20:12:50 -0400
commit8d8f6f704495b1135ef77b7860d833fda97ea841 (patch)
tree459c4abf0d5e908f8107caa0a9e174d274e62bb3 /drivers
parentb1c4f7feada5a5cf4e13db1631fb4784b1ddcb31 (diff)
parentdebcd8f954be2b1f643e76b2400bc7c3d12b4594 (diff)
Merge tag 'drm-misc-next-2019-04-18' of git://anongit.freedesktop.org/drm/drm-misc into drm-next
drm-misc-next for v5.2: UAPI Changes: - Document which feature flags belong to which command in virtio_gpu.h - Make the FB_DAMAGE_CLIPS available for atomic userspace only, it's useless for legacy. Cross-subsystem Changes: - Add device tree bindings for lg,acx467akm-7 panel and ST-Ericsson Multi Channel Display Engine MCDE - Add parameters to the device tree bindings for tfp410 - iommu/io-pgtable: Add ARM Mali midgard MMU page table format - dma-buf: Only do a 64-bits seqno compare when driver explicitly asks for it, else wraparound. - Use the 64-bits compare for dma-fence-chains Core Changes: - Make the fb conversion functions use __iomem dst. - Rename drm_client_add to drm_client_register - Move intel_fb_initial_config to core. - Add a drm_gem_objects_lookup helper - Add drm_gem_fence_array helpers, and use it in lima. - Add drm_format_helper.c to kerneldoc. Driver Changes: - Add panfrost driver for mali midgard/bitfrost. - Converts bochs to use the simple display type. - Small fixes to sun4i, tinydrm, ti-fp410. - Fid aspeed's Kconfig options. - Make some symbols/functions static in lima, sun4i and meson. - Add a driver for the lg,acx467akm-7 panel. Signed-off-by: Dave Airlie <airlied@redhat.com> From: Maarten Lankhorst <maarten.lankhorst@linux.intel.com> Link: https://patchwork.freedesktop.org/patch/msgid/737ad994-213d-45b5-207a-b99d795acd21@linux.intel.com
Diffstat (limited to 'drivers')
-rw-r--r--drivers/dma-buf/dma-fence-chain.c3
-rw-r--r--drivers/dma-buf/sw_sync.c2
-rw-r--r--drivers/dma-buf/sync_file.c3
-rw-r--r--drivers/gpu/drm/Kconfig2
-rw-r--r--drivers/gpu/drm/Makefile1
-rw-r--r--drivers/gpu/drm/aspeed/Kconfig6
-rw-r--r--drivers/gpu/drm/bochs/bochs.h6
-rw-r--r--drivers/gpu/drm/bochs/bochs_kms.c176
-rw-r--r--drivers/gpu/drm/bridge/ti-tfp410.c31
-rw-r--r--drivers/gpu/drm/cirrus/cirrus.c6
-rw-r--r--drivers/gpu/drm/drm_client.c11
-rw-r--r--drivers/gpu/drm/drm_fb_helper.c196
-rw-r--r--drivers/gpu/drm/drm_format_helper.c234
-rw-r--r--drivers/gpu/drm/drm_gem.c186
-rw-r--r--drivers/gpu/drm/drm_mode_config.c5
-rw-r--r--drivers/gpu/drm/i915/intel_fbdev.c218
-rw-r--r--drivers/gpu/drm/lima/lima_gem.c37
-rw-r--r--drivers/gpu/drm/lima/lima_sched.c68
-rw-r--r--drivers/gpu/drm/lima/lima_sched.h6
-rw-r--r--drivers/gpu/drm/meson/meson_drv.c14
-rw-r--r--drivers/gpu/drm/meson/meson_viu.c16
-rw-r--r--drivers/gpu/drm/panel/panel-simple.c31
-rw-r--r--drivers/gpu/drm/panfrost/Kconfig14
-rw-r--r--drivers/gpu/drm/panfrost/Makefile12
-rw-r--r--drivers/gpu/drm/panfrost/TODO27
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_devfreq.c218
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_devfreq.h14
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_device.c252
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_device.h124
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_drv.c463
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_features.h309
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gem.c95
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gem.h29
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gpu.c362
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_gpu.h19
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_issues.h176
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_job.c560
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_job.h51
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_mmu.c386
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_mmu.h17
-rw-r--r--drivers/gpu/drm/panfrost/panfrost_regs.h298
-rw-r--r--drivers/gpu/drm/sun4i/sun4i_backend.c28
-rw-r--r--drivers/gpu/drm/sun4i/sun4i_tcon.c4
-rw-r--r--drivers/gpu/drm/sun4i/sun8i_tcon_top.c4
-rw-r--r--drivers/gpu/drm/tinydrm/hx8357d.c2
-rw-r--r--drivers/gpu/drm/tinydrm/ili9225.c2
-rw-r--r--drivers/gpu/drm/tinydrm/ili9341.c2
-rw-r--r--drivers/gpu/drm/tinydrm/mi0283qt.c2
-rw-r--r--drivers/gpu/drm/tinydrm/repaper.c2
-rw-r--r--drivers/gpu/drm/tinydrm/st7586.c2
-rw-r--r--drivers/gpu/drm/tinydrm/st7735r.c2
-rw-r--r--drivers/iommu/io-pgtable-arm.c91
-rw-r--r--drivers/iommu/io-pgtable.c1
53 files changed, 4183 insertions, 643 deletions
diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c
index c729f98a7bd3..93c42078cb57 100644
--- a/drivers/dma-buf/dma-fence-chain.c
+++ b/drivers/dma-buf/dma-fence-chain.c
@@ -193,6 +193,7 @@ static void dma_fence_chain_release(struct dma_fence *fence)
193} 193}
194 194
195const struct dma_fence_ops dma_fence_chain_ops = { 195const struct dma_fence_ops dma_fence_chain_ops = {
196 .use_64bit_seqno = true,
196 .get_driver_name = dma_fence_chain_get_driver_name, 197 .get_driver_name = dma_fence_chain_get_driver_name,
197 .get_timeline_name = dma_fence_chain_get_timeline_name, 198 .get_timeline_name = dma_fence_chain_get_timeline_name,
198 .enable_signaling = dma_fence_chain_enable_signaling, 199 .enable_signaling = dma_fence_chain_enable_signaling,
@@ -225,7 +226,7 @@ void dma_fence_chain_init(struct dma_fence_chain *chain,
225 init_irq_work(&chain->work, dma_fence_chain_irq_work); 226 init_irq_work(&chain->work, dma_fence_chain_irq_work);
226 227
227 /* Try to reuse the context of the previous chain node. */ 228 /* Try to reuse the context of the previous chain node. */
228 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno)) { 229 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
229 context = prev->context; 230 context = prev->context;
230 chain->prev_seqno = prev->seqno; 231 chain->prev_seqno = prev->seqno;
231 } else { 232 } else {
diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 32dcf7b4c935..119b2ffbc2c9 100644
--- a/drivers/dma-buf/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -161,7 +161,7 @@ static bool timeline_fence_signaled(struct dma_fence *fence)
161{ 161{
162 struct sync_timeline *parent = dma_fence_parent(fence); 162 struct sync_timeline *parent = dma_fence_parent(fence);
163 163
164 return !__dma_fence_is_later(fence->seqno, parent->value); 164 return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops);
165} 165}
166 166
167static bool timeline_fence_enable_signaling(struct dma_fence *fence) 167static bool timeline_fence_enable_signaling(struct dma_fence *fence)
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 4f6305ca52c8..ed3fb6e5224c 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -258,7 +258,8 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
258 258
259 i_b++; 259 i_b++;
260 } else { 260 } else {
261 if (__dma_fence_is_later(pt_a->seqno, pt_b->seqno)) 261 if (__dma_fence_is_later(pt_a->seqno, pt_b->seqno,
262 pt_a->ops))
262 add_fence(fences, &i, pt_a); 263 add_fence(fences, &i, pt_a);
263 else 264 else
264 add_fence(fences, &i, pt_b); 265 add_fence(fences, &i, pt_b);
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index bcbc4234893a..39d5f7562f1c 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -337,6 +337,8 @@ source "drivers/gpu/drm/vboxvideo/Kconfig"
337 337
338source "drivers/gpu/drm/lima/Kconfig" 338source "drivers/gpu/drm/lima/Kconfig"
339 339
340source "drivers/gpu/drm/panfrost/Kconfig"
341
340source "drivers/gpu/drm/aspeed/Kconfig" 342source "drivers/gpu/drm/aspeed/Kconfig"
341 343
342# Keep legacy drivers last 344# Keep legacy drivers last
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 7ebae3d45505..3d0c75cd687c 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -112,4 +112,5 @@ obj-$(CONFIG_DRM_TVE200) += tve200/
112obj-$(CONFIG_DRM_XEN) += xen/ 112obj-$(CONFIG_DRM_XEN) += xen/
113obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ 113obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/
114obj-$(CONFIG_DRM_LIMA) += lima/ 114obj-$(CONFIG_DRM_LIMA) += lima/
115obj-$(CONFIG_DRM_PANFROST) += panfrost/
115obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ 116obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/
diff --git a/drivers/gpu/drm/aspeed/Kconfig b/drivers/gpu/drm/aspeed/Kconfig
index 42b74d18a41b..cccab520e02f 100644
--- a/drivers/gpu/drm/aspeed/Kconfig
+++ b/drivers/gpu/drm/aspeed/Kconfig
@@ -1,11 +1,11 @@
1config DRM_ASPEED_GFX 1config DRM_ASPEED_GFX
2 tristate "ASPEED BMC Display Controller" 2 tristate "ASPEED BMC Display Controller"
3 depends on DRM && OF 3 depends on DRM && OF
4 depends on (COMPILE_TEST || ARCH_ASPEED)
4 select DRM_KMS_HELPER 5 select DRM_KMS_HELPER
5 select DRM_KMS_CMA_HELPER 6 select DRM_KMS_CMA_HELPER
6 select DRM_PANEL 7 select DMA_CMA if HAVE_DMA_CONTIGUOUS
7 select DMA_CMA 8 select CMA if HAVE_DMA_CONTIGUOUS
8 select CMA
9 select MFD_SYSCON 9 select MFD_SYSCON
10 help 10 help
11 Chose this option if you have an ASPEED AST2500 SOC Display 11 Chose this option if you have an ASPEED AST2500 SOC Display
diff --git a/drivers/gpu/drm/bochs/bochs.h b/drivers/gpu/drm/bochs/bochs.h
index 049d058571d4..341cc9d1bab4 100644
--- a/drivers/gpu/drm/bochs/bochs.h
+++ b/drivers/gpu/drm/bochs/bochs.h
@@ -7,6 +7,7 @@
7#include <drm/drm_crtc_helper.h> 7#include <drm/drm_crtc_helper.h>
8#include <drm/drm_encoder.h> 8#include <drm/drm_encoder.h>
9#include <drm/drm_fb_helper.h> 9#include <drm/drm_fb_helper.h>
10#include <drm/drm_simple_kms_helper.h>
10 11
11#include <drm/drm_gem.h> 12#include <drm/drm_gem.h>
12 13
@@ -69,9 +70,8 @@ struct bochs_device {
69 struct edid *edid; 70 struct edid *edid;
70 71
71 /* drm */ 72 /* drm */
72 struct drm_device *dev; 73 struct drm_device *dev;
73 struct drm_crtc crtc; 74 struct drm_simple_display_pipe pipe;
74 struct drm_encoder encoder;
75 struct drm_connector connector; 75 struct drm_connector connector;
76 76
77 /* ttm */ 77 /* ttm */
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 485f9cf05e8b..5e905f50449d 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -22,76 +22,55 @@ MODULE_PARM_DESC(defy, "default y resolution");
22 22
23/* ---------------------------------------------------------------------- */ 23/* ---------------------------------------------------------------------- */
24 24
25static void bochs_crtc_mode_set_nofb(struct drm_crtc *crtc) 25static const uint32_t bochs_formats[] = {
26 DRM_FORMAT_XRGB8888,
27 DRM_FORMAT_BGRX8888,
28};
29
30static void bochs_plane_update(struct bochs_device *bochs,
31 struct drm_plane_state *state)
26{ 32{
27 struct bochs_device *bochs = 33 struct bochs_bo *bo;
28 container_of(crtc, struct bochs_device, crtc);
29 34
30 bochs_hw_setmode(bochs, &crtc->mode); 35 if (!state->fb || !bochs->stride)
36 return;
37
38 bo = gem_to_bochs_bo(state->fb->obj[0]);
39 bochs_hw_setbase(bochs,
40 state->crtc_x,
41 state->crtc_y,
42 bo->bo.offset);
43 bochs_hw_setformat(bochs, state->fb->format);
31} 44}
32 45
33static void bochs_crtc_atomic_enable(struct drm_crtc *crtc, 46static void bochs_pipe_enable(struct drm_simple_display_pipe *pipe,
34 struct drm_crtc_state *old_crtc_state) 47 struct drm_crtc_state *crtc_state,
48 struct drm_plane_state *plane_state)
35{ 49{
50 struct bochs_device *bochs = pipe->crtc.dev->dev_private;
51
52 bochs_hw_setmode(bochs, &crtc_state->mode);
53 bochs_plane_update(bochs, plane_state);
36} 54}
37 55
38static void bochs_crtc_atomic_flush(struct drm_crtc *crtc, 56static void bochs_pipe_update(struct drm_simple_display_pipe *pipe,
39 struct drm_crtc_state *old_crtc_state) 57 struct drm_plane_state *old_state)
40{ 58{
41 struct drm_device *dev = crtc->dev; 59 struct bochs_device *bochs = pipe->crtc.dev->dev_private;
42 struct drm_pending_vblank_event *event; 60 struct drm_crtc *crtc = &pipe->crtc;
43 61
44 if (crtc->state && crtc->state->event) { 62 bochs_plane_update(bochs, pipe->plane.state);
45 unsigned long irqflags;
46 63
47 spin_lock_irqsave(&dev->event_lock, irqflags); 64 if (crtc->state->event) {
48 event = crtc->state->event; 65 spin_lock_irq(&crtc->dev->event_lock);
66 drm_crtc_send_vblank_event(crtc, crtc->state->event);
49 crtc->state->event = NULL; 67 crtc->state->event = NULL;
50 drm_crtc_send_vblank_event(crtc, event); 68 spin_unlock_irq(&crtc->dev->event_lock);
51 spin_unlock_irqrestore(&dev->event_lock, irqflags);
52 } 69 }
53} 70}
54 71
55 72static int bochs_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
56/* These provide the minimum set of functions required to handle a CRTC */ 73 struct drm_plane_state *new_state)
57static const struct drm_crtc_funcs bochs_crtc_funcs = {
58 .set_config = drm_atomic_helper_set_config,
59 .destroy = drm_crtc_cleanup,
60 .page_flip = drm_atomic_helper_page_flip,
61 .reset = drm_atomic_helper_crtc_reset,
62 .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
63 .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
64};
65
66static const struct drm_crtc_helper_funcs bochs_helper_funcs = {
67 .mode_set_nofb = bochs_crtc_mode_set_nofb,
68 .atomic_enable = bochs_crtc_atomic_enable,
69 .atomic_flush = bochs_crtc_atomic_flush,
70};
71
72static const uint32_t bochs_formats[] = {
73 DRM_FORMAT_XRGB8888,
74 DRM_FORMAT_BGRX8888,
75};
76
77static void bochs_plane_atomic_update(struct drm_plane *plane,
78 struct drm_plane_state *old_state)
79{
80 struct bochs_device *bochs = plane->dev->dev_private;
81 struct bochs_bo *bo;
82
83 if (!plane->state->fb)
84 return;
85 bo = gem_to_bochs_bo(plane->state->fb->obj[0]);
86 bochs_hw_setbase(bochs,
87 plane->state->crtc_x,
88 plane->state->crtc_y,
89 bo->bo.offset);
90 bochs_hw_setformat(bochs, plane->state->fb->format);
91}
92
93static int bochs_plane_prepare_fb(struct drm_plane *plane,
94 struct drm_plane_state *new_state)
95{ 74{
96 struct bochs_bo *bo; 75 struct bochs_bo *bo;
97 76
@@ -101,8 +80,8 @@ static int bochs_plane_prepare_fb(struct drm_plane *plane,
101 return bochs_bo_pin(bo, TTM_PL_FLAG_VRAM); 80 return bochs_bo_pin(bo, TTM_PL_FLAG_VRAM);
102} 81}
103 82
104static void bochs_plane_cleanup_fb(struct drm_plane *plane, 83static void bochs_pipe_cleanup_fb(struct drm_simple_display_pipe *pipe,
105 struct drm_plane_state *old_state) 84 struct drm_plane_state *old_state)
106{ 85{
107 struct bochs_bo *bo; 86 struct bochs_bo *bo;
108 87
@@ -112,73 +91,13 @@ static void bochs_plane_cleanup_fb(struct drm_plane *plane,
112 bochs_bo_unpin(bo); 91 bochs_bo_unpin(bo);
113} 92}
114 93
115static const struct drm_plane_helper_funcs bochs_plane_helper_funcs = { 94static const struct drm_simple_display_pipe_funcs bochs_pipe_funcs = {
116 .atomic_update = bochs_plane_atomic_update, 95 .enable = bochs_pipe_enable,
117 .prepare_fb = bochs_plane_prepare_fb, 96 .update = bochs_pipe_update,
118 .cleanup_fb = bochs_plane_cleanup_fb, 97 .prepare_fb = bochs_pipe_prepare_fb,
119}; 98 .cleanup_fb = bochs_pipe_cleanup_fb,
120
121static const struct drm_plane_funcs bochs_plane_funcs = {
122 .update_plane = drm_atomic_helper_update_plane,
123 .disable_plane = drm_atomic_helper_disable_plane,
124 .destroy = drm_primary_helper_destroy,
125 .reset = drm_atomic_helper_plane_reset,
126 .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
127 .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
128};
129
130static struct drm_plane *bochs_primary_plane(struct drm_device *dev)
131{
132 struct drm_plane *primary;
133 int ret;
134
135 primary = kzalloc(sizeof(*primary), GFP_KERNEL);
136 if (primary == NULL) {
137 DRM_DEBUG_KMS("Failed to allocate primary plane\n");
138 return NULL;
139 }
140
141 ret = drm_universal_plane_init(dev, primary, 0,
142 &bochs_plane_funcs,
143 bochs_formats,
144 ARRAY_SIZE(bochs_formats),
145 NULL,
146 DRM_PLANE_TYPE_PRIMARY, NULL);
147 if (ret) {
148 kfree(primary);
149 return NULL;
150 }
151
152 drm_plane_helper_add(primary, &bochs_plane_helper_funcs);
153 return primary;
154}
155
156static void bochs_crtc_init(struct drm_device *dev)
157{
158 struct bochs_device *bochs = dev->dev_private;
159 struct drm_crtc *crtc = &bochs->crtc;
160 struct drm_plane *primary = bochs_primary_plane(dev);
161
162 drm_crtc_init_with_planes(dev, crtc, primary, NULL,
163 &bochs_crtc_funcs, NULL);
164 drm_crtc_helper_add(crtc, &bochs_helper_funcs);
165}
166
167static const struct drm_encoder_funcs bochs_encoder_encoder_funcs = {
168 .destroy = drm_encoder_cleanup,
169}; 99};
170 100
171static void bochs_encoder_init(struct drm_device *dev)
172{
173 struct bochs_device *bochs = dev->dev_private;
174 struct drm_encoder *encoder = &bochs->encoder;
175
176 encoder->possible_crtcs = 0x1;
177 drm_encoder_init(dev, encoder, &bochs_encoder_encoder_funcs,
178 DRM_MODE_ENCODER_DAC, NULL);
179}
180
181
182static int bochs_connector_get_modes(struct drm_connector *connector) 101static int bochs_connector_get_modes(struct drm_connector *connector)
183{ 102{
184 struct bochs_device *bochs = 103 struct bochs_device *bochs =
@@ -278,11 +197,14 @@ int bochs_kms_init(struct bochs_device *bochs)
278 197
279 bochs->dev->mode_config.funcs = &bochs_mode_funcs; 198 bochs->dev->mode_config.funcs = &bochs_mode_funcs;
280 199
281 bochs_crtc_init(bochs->dev);
282 bochs_encoder_init(bochs->dev);
283 bochs_connector_init(bochs->dev); 200 bochs_connector_init(bochs->dev);
284 drm_connector_attach_encoder(&bochs->connector, 201 drm_simple_display_pipe_init(bochs->dev,
285 &bochs->encoder); 202 &bochs->pipe,
203 &bochs_pipe_funcs,
204 bochs_formats,
205 ARRAY_SIZE(bochs_formats),
206 NULL,
207 &bochs->connector);
286 208
287 drm_mode_config_reset(bochs->dev); 209 drm_mode_config_reset(bochs->dev);
288 210
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index 285be4a0f4bd..8b0e71bd3ca7 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -29,8 +29,10 @@ struct tfp410 {
29 struct drm_connector connector; 29 struct drm_connector connector;
30 unsigned int connector_type; 30 unsigned int connector_type;
31 31
32 u32 bus_format;
32 struct i2c_adapter *ddc; 33 struct i2c_adapter *ddc;
33 struct gpio_desc *hpd; 34 struct gpio_desc *hpd;
35 int hpd_irq;
34 struct delayed_work hpd_work; 36 struct delayed_work hpd_work;
35 struct gpio_desc *powerdown; 37 struct gpio_desc *powerdown;
36 38
@@ -124,8 +126,10 @@ static int tfp410_attach(struct drm_bridge *bridge)
124 return -ENODEV; 126 return -ENODEV;
125 } 127 }
126 128
127 if (dvi->hpd) 129 if (dvi->hpd_irq >= 0)
128 dvi->connector.polled = DRM_CONNECTOR_POLL_HPD; 130 dvi->connector.polled = DRM_CONNECTOR_POLL_HPD;
131 else
132 dvi->connector.polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
129 133
130 drm_connector_helper_add(&dvi->connector, 134 drm_connector_helper_add(&dvi->connector,
131 &tfp410_con_helper_funcs); 135 &tfp410_con_helper_funcs);
@@ -136,6 +140,9 @@ static int tfp410_attach(struct drm_bridge *bridge)
136 return ret; 140 return ret;
137 } 141 }
138 142
143 drm_display_info_set_bus_formats(&dvi->connector.display_info,
144 &dvi->bus_format, 1);
145
139 drm_connector_attach_encoder(&dvi->connector, 146 drm_connector_attach_encoder(&dvi->connector,
140 bridge->encoder); 147 bridge->encoder);
141 148
@@ -194,6 +201,7 @@ static int tfp410_parse_timings(struct tfp410 *dvi, bool i2c)
194 struct drm_bridge_timings *timings = &dvi->timings; 201 struct drm_bridge_timings *timings = &dvi->timings;
195 struct device_node *ep; 202 struct device_node *ep;
196 u32 pclk_sample = 0; 203 u32 pclk_sample = 0;
204 u32 bus_width = 24;
197 s32 deskew = 0; 205 s32 deskew = 0;
198 206
199 /* Start with defaults. */ 207 /* Start with defaults. */
@@ -218,6 +226,7 @@ static int tfp410_parse_timings(struct tfp410 *dvi, bool i2c)
218 226
219 /* Get the sampling edge from the endpoint. */ 227 /* Get the sampling edge from the endpoint. */
220 of_property_read_u32(ep, "pclk-sample", &pclk_sample); 228 of_property_read_u32(ep, "pclk-sample", &pclk_sample);
229 of_property_read_u32(ep, "bus-width", &bus_width);
221 of_node_put(ep); 230 of_node_put(ep);
222 231
223 timings->input_bus_flags = DRM_BUS_FLAG_DE_HIGH; 232 timings->input_bus_flags = DRM_BUS_FLAG_DE_HIGH;
@@ -235,6 +244,17 @@ static int tfp410_parse_timings(struct tfp410 *dvi, bool i2c)
235 return -EINVAL; 244 return -EINVAL;
236 } 245 }
237 246
247 switch (bus_width) {
248 case 12:
249 dvi->bus_format = MEDIA_BUS_FMT_RGB888_2X12_LE;
250 break;
251 case 24:
252 dvi->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
253 break;
254 default:
255 return -EINVAL;
256 }
257
238 /* Get the setup and hold time from vendor-specific properties. */ 258 /* Get the setup and hold time from vendor-specific properties. */
239 of_property_read_u32(dvi->dev->of_node, "ti,deskew", (u32 *)&deskew); 259 of_property_read_u32(dvi->dev->of_node, "ti,deskew", (u32 *)&deskew);
240 if (deskew < -4 || deskew > 3) 260 if (deskew < -4 || deskew > 3)
@@ -324,10 +344,15 @@ static int tfp410_init(struct device *dev, bool i2c)
324 return PTR_ERR(dvi->powerdown); 344 return PTR_ERR(dvi->powerdown);
325 } 345 }
326 346
327 if (dvi->hpd) { 347 if (dvi->hpd)
348 dvi->hpd_irq = gpiod_to_irq(dvi->hpd);
349 else
350 dvi->hpd_irq = -ENXIO;
351
352 if (dvi->hpd_irq >= 0) {
328 INIT_DELAYED_WORK(&dvi->hpd_work, tfp410_hpd_work_func); 353 INIT_DELAYED_WORK(&dvi->hpd_work, tfp410_hpd_work_func);
329 354
330 ret = devm_request_threaded_irq(dev, gpiod_to_irq(dvi->hpd), 355 ret = devm_request_threaded_irq(dev, dvi->hpd_irq,
331 NULL, tfp410_hpd_irq_thread, IRQF_TRIGGER_RISING | 356 NULL, tfp410_hpd_irq_thread, IRQF_TRIGGER_RISING |
332 IRQF_TRIGGER_FALLING | IRQF_ONESHOT, 357 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
333 "hdmi-hpd", dvi); 358 "hdmi-hpd", dvi);
diff --git a/drivers/gpu/drm/cirrus/cirrus.c b/drivers/gpu/drm/cirrus/cirrus.c
index 5095b8ce52c2..be4ea370ba31 100644
--- a/drivers/gpu/drm/cirrus/cirrus.c
+++ b/drivers/gpu/drm/cirrus/cirrus.c
@@ -307,16 +307,16 @@ static int cirrus_fb_blit_rect(struct drm_framebuffer *fb,
307 return -ENOMEM; 307 return -ENOMEM;
308 308
309 if (cirrus->cpp == fb->format->cpp[0]) 309 if (cirrus->cpp == fb->format->cpp[0])
310 drm_fb_memcpy_dstclip(__io_virt(cirrus->vram), 310 drm_fb_memcpy_dstclip(cirrus->vram,
311 vmap, fb, rect); 311 vmap, fb, rect);
312 312
313 else if (fb->format->cpp[0] == 4 && cirrus->cpp == 2) 313 else if (fb->format->cpp[0] == 4 && cirrus->cpp == 2)
314 drm_fb_xrgb8888_to_rgb565_dstclip(__io_virt(cirrus->vram), 314 drm_fb_xrgb8888_to_rgb565_dstclip(cirrus->vram,
315 cirrus->pitch, 315 cirrus->pitch,
316 vmap, fb, rect, false); 316 vmap, fb, rect, false);
317 317
318 else if (fb->format->cpp[0] == 4 && cirrus->cpp == 3) 318 else if (fb->format->cpp[0] == 4 && cirrus->cpp == 3)
319 drm_fb_xrgb8888_to_rgb888_dstclip(__io_virt(cirrus->vram), 319 drm_fb_xrgb8888_to_rgb888_dstclip(cirrus->vram,
320 cirrus->pitch, 320 cirrus->pitch,
321 vmap, fb, rect); 321 vmap, fb, rect);
322 322
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index 9b2bd28dde0a..f20d1dda3961 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -69,7 +69,8 @@ EXPORT_SYMBOL(drm_client_close);
69 * @name: Client name 69 * @name: Client name
70 * @funcs: DRM client functions (optional) 70 * @funcs: DRM client functions (optional)
71 * 71 *
72 * This initialises the client and opens a &drm_file. Use drm_client_add() to complete the process. 72 * This initialises the client and opens a &drm_file.
73 * Use drm_client_register() to complete the process.
73 * The caller needs to hold a reference on @dev before calling this function. 74 * The caller needs to hold a reference on @dev before calling this function.
74 * The client is freed when the &drm_device is unregistered. See drm_client_release(). 75 * The client is freed when the &drm_device is unregistered. See drm_client_release().
75 * 76 *
@@ -108,16 +109,16 @@ err_put_module:
108EXPORT_SYMBOL(drm_client_init); 109EXPORT_SYMBOL(drm_client_init);
109 110
110/** 111/**
111 * drm_client_add - Add client to the device list 112 * drm_client_register - Register client
112 * @client: DRM client 113 * @client: DRM client
113 * 114 *
114 * Add the client to the &drm_device client list to activate its callbacks. 115 * Add the client to the &drm_device client list to activate its callbacks.
115 * @client must be initialized by a call to drm_client_init(). After 116 * @client must be initialized by a call to drm_client_init(). After
116 * drm_client_add() it is no longer permissible to call drm_client_release() 117 * drm_client_register() it is no longer permissible to call drm_client_release()
117 * directly (outside the unregister callback), instead cleanup will happen 118 * directly (outside the unregister callback), instead cleanup will happen
118 * automatically on driver unload. 119 * automatically on driver unload.
119 */ 120 */
120void drm_client_add(struct drm_client_dev *client) 121void drm_client_register(struct drm_client_dev *client)
121{ 122{
122 struct drm_device *dev = client->dev; 123 struct drm_device *dev = client->dev;
123 124
@@ -125,7 +126,7 @@ void drm_client_add(struct drm_client_dev *client)
125 list_add(&client->list, &dev->clientlist); 126 list_add(&client->list, &dev->clientlist);
126 mutex_unlock(&dev->clientlist_mutex); 127 mutex_unlock(&dev->clientlist_mutex);
127} 128}
128EXPORT_SYMBOL(drm_client_add); 129EXPORT_SYMBOL(drm_client_register);
129 130
130/** 131/**
131 * drm_client_release - Release DRM client resources 132 * drm_client_release - Release DRM client resources
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index d1ce7bd04cad..4de4b9d59d49 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -2559,6 +2559,194 @@ static void drm_setup_crtc_rotation(struct drm_fb_helper *fb_helper,
2559 fb_helper->sw_rotations |= DRM_MODE_ROTATE_0; 2559 fb_helper->sw_rotations |= DRM_MODE_ROTATE_0;
2560} 2560}
2561 2561
2562static struct drm_fb_helper_crtc *
2563drm_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
2564{
2565 int i;
2566
2567 for (i = 0; i < fb_helper->crtc_count; i++)
2568 if (fb_helper->crtc_info[i].mode_set.crtc == crtc)
2569 return &fb_helper->crtc_info[i];
2570
2571 return NULL;
2572}
2573
2574/* Try to read the BIOS display configuration and use it for the initial config */
2575static bool drm_fb_helper_firmware_config(struct drm_fb_helper *fb_helper,
2576 struct drm_fb_helper_crtc **crtcs,
2577 struct drm_display_mode **modes,
2578 struct drm_fb_offset *offsets,
2579 bool *enabled, int width, int height)
2580{
2581 struct drm_device *dev = fb_helper->dev;
2582 unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
2583 unsigned long conn_configured, conn_seq;
2584 int i, j;
2585 bool *save_enabled;
2586 bool fallback = true, ret = true;
2587 int num_connectors_enabled = 0;
2588 int num_connectors_detected = 0;
2589 struct drm_modeset_acquire_ctx ctx;
2590
2591 save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
2592 if (!save_enabled)
2593 return false;
2594
2595 drm_modeset_acquire_init(&ctx, 0);
2596
2597 while (drm_modeset_lock_all_ctx(dev, &ctx) != 0)
2598 drm_modeset_backoff(&ctx);
2599
2600 memcpy(save_enabled, enabled, count);
2601 conn_seq = GENMASK(count - 1, 0);
2602 conn_configured = 0;
2603retry:
2604 for (i = 0; i < count; i++) {
2605 struct drm_fb_helper_connector *fb_conn;
2606 struct drm_connector *connector;
2607 struct drm_encoder *encoder;
2608 struct drm_fb_helper_crtc *new_crtc;
2609
2610 fb_conn = fb_helper->connector_info[i];
2611 connector = fb_conn->connector;
2612
2613 if (conn_configured & BIT(i))
2614 continue;
2615
2616 /* First pass, only consider tiled connectors */
2617 if (conn_seq == GENMASK(count - 1, 0) && !connector->has_tile)
2618 continue;
2619
2620 if (connector->status == connector_status_connected)
2621 num_connectors_detected++;
2622
2623 if (!enabled[i]) {
2624 DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
2625 connector->name);
2626 conn_configured |= BIT(i);
2627 continue;
2628 }
2629
2630 if (connector->force == DRM_FORCE_OFF) {
2631 DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
2632 connector->name);
2633 enabled[i] = false;
2634 continue;
2635 }
2636
2637 encoder = connector->state->best_encoder;
2638 if (!encoder || WARN_ON(!connector->state->crtc)) {
2639 if (connector->force > DRM_FORCE_OFF)
2640 goto bail;
2641
2642 DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
2643 connector->name);
2644 enabled[i] = false;
2645 conn_configured |= BIT(i);
2646 continue;
2647 }
2648
2649 num_connectors_enabled++;
2650
2651 new_crtc = drm_fb_helper_crtc(fb_helper, connector->state->crtc);
2652
2653 /*
2654 * Make sure we're not trying to drive multiple connectors
2655 * with a single CRTC, since our cloning support may not
2656 * match the BIOS.
2657 */
2658 for (j = 0; j < count; j++) {
2659 if (crtcs[j] == new_crtc) {
2660 DRM_DEBUG_KMS("fallback: cloned configuration\n");
2661 goto bail;
2662 }
2663 }
2664
2665 DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n",
2666 connector->name);
2667
2668 /* go for command line mode first */
2669 modes[i] = drm_pick_cmdline_mode(fb_conn);
2670
2671 /* try for preferred next */
2672 if (!modes[i]) {
2673 DRM_DEBUG_KMS("looking for preferred mode on connector %s %d\n",
2674 connector->name, connector->has_tile);
2675 modes[i] = drm_has_preferred_mode(fb_conn, width,
2676 height);
2677 }
2678
2679 /* No preferred mode marked by the EDID? Are there any modes? */
2680 if (!modes[i] && !list_empty(&connector->modes)) {
2681 DRM_DEBUG_KMS("using first mode listed on connector %s\n",
2682 connector->name);
2683 modes[i] = list_first_entry(&connector->modes,
2684 struct drm_display_mode,
2685 head);
2686 }
2687
2688 /* last resort: use current mode */
2689 if (!modes[i]) {
2690 /*
2691 * IMPORTANT: We want to use the adjusted mode (i.e.
2692 * after the panel fitter upscaling) as the initial
2693 * config, not the input mode, which is what crtc->mode
2694 * usually contains. But since our current
2695 * code puts a mode derived from the post-pfit timings
2696 * into crtc->mode this works out correctly.
2697 *
2698 * This is crtc->mode and not crtc->state->mode for the
2699 * fastboot check to work correctly.
2700 */
2701 DRM_DEBUG_KMS("looking for current mode on connector %s\n",
2702 connector->name);
2703 modes[i] = &connector->state->crtc->mode;
2704 }
2705 crtcs[i] = new_crtc;
2706
2707 DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
2708 connector->name,
2709 connector->state->crtc->base.id,
2710 connector->state->crtc->name,
2711 modes[i]->hdisplay, modes[i]->vdisplay,
2712 modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" : "");
2713
2714 fallback = false;
2715 conn_configured |= BIT(i);
2716 }
2717
2718 if (conn_configured != conn_seq) { /* repeat until no more are found */
2719 conn_seq = conn_configured;
2720 goto retry;
2721 }
2722
2723 /*
2724 * If the BIOS didn't enable everything it could, fall back to have the
2725 * same user experiencing of lighting up as much as possible like the
2726 * fbdev helper library.
2727 */
2728 if (num_connectors_enabled != num_connectors_detected &&
2729 num_connectors_enabled < dev->mode_config.num_crtc) {
2730 DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
2731 DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
2732 num_connectors_detected);
2733 fallback = true;
2734 }
2735
2736 if (fallback) {
2737bail:
2738 DRM_DEBUG_KMS("Not using firmware configuration\n");
2739 memcpy(enabled, save_enabled, count);
2740 ret = false;
2741 }
2742
2743 drm_modeset_drop_locks(&ctx);
2744 drm_modeset_acquire_fini(&ctx);
2745
2746 kfree(save_enabled);
2747 return ret;
2748}
2749
2562static void drm_setup_crtcs(struct drm_fb_helper *fb_helper, 2750static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
2563 u32 width, u32 height) 2751 u32 width, u32 height)
2564{ 2752{
@@ -2591,10 +2779,8 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
2591 DRM_DEBUG_KMS("No connectors reported connected with modes\n"); 2779 DRM_DEBUG_KMS("No connectors reported connected with modes\n");
2592 drm_enable_connectors(fb_helper, enabled); 2780 drm_enable_connectors(fb_helper, enabled);
2593 2781
2594 if (!(fb_helper->funcs->initial_config && 2782 if (!drm_fb_helper_firmware_config(fb_helper, crtcs, modes, offsets,
2595 fb_helper->funcs->initial_config(fb_helper, crtcs, modes, 2783 enabled, width, height)) {
2596 offsets,
2597 enabled, width, height))) {
2598 memset(modes, 0, fb_helper->connector_count*sizeof(modes[0])); 2784 memset(modes, 0, fb_helper->connector_count*sizeof(modes[0]));
2599 memset(crtcs, 0, fb_helper->connector_count*sizeof(crtcs[0])); 2785 memset(crtcs, 0, fb_helper->connector_count*sizeof(crtcs[0]));
2600 memset(offsets, 0, fb_helper->connector_count*sizeof(offsets[0])); 2786 memset(offsets, 0, fb_helper->connector_count*sizeof(offsets[0]));
@@ -3322,7 +3508,7 @@ int drm_fbdev_generic_setup(struct drm_device *dev, unsigned int preferred_bpp)
3322 if (ret) 3508 if (ret)
3323 DRM_DEV_DEBUG(dev->dev, "client hotplug ret=%d\n", ret); 3509 DRM_DEV_DEBUG(dev->dev, "client hotplug ret=%d\n", ret);
3324 3510
3325 drm_client_add(&fb_helper->client); 3511 drm_client_register(&fb_helper->client);
3326 3512
3327 return 0; 3513 return 0;
3328} 3514}
diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c
index 00d716f14173..a18da35145b7 100644
--- a/drivers/gpu/drm/drm_format_helper.c
+++ b/drivers/gpu/drm/drm_format_helper.c
@@ -10,23 +10,17 @@
10 10
11#include <linux/module.h> 11#include <linux/module.h>
12#include <linux/slab.h> 12#include <linux/slab.h>
13#include <linux/io.h>
13 14
14#include <drm/drm_format_helper.h> 15#include <drm/drm_format_helper.h>
15#include <drm/drm_framebuffer.h> 16#include <drm/drm_framebuffer.h>
16#include <drm/drm_fourcc.h> 17#include <drm/drm_fourcc.h>
17#include <drm/drm_rect.h> 18#include <drm/drm_rect.h>
18 19
19static void drm_fb_memcpy_lines(void *dst, unsigned int dst_pitch, 20static unsigned int clip_offset(struct drm_rect *clip,
20 void *src, unsigned int src_pitch, 21 unsigned int pitch, unsigned int cpp)
21 unsigned int linelength, unsigned int lines)
22{ 22{
23 int line; 23 return clip->y1 * pitch + clip->x1 * cpp;
24
25 for (line = 0; line < lines; line++) {
26 memcpy(dst, src, linelength);
27 src += src_pitch;
28 dst += dst_pitch;
29 }
30} 24}
31 25
32/** 26/**
@@ -43,35 +37,44 @@ void drm_fb_memcpy(void *dst, void *vaddr, struct drm_framebuffer *fb,
43 struct drm_rect *clip) 37 struct drm_rect *clip)
44{ 38{
45 unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0); 39 unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0);
46 unsigned int offset = (clip->y1 * fb->pitches[0]) + (clip->x1 * cpp);
47 size_t len = (clip->x2 - clip->x1) * cpp; 40 size_t len = (clip->x2 - clip->x1) * cpp;
41 unsigned int y, lines = clip->y2 - clip->y1;
48 42
49 drm_fb_memcpy_lines(dst, len, 43 vaddr += clip_offset(clip, fb->pitches[0], cpp);
50 vaddr + offset, fb->pitches[0], 44 for (y = 0; y < lines; y++) {
51 len, clip->y2 - clip->y1); 45 memcpy(dst, vaddr, len);
46 vaddr += fb->pitches[0];
47 dst += len;
48 }
52} 49}
53EXPORT_SYMBOL(drm_fb_memcpy); 50EXPORT_SYMBOL(drm_fb_memcpy);
54 51
55/** 52/**
56 * drm_fb_memcpy_dstclip - Copy clip buffer 53 * drm_fb_memcpy_dstclip - Copy clip buffer
57 * @dst: Destination buffer 54 * @dst: Destination buffer (iomem)
58 * @vaddr: Source buffer 55 * @vaddr: Source buffer
59 * @fb: DRM framebuffer 56 * @fb: DRM framebuffer
60 * @clip: Clip rectangle area to copy 57 * @clip: Clip rectangle area to copy
61 * 58 *
62 * This function applies clipping on dst, i.e. the destination is a 59 * This function applies clipping on dst, i.e. the destination is a
63 * full framebuffer but only the clip rect content is copied over. 60 * full (iomem) framebuffer but only the clip rect content is copied over.
64 */ 61 */
65void drm_fb_memcpy_dstclip(void *dst, void *vaddr, struct drm_framebuffer *fb, 62void drm_fb_memcpy_dstclip(void __iomem *dst, void *vaddr,
63 struct drm_framebuffer *fb,
66 struct drm_rect *clip) 64 struct drm_rect *clip)
67{ 65{
68 unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0); 66 unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0);
69 unsigned int offset = (clip->y1 * fb->pitches[0]) + (clip->x1 * cpp); 67 unsigned int offset = clip_offset(clip, fb->pitches[0], cpp);
70 size_t len = (clip->x2 - clip->x1) * cpp; 68 size_t len = (clip->x2 - clip->x1) * cpp;
69 unsigned int y, lines = clip->y2 - clip->y1;
71 70
72 drm_fb_memcpy_lines(dst + offset, fb->pitches[0], 71 vaddr += offset;
73 vaddr + offset, fb->pitches[0], 72 dst += offset;
74 len, clip->y2 - clip->y1); 73 for (y = 0; y < lines; y++) {
74 memcpy_toio(dst, vaddr, len);
75 vaddr += fb->pitches[0];
76 dst += fb->pitches[0];
77 }
75} 78}
76EXPORT_SYMBOL(drm_fb_memcpy_dstclip); 79EXPORT_SYMBOL(drm_fb_memcpy_dstclip);
77 80
@@ -110,42 +113,22 @@ void drm_fb_swab16(u16 *dst, void *vaddr, struct drm_framebuffer *fb,
110} 113}
111EXPORT_SYMBOL(drm_fb_swab16); 114EXPORT_SYMBOL(drm_fb_swab16);
112 115
113static void drm_fb_xrgb8888_to_rgb565_lines(void *dst, unsigned int dst_pitch, 116static void drm_fb_xrgb8888_to_rgb565_line(u16 *dbuf, u32 *sbuf,
114 void *src, unsigned int src_pitch, 117 unsigned int pixels,
115 unsigned int src_linelength, 118 bool swab)
116 unsigned int lines,
117 bool swap)
118{ 119{
119 unsigned int linepixels = src_linelength / sizeof(u32); 120 unsigned int x;
120 unsigned int x, y; 121 u16 val16;
121 u32 *sbuf; 122
122 u16 *dbuf, val16; 123 for (x = 0; x < pixels; x++) {
123 124 val16 = ((sbuf[x] & 0x00F80000) >> 8) |
124 /* 125 ((sbuf[x] & 0x0000FC00) >> 5) |
125 * The cma memory is write-combined so reads are uncached. 126 ((sbuf[x] & 0x000000F8) >> 3);
126 * Speed up by fetching one line at a time. 127 if (swab)
127 */ 128 dbuf[x] = swab16(val16);
128 sbuf = kmalloc(src_linelength, GFP_KERNEL); 129 else
129 if (!sbuf) 130 dbuf[x] = val16;
130 return;
131
132 for (y = 0; y < lines; y++) {
133 memcpy(sbuf, src, src_linelength);
134 dbuf = dst;
135 for (x = 0; x < linepixels; x++) {
136 val16 = ((sbuf[x] & 0x00F80000) >> 8) |
137 ((sbuf[x] & 0x0000FC00) >> 5) |
138 ((sbuf[x] & 0x000000F8) >> 3);
139 if (swap)
140 *dbuf++ = swab16(val16);
141 else
142 *dbuf++ = val16;
143 }
144 src += src_pitch;
145 dst += dst_pitch;
146 } 131 }
147
148 kfree(sbuf);
149} 132}
150 133
151/** 134/**
@@ -154,7 +137,7 @@ static void drm_fb_xrgb8888_to_rgb565_lines(void *dst, unsigned int dst_pitch,
154 * @vaddr: XRGB8888 source buffer 137 * @vaddr: XRGB8888 source buffer
155 * @fb: DRM framebuffer 138 * @fb: DRM framebuffer
156 * @clip: Clip rectangle area to copy 139 * @clip: Clip rectangle area to copy
157 * @swap: Swap bytes 140 * @swab: Swap bytes
158 * 141 *
159 * Drivers can use this function for RGB565 devices that don't natively 142 * Drivers can use this function for RGB565 devices that don't natively
160 * support XRGB8888. 143 * support XRGB8888.
@@ -164,109 +147,124 @@ static void drm_fb_xrgb8888_to_rgb565_lines(void *dst, unsigned int dst_pitch,
164 */ 147 */
165void drm_fb_xrgb8888_to_rgb565(void *dst, void *vaddr, 148void drm_fb_xrgb8888_to_rgb565(void *dst, void *vaddr,
166 struct drm_framebuffer *fb, 149 struct drm_framebuffer *fb,
167 struct drm_rect *clip, bool swap) 150 struct drm_rect *clip, bool swab)
168{ 151{
169 unsigned int src_offset = (clip->y1 * fb->pitches[0]) 152 size_t linepixels = clip->x2 - clip->x1;
170 + (clip->x1 * sizeof(u32)); 153 size_t src_len = linepixels * sizeof(u32);
171 size_t src_len = (clip->x2 - clip->x1) * sizeof(u32); 154 size_t dst_len = linepixels * sizeof(u16);
172 size_t dst_len = (clip->x2 - clip->x1) * sizeof(u16); 155 unsigned y, lines = clip->y2 - clip->y1;
173 156 void *sbuf;
174 drm_fb_xrgb8888_to_rgb565_lines(dst, dst_len, 157
175 vaddr + src_offset, fb->pitches[0], 158 /*
176 src_len, clip->y2 - clip->y1, 159 * The cma memory is write-combined so reads are uncached.
177 swap); 160 * Speed up by fetching one line at a time.
161 */
162 sbuf = kmalloc(src_len, GFP_KERNEL);
163 if (!sbuf)
164 return;
165
166 vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
167 for (y = 0; y < lines; y++) {
168 memcpy(sbuf, vaddr, src_len);
169 drm_fb_xrgb8888_to_rgb565_line(dst, sbuf, linepixels, swab);
170 vaddr += fb->pitches[0];
171 dst += dst_len;
172 }
173
174 kfree(sbuf);
178} 175}
179EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565); 176EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
180 177
181/** 178/**
182 * drm_fb_xrgb8888_to_rgb565_dstclip - Convert XRGB8888 to RGB565 clip buffer 179 * drm_fb_xrgb8888_to_rgb565_dstclip - Convert XRGB8888 to RGB565 clip buffer
183 * @dst: RGB565 destination buffer 180 * @dst: RGB565 destination buffer (iomem)
184 * @dst_pitch: destination buffer pitch 181 * @dst_pitch: destination buffer pitch
185 * @vaddr: XRGB8888 source buffer 182 * @vaddr: XRGB8888 source buffer
186 * @fb: DRM framebuffer 183 * @fb: DRM framebuffer
187 * @clip: Clip rectangle area to copy 184 * @clip: Clip rectangle area to copy
188 * @swap: Swap bytes 185 * @swab: Swap bytes
189 * 186 *
190 * Drivers can use this function for RGB565 devices that don't natively 187 * Drivers can use this function for RGB565 devices that don't natively
191 * support XRGB8888. 188 * support XRGB8888.
192 * 189 *
193 * This function applies clipping on dst, i.e. the destination is a 190 * This function applies clipping on dst, i.e. the destination is a
194 * full framebuffer but only the clip rect content is copied over. 191 * full (iomem) framebuffer but only the clip rect content is copied over.
195 */ 192 */
196void drm_fb_xrgb8888_to_rgb565_dstclip(void *dst, unsigned int dst_pitch, 193void drm_fb_xrgb8888_to_rgb565_dstclip(void __iomem *dst, unsigned int dst_pitch,
197 void *vaddr, struct drm_framebuffer *fb, 194 void *vaddr, struct drm_framebuffer *fb,
198 struct drm_rect *clip, bool swap) 195 struct drm_rect *clip, bool swab)
199{
200 unsigned int src_offset = (clip->y1 * fb->pitches[0])
201 + (clip->x1 * sizeof(u32));
202 unsigned int dst_offset = (clip->y1 * dst_pitch)
203 + (clip->x1 * sizeof(u16));
204 size_t src_len = (clip->x2 - clip->x1) * sizeof(u32);
205
206 drm_fb_xrgb8888_to_rgb565_lines(dst + dst_offset, dst_pitch,
207 vaddr + src_offset, fb->pitches[0],
208 src_len, clip->y2 - clip->y1,
209 swap);
210}
211EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565_dstclip);
212
213static void drm_fb_xrgb8888_to_rgb888_lines(void *dst, unsigned int dst_pitch,
214 void *src, unsigned int src_pitch,
215 unsigned int src_linelength,
216 unsigned int lines)
217{ 196{
218 unsigned int linepixels = src_linelength / 3; 197 size_t linepixels = clip->x2 - clip->x1;
219 unsigned int x, y; 198 size_t dst_len = linepixels * sizeof(u16);
220 u32 *sbuf; 199 unsigned y, lines = clip->y2 - clip->y1;
221 u8 *dbuf; 200 void *dbuf;
222 201
223 sbuf = kmalloc(src_linelength, GFP_KERNEL); 202 dbuf = kmalloc(dst_len, GFP_KERNEL);
224 if (!sbuf) 203 if (!dbuf)
225 return; 204 return;
226 205
206 vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
207 dst += clip_offset(clip, dst_pitch, sizeof(u16));
227 for (y = 0; y < lines; y++) { 208 for (y = 0; y < lines; y++) {
228 memcpy(sbuf, src, src_linelength); 209 drm_fb_xrgb8888_to_rgb565_line(dbuf, vaddr, linepixels, swab);
229 dbuf = dst; 210 memcpy_toio(dst, dbuf, dst_len);
230 for (x = 0; x < linepixels; x++) { 211 vaddr += fb->pitches[0];
231 *dbuf++ = (sbuf[x] & 0x000000FF) >> 0; 212 dst += dst_len;
232 *dbuf++ = (sbuf[x] & 0x0000FF00) >> 8;
233 *dbuf++ = (sbuf[x] & 0x00FF0000) >> 16;
234 }
235 src += src_pitch;
236 dst += dst_pitch;
237 } 213 }
238 214
239 kfree(sbuf); 215 kfree(dbuf);
216}
217EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565_dstclip);
218
219static void drm_fb_xrgb8888_to_rgb888_line(u8 *dbuf, u32 *sbuf,
220 unsigned int pixels)
221{
222 unsigned int x;
223
224 for (x = 0; x < pixels; x++) {
225 *dbuf++ = (sbuf[x] & 0x000000FF) >> 0;
226 *dbuf++ = (sbuf[x] & 0x0000FF00) >> 8;
227 *dbuf++ = (sbuf[x] & 0x00FF0000) >> 16;
228 }
240} 229}
241 230
242/** 231/**
243 * drm_fb_xrgb8888_to_rgb888_dstclip - Convert XRGB8888 to RGB888 clip buffer 232 * drm_fb_xrgb8888_to_rgb888_dstclip - Convert XRGB8888 to RGB888 clip buffer
244 * @dst: RGB565 destination buffer 233 * @dst: RGB565 destination buffer (iomem)
245 * @dst_pitch: destination buffer pitch 234 * @dst_pitch: destination buffer pitch
246 * @vaddr: XRGB8888 source buffer 235 * @vaddr: XRGB8888 source buffer
247 * @fb: DRM framebuffer 236 * @fb: DRM framebuffer
248 * @clip: Clip rectangle area to copy 237 * @clip: Clip rectangle area to copy
249 * @dstclip: Clip destination too.
250 * 238 *
251 * Drivers can use this function for RGB888 devices that don't natively 239 * Drivers can use this function for RGB888 devices that don't natively
252 * support XRGB8888. 240 * support XRGB8888.
253 * 241 *
254 * This function applies clipping on dst, i.e. the destination is a 242 * This function applies clipping on dst, i.e. the destination is a
255 * full framebuffer but only the clip rect content is copied over. 243 * full (iomem) framebuffer but only the clip rect content is copied over.
256 */ 244 */
257void drm_fb_xrgb8888_to_rgb888_dstclip(void *dst, unsigned int dst_pitch, 245void drm_fb_xrgb8888_to_rgb888_dstclip(void __iomem *dst, unsigned int dst_pitch,
258 void *vaddr, struct drm_framebuffer *fb, 246 void *vaddr, struct drm_framebuffer *fb,
259 struct drm_rect *clip) 247 struct drm_rect *clip)
260{ 248{
261 unsigned int src_offset = (clip->y1 * fb->pitches[0]) 249 size_t linepixels = clip->x2 - clip->x1;
262 + (clip->x1 * sizeof(u32)); 250 size_t dst_len = linepixels * 3;
263 unsigned int dst_offset = (clip->y1 * dst_pitch) 251 unsigned y, lines = clip->y2 - clip->y1;
264 + (clip->x1 * 3); 252 void *dbuf;
265 size_t src_len = (clip->x2 - clip->x1) * sizeof(u32); 253
266 254 dbuf = kmalloc(dst_len, GFP_KERNEL);
267 drm_fb_xrgb8888_to_rgb888_lines(dst + dst_offset, dst_pitch, 255 if (!dbuf)
268 vaddr + src_offset, fb->pitches[0], 256 return;
269 src_len, clip->y2 - clip->y1); 257
258 vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
259 dst += clip_offset(clip, dst_pitch, sizeof(u16));
260 for (y = 0; y < lines; y++) {
261 drm_fb_xrgb8888_to_rgb888_line(dbuf, vaddr, linepixels);
262 memcpy_toio(dst, dbuf, dst_len);
263 vaddr += fb->pitches[0];
264 dst += dst_len;
265 }
266
267 kfree(dbuf);
270} 268}
271EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888_dstclip); 269EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888_dstclip);
272 270
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 52c0a837a3b2..fae4676707b6 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -646,6 +646,85 @@ void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
646} 646}
647EXPORT_SYMBOL(drm_gem_put_pages); 647EXPORT_SYMBOL(drm_gem_put_pages);
648 648
649static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
650 struct drm_gem_object **objs)
651{
652 int i, ret = 0;
653 struct drm_gem_object *obj;
654
655 spin_lock(&filp->table_lock);
656
657 for (i = 0; i < count; i++) {
658 /* Check if we currently have a reference on the object */
659 obj = idr_find(&filp->object_idr, handle[i]);
660 if (!obj) {
661 ret = -ENOENT;
662 break;
663 }
664 drm_gem_object_get(obj);
665 objs[i] = obj;
666 }
667 spin_unlock(&filp->table_lock);
668
669 return ret;
670}
671
672/**
673 * drm_gem_objects_lookup - look up GEM objects from an array of handles
674 * @filp: DRM file private date
675 * @bo_handles: user pointer to array of userspace handle
676 * @count: size of handle array
677 * @objs_out: returned pointer to array of drm_gem_object pointers
678 *
679 * Takes an array of userspace handles and returns a newly allocated array of
680 * GEM objects.
681 *
682 * For a single handle lookup, use drm_gem_object_lookup().
683 *
684 * Returns:
685 *
686 * @objs filled in with GEM object pointers. Returned GEM objects need to be
687 * released with drm_gem_object_put(). -ENOENT is returned on a lookup
688 * failure. 0 is returned on success.
689 *
690 */
691int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
692 int count, struct drm_gem_object ***objs_out)
693{
694 int ret;
695 u32 *handles;
696 struct drm_gem_object **objs;
697
698 if (!count)
699 return 0;
700
701 objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
702 GFP_KERNEL | __GFP_ZERO);
703 if (!objs)
704 return -ENOMEM;
705
706 handles = kvmalloc_array(count, sizeof(u32), GFP_KERNEL);
707 if (!handles) {
708 ret = -ENOMEM;
709 goto out;
710 }
711
712 if (copy_from_user(handles, bo_handles, count * sizeof(u32))) {
713 ret = -EFAULT;
714 DRM_DEBUG("Failed to copy in GEM handles\n");
715 goto out;
716 }
717
718 ret = objects_lookup(filp, handles, count, objs);
719 *objs_out = objs;
720
721out:
722 kvfree(handles);
723 return ret;
724
725}
726EXPORT_SYMBOL(drm_gem_objects_lookup);
727
649/** 728/**
650 * drm_gem_object_lookup - look up a GEM object from its handle 729 * drm_gem_object_lookup - look up a GEM object from its handle
651 * @filp: DRM file private date 730 * @filp: DRM file private date
@@ -655,21 +734,15 @@ EXPORT_SYMBOL(drm_gem_put_pages);
655 * 734 *
656 * A reference to the object named by the handle if such exists on @filp, NULL 735 * A reference to the object named by the handle if such exists on @filp, NULL
657 * otherwise. 736 * otherwise.
737 *
738 * If looking up an array of handles, use drm_gem_objects_lookup().
658 */ 739 */
659struct drm_gem_object * 740struct drm_gem_object *
660drm_gem_object_lookup(struct drm_file *filp, u32 handle) 741drm_gem_object_lookup(struct drm_file *filp, u32 handle)
661{ 742{
662 struct drm_gem_object *obj; 743 struct drm_gem_object *obj = NULL;
663
664 spin_lock(&filp->table_lock);
665
666 /* Check if we currently have a reference on the object */
667 obj = idr_find(&filp->object_idr, handle);
668 if (obj)
669 drm_gem_object_get(obj);
670
671 spin_unlock(&filp->table_lock);
672 744
745 objects_lookup(filp, &handle, 1, &obj);
673 return obj; 746 return obj;
674} 747}
675EXPORT_SYMBOL(drm_gem_object_lookup); 748EXPORT_SYMBOL(drm_gem_object_lookup);
@@ -1294,3 +1367,96 @@ drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1294 ww_acquire_fini(acquire_ctx); 1367 ww_acquire_fini(acquire_ctx);
1295} 1368}
1296EXPORT_SYMBOL(drm_gem_unlock_reservations); 1369EXPORT_SYMBOL(drm_gem_unlock_reservations);
1370
1371/**
1372 * drm_gem_fence_array_add - Adds the fence to an array of fences to be
1373 * waited on, deduplicating fences from the same context.
1374 *
1375 * @fence_array array of dma_fence * for the job to block on.
1376 * @fence the dma_fence to add to the list of dependencies.
1377 *
1378 * Returns:
1379 * 0 on success, or an error on failing to expand the array.
1380 */
1381int drm_gem_fence_array_add(struct xarray *fence_array,
1382 struct dma_fence *fence)
1383{
1384 struct dma_fence *entry;
1385 unsigned long index;
1386 u32 id = 0;
1387 int ret;
1388
1389 if (!fence)
1390 return 0;
1391
1392 /* Deduplicate if we already depend on a fence from the same context.
1393 * This lets the size of the array of deps scale with the number of
1394 * engines involved, rather than the number of BOs.
1395 */
1396 xa_for_each(fence_array, index, entry) {
1397 if (entry->context != fence->context)
1398 continue;
1399
1400 if (dma_fence_is_later(fence, entry)) {
1401 dma_fence_put(entry);
1402 xa_store(fence_array, index, fence, GFP_KERNEL);
1403 } else {
1404 dma_fence_put(fence);
1405 }
1406 return 0;
1407 }
1408
1409 ret = xa_alloc(fence_array, &id, fence, xa_limit_32b, GFP_KERNEL);
1410 if (ret != 0)
1411 dma_fence_put(fence);
1412
1413 return ret;
1414}
1415EXPORT_SYMBOL(drm_gem_fence_array_add);
1416
1417/**
1418 * drm_gem_fence_array_add_implicit - Adds the implicit dependencies tracked
1419 * in the GEM object's reservation object to an array of dma_fences for use in
1420 * scheduling a rendering job.
1421 *
1422 * This should be called after drm_gem_lock_reservations() on your array of
1423 * GEM objects used in the job but before updating the reservations with your
1424 * own fences.
1425 *
1426 * @fence_array array of dma_fence * for the job to block on.
1427 * @obj the gem object to add new dependencies from.
1428 * @write whether the job might write the object (so we need to depend on
1429 * shared fences in the reservation object).
1430 */
1431int drm_gem_fence_array_add_implicit(struct xarray *fence_array,
1432 struct drm_gem_object *obj,
1433 bool write)
1434{
1435 int ret;
1436 struct dma_fence **fences;
1437 unsigned int i, fence_count;
1438
1439 if (!write) {
1440 struct dma_fence *fence =
1441 reservation_object_get_excl_rcu(obj->resv);
1442
1443 return drm_gem_fence_array_add(fence_array, fence);
1444 }
1445
1446 ret = reservation_object_get_fences_rcu(obj->resv, NULL,
1447 &fence_count, &fences);
1448 if (ret || !fence_count)
1449 return ret;
1450
1451 for (i = 0; i < fence_count; i++) {
1452 ret = drm_gem_fence_array_add(fence_array, fences[i]);
1453 if (ret)
1454 break;
1455 }
1456
1457 for (; i < fence_count; i++)
1458 dma_fence_put(fences[i]);
1459 kfree(fences);
1460 return ret;
1461}
1462EXPORT_SYMBOL(drm_gem_fence_array_add_implicit);
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 4a1c2023ccf0..1a346ae1599d 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -297,8 +297,9 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
297 return -ENOMEM; 297 return -ENOMEM;
298 dev->mode_config.prop_crtc_id = prop; 298 dev->mode_config.prop_crtc_id = prop;
299 299
300 prop = drm_property_create(dev, DRM_MODE_PROP_BLOB, "FB_DAMAGE_CLIPS", 300 prop = drm_property_create(dev,
301 0); 301 DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
302 "FB_DAMAGE_CLIPS", 0);
302 if (!prop) 303 if (!prop)
303 return -ENOMEM; 304 return -ENOMEM;
304 dev->mode_config.prop_fb_damage_clips = prop; 305 dev->mode_config.prop_fb_damage_clips = prop;
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index bc532e99b5dc..89db71996148 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -285,225 +285,7 @@ out_unlock:
285 return ret; 285 return ret;
286} 286}
287 287
288static struct drm_fb_helper_crtc *
289intel_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
290{
291 int i;
292
293 for (i = 0; i < fb_helper->crtc_count; i++)
294 if (fb_helper->crtc_info[i].mode_set.crtc == crtc)
295 return &fb_helper->crtc_info[i];
296
297 return NULL;
298}
299
300/*
301 * Try to read the BIOS display configuration and use it for the initial
302 * fb configuration.
303 *
304 * The BIOS or boot loader will generally create an initial display
305 * configuration for us that includes some set of active pipes and displays.
306 * This routine tries to figure out which pipes and connectors are active
307 * and stuffs them into the crtcs and modes array given to us by the
308 * drm_fb_helper code.
309 *
310 * The overall sequence is:
311 * intel_fbdev_init - from driver load
312 * intel_fbdev_init_bios - initialize the intel_fbdev using BIOS data
313 * drm_fb_helper_init - build fb helper structs
314 * drm_fb_helper_single_add_all_connectors - more fb helper structs
315 * intel_fbdev_initial_config - apply the config
316 * drm_fb_helper_initial_config - call ->probe then register_framebuffer()
317 * drm_setup_crtcs - build crtc config for fbdev
318 * intel_fb_initial_config - find active connectors etc
319 * drm_fb_helper_single_fb_probe - set up fbdev
320 * intelfb_create - re-use or alloc fb, build out fbdev structs
321 *
322 * Note that we don't make special consideration whether we could actually
323 * switch to the selected modes without a full modeset. E.g. when the display
324 * is in VGA mode we need to recalculate watermarks and set a new high-res
325 * framebuffer anyway.
326 */
327static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
328 struct drm_fb_helper_crtc **crtcs,
329 struct drm_display_mode **modes,
330 struct drm_fb_offset *offsets,
331 bool *enabled, int width, int height)
332{
333 struct drm_i915_private *dev_priv = to_i915(fb_helper->dev);
334 unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
335 unsigned long conn_configured, conn_seq;
336 int i, j;
337 bool *save_enabled;
338 bool fallback = true, ret = true;
339 int num_connectors_enabled = 0;
340 int num_connectors_detected = 0;
341 struct drm_modeset_acquire_ctx ctx;
342
343 save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
344 if (!save_enabled)
345 return false;
346
347 drm_modeset_acquire_init(&ctx, 0);
348
349 while (drm_modeset_lock_all_ctx(fb_helper->dev, &ctx) != 0)
350 drm_modeset_backoff(&ctx);
351
352 memcpy(save_enabled, enabled, count);
353 conn_seq = GENMASK(count - 1, 0);
354 conn_configured = 0;
355retry:
356 for (i = 0; i < count; i++) {
357 struct drm_fb_helper_connector *fb_conn;
358 struct drm_connector *connector;
359 struct drm_encoder *encoder;
360 struct drm_fb_helper_crtc *new_crtc;
361
362 fb_conn = fb_helper->connector_info[i];
363 connector = fb_conn->connector;
364
365 if (conn_configured & BIT(i))
366 continue;
367
368 /* First pass, only consider tiled connectors */
369 if (conn_seq == GENMASK(count - 1, 0) && !connector->has_tile)
370 continue;
371
372 if (connector->status == connector_status_connected)
373 num_connectors_detected++;
374
375 if (!enabled[i]) {
376 DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
377 connector->name);
378 conn_configured |= BIT(i);
379 continue;
380 }
381
382 if (connector->force == DRM_FORCE_OFF) {
383 DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
384 connector->name);
385 enabled[i] = false;
386 continue;
387 }
388
389 encoder = connector->state->best_encoder;
390 if (!encoder || WARN_ON(!connector->state->crtc)) {
391 if (connector->force > DRM_FORCE_OFF)
392 goto bail;
393
394 DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
395 connector->name);
396 enabled[i] = false;
397 conn_configured |= BIT(i);
398 continue;
399 }
400
401 num_connectors_enabled++;
402
403 new_crtc = intel_fb_helper_crtc(fb_helper,
404 connector->state->crtc);
405
406 /*
407 * Make sure we're not trying to drive multiple connectors
408 * with a single CRTC, since our cloning support may not
409 * match the BIOS.
410 */
411 for (j = 0; j < count; j++) {
412 if (crtcs[j] == new_crtc) {
413 DRM_DEBUG_KMS("fallback: cloned configuration\n");
414 goto bail;
415 }
416 }
417
418 DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n",
419 connector->name);
420
421 /* go for command line mode first */
422 modes[i] = drm_pick_cmdline_mode(fb_conn);
423
424 /* try for preferred next */
425 if (!modes[i]) {
426 DRM_DEBUG_KMS("looking for preferred mode on connector %s %d\n",
427 connector->name, connector->has_tile);
428 modes[i] = drm_has_preferred_mode(fb_conn, width,
429 height);
430 }
431
432 /* No preferred mode marked by the EDID? Are there any modes? */
433 if (!modes[i] && !list_empty(&connector->modes)) {
434 DRM_DEBUG_KMS("using first mode listed on connector %s\n",
435 connector->name);
436 modes[i] = list_first_entry(&connector->modes,
437 struct drm_display_mode,
438 head);
439 }
440
441 /* last resort: use current mode */
442 if (!modes[i]) {
443 /*
444 * IMPORTANT: We want to use the adjusted mode (i.e.
445 * after the panel fitter upscaling) as the initial
446 * config, not the input mode, which is what crtc->mode
447 * usually contains. But since our current
448 * code puts a mode derived from the post-pfit timings
449 * into crtc->mode this works out correctly.
450 *
451 * This is crtc->mode and not crtc->state->mode for the
452 * fastboot check to work correctly. crtc_state->mode has
453 * I915_MODE_FLAG_INHERITED, which we clear to force check
454 * state.
455 */
456 DRM_DEBUG_KMS("looking for current mode on connector %s\n",
457 connector->name);
458 modes[i] = &connector->state->crtc->mode;
459 }
460 crtcs[i] = new_crtc;
461
462 DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
463 connector->name,
464 connector->state->crtc->base.id,
465 connector->state->crtc->name,
466 modes[i]->hdisplay, modes[i]->vdisplay,
467 modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" :"");
468
469 fallback = false;
470 conn_configured |= BIT(i);
471 }
472
473 if (conn_configured != conn_seq) { /* repeat until no more are found */
474 conn_seq = conn_configured;
475 goto retry;
476 }
477
478 /*
479 * If the BIOS didn't enable everything it could, fall back to have the
480 * same user experiencing of lighting up as much as possible like the
481 * fbdev helper library.
482 */
483 if (num_connectors_enabled != num_connectors_detected &&
484 num_connectors_enabled < INTEL_INFO(dev_priv)->num_pipes) {
485 DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
486 DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
487 num_connectors_detected);
488 fallback = true;
489 }
490
491 if (fallback) {
492bail:
493 DRM_DEBUG_KMS("Not using firmware configuration\n");
494 memcpy(enabled, save_enabled, count);
495 ret = false;
496 }
497
498 drm_modeset_drop_locks(&ctx);
499 drm_modeset_acquire_fini(&ctx);
500
501 kfree(save_enabled);
502 return ret;
503}
504
505static const struct drm_fb_helper_funcs intel_fb_helper_funcs = { 288static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
506 .initial_config = intel_fb_initial_config,
507 .fb_probe = intelfb_create, 289 .fb_probe = intelfb_create,
508}; 290};
509 291
diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c
index 1d69498bc17e..477c0f766663 100644
--- a/drivers/gpu/drm/lima/lima_gem.c
+++ b/drivers/gpu/drm/lima/lima_gem.c
@@ -145,40 +145,7 @@ static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo,
145 if (explicit) 145 if (explicit)
146 return 0; 146 return 0;
147 147
148 /* implicit sync use bo fence in resv obj */ 148 return drm_gem_fence_array_add_implicit(&task->deps, &bo->gem, write);
149 if (write) {
150 unsigned nr_fences;
151 struct dma_fence **fences;
152 int i;
153
154 err = reservation_object_get_fences_rcu(
155 bo->gem.resv, NULL, &nr_fences, &fences);
156 if (err || !nr_fences)
157 return err;
158
159 for (i = 0; i < nr_fences; i++) {
160 err = lima_sched_task_add_dep(task, fences[i]);
161 if (err)
162 break;
163 }
164
165 /* for error case free remaining fences */
166 for ( ; i < nr_fences; i++)
167 dma_fence_put(fences[i]);
168
169 kfree(fences);
170 } else {
171 struct dma_fence *fence;
172
173 fence = reservation_object_get_excl_rcu(bo->gem.resv);
174 if (fence) {
175 err = lima_sched_task_add_dep(task, fence);
176 if (err)
177 dma_fence_put(fence);
178 }
179 }
180
181 return err;
182} 149}
183 150
184static int lima_gem_lock_bos(struct lima_bo **bos, u32 nr_bos, 151static int lima_gem_lock_bos(struct lima_bo **bos, u32 nr_bos,
@@ -251,7 +218,7 @@ static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
251 if (err) 218 if (err)
252 return err; 219 return err;
253 220
254 err = lima_sched_task_add_dep(submit->task, fence); 221 err = drm_gem_fence_array_add(&submit->task->deps, fence);
255 if (err) { 222 if (err) {
256 dma_fence_put(fence); 223 dma_fence_put(fence);
257 return err; 224 return err;
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
index 97bd9c1deb87..d53bd45f8d96 100644
--- a/drivers/gpu/drm/lima/lima_sched.c
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -3,6 +3,7 @@
3 3
4#include <linux/kthread.h> 4#include <linux/kthread.h>
5#include <linux/slab.h> 5#include <linux/slab.h>
6#include <linux/xarray.h>
6 7
7#include "lima_drv.h" 8#include "lima_drv.h"
8#include "lima_sched.h" 9#include "lima_sched.h"
@@ -126,19 +127,24 @@ int lima_sched_task_init(struct lima_sched_task *task,
126 127
127 task->num_bos = num_bos; 128 task->num_bos = num_bos;
128 task->vm = lima_vm_get(vm); 129 task->vm = lima_vm_get(vm);
130
131 xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
132
129 return 0; 133 return 0;
130} 134}
131 135
132void lima_sched_task_fini(struct lima_sched_task *task) 136void lima_sched_task_fini(struct lima_sched_task *task)
133{ 137{
138 struct dma_fence *fence;
139 unsigned long index;
134 int i; 140 int i;
135 141
136 drm_sched_job_cleanup(&task->base); 142 drm_sched_job_cleanup(&task->base);
137 143
138 for (i = 0; i < task->num_dep; i++) 144 xa_for_each(&task->deps, index, fence) {
139 dma_fence_put(task->dep[i]); 145 dma_fence_put(fence);
140 146 }
141 kfree(task->dep); 147 xa_destroy(&task->deps);
142 148
143 if (task->bos) { 149 if (task->bos) {
144 for (i = 0; i < task->num_bos; i++) 150 for (i = 0; i < task->num_bos; i++)
@@ -149,42 +155,6 @@ void lima_sched_task_fini(struct lima_sched_task *task)
149 lima_vm_put(task->vm); 155 lima_vm_put(task->vm);
150} 156}
151 157
152int lima_sched_task_add_dep(struct lima_sched_task *task, struct dma_fence *fence)
153{
154 int i, new_dep = 4;
155
156 /* same context's fence is definitly earlier then this task */
157 if (fence->context == task->base.s_fence->finished.context) {
158 dma_fence_put(fence);
159 return 0;
160 }
161
162 if (task->dep && task->num_dep == task->max_dep)
163 new_dep = task->max_dep * 2;
164
165 if (task->max_dep < new_dep) {
166 void *dep = krealloc(task->dep, sizeof(*task->dep) * new_dep, GFP_KERNEL);
167
168 if (!dep)
169 return -ENOMEM;
170
171 task->max_dep = new_dep;
172 task->dep = dep;
173 }
174
175 for (i = 0; i < task->num_dep; i++) {
176 if (task->dep[i]->context == fence->context &&
177 dma_fence_is_later(fence, task->dep[i])) {
178 dma_fence_put(task->dep[i]);
179 task->dep[i] = fence;
180 return 0;
181 }
182 }
183
184 task->dep[task->num_dep++] = fence;
185 return 0;
186}
187
188int lima_sched_context_init(struct lima_sched_pipe *pipe, 158int lima_sched_context_init(struct lima_sched_pipe *pipe,
189 struct lima_sched_context *context, 159 struct lima_sched_context *context,
190 atomic_t *guilty) 160 atomic_t *guilty)
@@ -213,21 +183,9 @@ static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
213 struct drm_sched_entity *entity) 183 struct drm_sched_entity *entity)
214{ 184{
215 struct lima_sched_task *task = to_lima_task(job); 185 struct lima_sched_task *task = to_lima_task(job);
216 int i;
217
218 for (i = 0; i < task->num_dep; i++) {
219 struct dma_fence *fence = task->dep[i];
220
221 if (!task->dep[i])
222 continue;
223
224 task->dep[i] = NULL;
225 186
226 if (!dma_fence_is_signaled(fence)) 187 if (!xa_empty(&task->deps))
227 return fence; 188 return xa_erase(&task->deps, task->last_dep++);
228
229 dma_fence_put(fence);
230 }
231 189
232 return NULL; 190 return NULL;
233} 191}
@@ -353,7 +311,7 @@ static void lima_sched_free_job(struct drm_sched_job *job)
353 kmem_cache_free(pipe->task_slab, task); 311 kmem_cache_free(pipe->task_slab, task);
354} 312}
355 313
356const struct drm_sched_backend_ops lima_sched_ops = { 314static const struct drm_sched_backend_ops lima_sched_ops = {
357 .dependency = lima_sched_dependency, 315 .dependency = lima_sched_dependency,
358 .run_job = lima_sched_run_job, 316 .run_job = lima_sched_run_job,
359 .timedout_job = lima_sched_timedout_job, 317 .timedout_job = lima_sched_timedout_job,
diff --git a/drivers/gpu/drm/lima/lima_sched.h b/drivers/gpu/drm/lima/lima_sched.h
index b017cfa7e327..928af91c1118 100644
--- a/drivers/gpu/drm/lima/lima_sched.h
+++ b/drivers/gpu/drm/lima/lima_sched.h
@@ -14,9 +14,8 @@ struct lima_sched_task {
14 struct lima_vm *vm; 14 struct lima_vm *vm;
15 void *frame; 15 void *frame;
16 16
17 struct dma_fence **dep; 17 struct xarray deps;
18 int num_dep; 18 unsigned long last_dep;
19 int max_dep;
20 19
21 struct lima_bo **bos; 20 struct lima_bo **bos;
22 int num_bos; 21 int num_bos;
@@ -78,7 +77,6 @@ int lima_sched_task_init(struct lima_sched_task *task,
78 struct lima_bo **bos, int num_bos, 77 struct lima_bo **bos, int num_bos,
79 struct lima_vm *vm); 78 struct lima_vm *vm);
80void lima_sched_task_fini(struct lima_sched_task *task); 79void lima_sched_task_fini(struct lima_sched_task *task);
81int lima_sched_task_add_dep(struct lima_sched_task *task, struct dma_fence *fence);
82 80
83int lima_sched_context_init(struct lima_sched_pipe *pipe, 81int lima_sched_context_init(struct lima_sched_pipe *pipe,
84 struct lima_sched_context *context, 82 struct lima_sched_context *context,
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index faf1b1b0357c..72b01e6be0d9 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -90,6 +90,18 @@ static irqreturn_t meson_irq(int irq, void *arg)
90 return IRQ_HANDLED; 90 return IRQ_HANDLED;
91} 91}
92 92
93static int meson_dumb_create(struct drm_file *file, struct drm_device *dev,
94 struct drm_mode_create_dumb *args)
95{
96 /*
97 * We need 64bytes aligned stride, and PAGE aligned size
98 */
99 args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), SZ_64);
100 args->size = PAGE_ALIGN(args->pitch * args->height);
101
102 return drm_gem_cma_dumb_create_internal(file, dev, args);
103}
104
93DEFINE_DRM_GEM_CMA_FOPS(fops); 105DEFINE_DRM_GEM_CMA_FOPS(fops);
94 106
95static struct drm_driver meson_driver = { 107static struct drm_driver meson_driver = {
@@ -112,7 +124,7 @@ static struct drm_driver meson_driver = {
112 .gem_prime_mmap = drm_gem_cma_prime_mmap, 124 .gem_prime_mmap = drm_gem_cma_prime_mmap,
113 125
114 /* GEM Ops */ 126 /* GEM Ops */
115 .dumb_create = drm_gem_cma_dumb_create, 127 .dumb_create = meson_dumb_create,
116 .gem_free_object_unlocked = drm_gem_cma_free_object, 128 .gem_free_object_unlocked = drm_gem_cma_free_object,
117 .gem_vm_ops = &drm_gem_cma_vm_ops, 129 .gem_vm_ops = &drm_gem_cma_vm_ops,
118 130
diff --git a/drivers/gpu/drm/meson/meson_viu.c b/drivers/gpu/drm/meson/meson_viu.c
index 0169c98b01c9..b59072342cae 100644
--- a/drivers/gpu/drm/meson/meson_viu.c
+++ b/drivers/gpu/drm/meson/meson_viu.c
@@ -90,8 +90,8 @@ static int eotf_bypass_coeff[EOTF_COEFF_SIZE] = {
90 EOTF_COEFF_RIGHTSHIFT /* right shift */ 90 EOTF_COEFF_RIGHTSHIFT /* right shift */
91}; 91};
92 92
93void meson_viu_set_g12a_osd1_matrix(struct meson_drm *priv, int *m, 93static void meson_viu_set_g12a_osd1_matrix(struct meson_drm *priv,
94 bool csc_on) 94 int *m, bool csc_on)
95{ 95{
96 /* VPP WRAP OSD1 matrix */ 96 /* VPP WRAP OSD1 matrix */
97 writel(((m[0] & 0xfff) << 16) | (m[1] & 0xfff), 97 writel(((m[0] & 0xfff) << 16) | (m[1] & 0xfff),
@@ -118,8 +118,8 @@ void meson_viu_set_g12a_osd1_matrix(struct meson_drm *priv, int *m,
118 priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_EN_CTRL)); 118 priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_EN_CTRL));
119} 119}
120 120
121void meson_viu_set_osd_matrix(struct meson_drm *priv, 121static void meson_viu_set_osd_matrix(struct meson_drm *priv,
122 enum viu_matrix_sel_e m_select, 122 enum viu_matrix_sel_e m_select,
123 int *m, bool csc_on) 123 int *m, bool csc_on)
124{ 124{
125 if (m_select == VIU_MATRIX_OSD) { 125 if (m_select == VIU_MATRIX_OSD) {
@@ -187,10 +187,10 @@ void meson_viu_set_osd_matrix(struct meson_drm *priv,
187#define OSD_EOTF_LUT_SIZE 33 187#define OSD_EOTF_LUT_SIZE 33
188#define OSD_OETF_LUT_SIZE 41 188#define OSD_OETF_LUT_SIZE 41
189 189
190void meson_viu_set_osd_lut(struct meson_drm *priv, enum viu_lut_sel_e lut_sel, 190static void
191 unsigned int *r_map, unsigned int *g_map, 191meson_viu_set_osd_lut(struct meson_drm *priv, enum viu_lut_sel_e lut_sel,
192 unsigned int *b_map, 192 unsigned int *r_map, unsigned int *g_map,
193 bool csc_on) 193 unsigned int *b_map, bool csc_on)
194{ 194{
195 unsigned int addr_port; 195 unsigned int addr_port;
196 unsigned int data_port; 196 unsigned int data_port;
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 8fee7a8b29d9..569be4efd8d1 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -3025,6 +3025,34 @@ static const struct panel_desc_dsi panasonic_vvx10f004b00 = {
3025 .lanes = 4, 3025 .lanes = 4,
3026}; 3026};
3027 3027
3028static const struct drm_display_mode lg_acx467akm_7_mode = {
3029 .clock = 150000,
3030 .hdisplay = 1080,
3031 .hsync_start = 1080 + 2,
3032 .hsync_end = 1080 + 2 + 2,
3033 .htotal = 1080 + 2 + 2 + 2,
3034 .vdisplay = 1920,
3035 .vsync_start = 1920 + 2,
3036 .vsync_end = 1920 + 2 + 2,
3037 .vtotal = 1920 + 2 + 2 + 2,
3038 .vrefresh = 60,
3039};
3040
3041static const struct panel_desc_dsi lg_acx467akm_7 = {
3042 .desc = {
3043 .modes = &lg_acx467akm_7_mode,
3044 .num_modes = 1,
3045 .bpc = 8,
3046 .size = {
3047 .width = 62,
3048 .height = 110,
3049 },
3050 },
3051 .flags = 0,
3052 .format = MIPI_DSI_FMT_RGB888,
3053 .lanes = 4,
3054};
3055
3028static const struct of_device_id dsi_of_match[] = { 3056static const struct of_device_id dsi_of_match[] = {
3029 { 3057 {
3030 .compatible = "auo,b080uan01", 3058 .compatible = "auo,b080uan01",
@@ -3042,6 +3070,9 @@ static const struct of_device_id dsi_of_match[] = {
3042 .compatible = "panasonic,vvx10f004b00", 3070 .compatible = "panasonic,vvx10f004b00",
3043 .data = &panasonic_vvx10f004b00 3071 .data = &panasonic_vvx10f004b00
3044 }, { 3072 }, {
3073 .compatible = "lg,acx467akm-7",
3074 .data = &lg_acx467akm_7
3075 }, {
3045 /* sentinel */ 3076 /* sentinel */
3046 } 3077 }
3047}; 3078};
diff --git a/drivers/gpu/drm/panfrost/Kconfig b/drivers/gpu/drm/panfrost/Kconfig
new file mode 100644
index 000000000000..7f5e572daa2d
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/Kconfig
@@ -0,0 +1,14 @@
1# SPDX-License-Identifier: GPL-2.0
2
3config DRM_PANFROST
4 tristate "Panfrost (DRM support for ARM Mali Midgard/Bifrost GPUs)"
5 depends on DRM
6 depends on ARM || ARM64 || COMPILE_TEST
7 depends on MMU
8 select DRM_SCHED
9 select IOMMU_SUPPORT
10 select IOMMU_IO_PGTABLE_LPAE
11 select DRM_GEM_SHMEM_HELPER
12 help
13 DRM driver for ARM Mali Midgard (T6xx, T7xx, T8xx) and
14 Bifrost (G3x, G5x, G7x) GPUs.
diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile
new file mode 100644
index 000000000000..6de72d13c58f
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/Makefile
@@ -0,0 +1,12 @@
1# SPDX-License-Identifier: GPL-2.0
2
3panfrost-y := \
4 panfrost_drv.o \
5 panfrost_device.o \
6 panfrost_devfreq.o \
7 panfrost_gem.o \
8 panfrost_gpu.o \
9 panfrost_job.o \
10 panfrost_mmu.o
11
12obj-$(CONFIG_DRM_PANFROST) += panfrost.o
diff --git a/drivers/gpu/drm/panfrost/TODO b/drivers/gpu/drm/panfrost/TODO
new file mode 100644
index 000000000000..c2e44add37d8
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/TODO
@@ -0,0 +1,27 @@
1- Thermal support.
2
3- Bifrost support:
4 - DT bindings (Neil, WIP)
5 - MMU page table format and address space setup
6 - Bifrost specific feature and issue handling
7 - Coherent DMA support
8
9- Support for 2MB pages. The io-pgtable code already supports this. Finishing
10 support involves either copying or adapting the iommu API to handle passing
11 aligned addresses and sizes to the io-pgtable code.
12
13- Per FD address space support. The h/w supports multiple addresses spaces.
14 The hard part is handling when more address spaces are needed than what
15 the h/w provides.
16
17- Support pinning pages on demand (GPU page faults).
18
19- Support userspace controlled GPU virtual addresses. Needed for Vulkan. (Tomeu)
20
21- Support for madvise and a shrinker.
22
23- Compute job support. So called 'compute only' jobs need to be plumbed up to
24 userspace.
25
26- Performance counter support. (Boris)
27
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
new file mode 100644
index 000000000000..a8121ae67ee3
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
@@ -0,0 +1,218 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Collabora ltd. */
3#include <linux/devfreq.h>
4#include <linux/platform_device.h>
5#include <linux/pm_opp.h>
6#include <linux/clk.h>
7#include <linux/regulator/consumer.h>
8
9#include "panfrost_device.h"
10#include "panfrost_features.h"
11#include "panfrost_issues.h"
12#include "panfrost_gpu.h"
13#include "panfrost_regs.h"
14
15static void panfrost_devfreq_update_utilization(struct panfrost_device *pfdev, int slot);
16
17static int panfrost_devfreq_target(struct device *dev, unsigned long *freq,
18 u32 flags)
19{
20 struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
21 struct dev_pm_opp *opp;
22 unsigned long old_clk_rate = pfdev->devfreq.cur_freq;
23 unsigned long target_volt, target_rate;
24 int err;
25
26 opp = devfreq_recommended_opp(dev, freq, flags);
27 if (IS_ERR(opp))
28 return PTR_ERR(opp);
29
30 target_rate = dev_pm_opp_get_freq(opp);
31 target_volt = dev_pm_opp_get_voltage(opp);
32 dev_pm_opp_put(opp);
33
34 if (old_clk_rate == target_rate)
35 return 0;
36
37 /*
38 * If frequency scaling from low to high, adjust voltage first.
39 * If frequency scaling from high to low, adjust frequency first.
40 */
41 if (old_clk_rate < target_rate) {
42 err = regulator_set_voltage(pfdev->regulator, target_volt,
43 target_volt);
44 if (err) {
45 dev_err(dev, "Cannot set voltage %lu uV\n",
46 target_volt);
47 return err;
48 }
49 }
50
51 err = clk_set_rate(pfdev->clock, target_rate);
52 if (err) {
53 dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate,
54 err);
55 regulator_set_voltage(pfdev->regulator, pfdev->devfreq.cur_volt,
56 pfdev->devfreq.cur_volt);
57 return err;
58 }
59
60 if (old_clk_rate > target_rate) {
61 err = regulator_set_voltage(pfdev->regulator, target_volt,
62 target_volt);
63 if (err)
64 dev_err(dev, "Cannot set voltage %lu uV\n", target_volt);
65 }
66
67 pfdev->devfreq.cur_freq = target_rate;
68 pfdev->devfreq.cur_volt = target_volt;
69
70 return 0;
71}
72
73static void panfrost_devfreq_reset(struct panfrost_device *pfdev)
74{
75 ktime_t now = ktime_get();
76 int i;
77
78 for (i = 0; i < NUM_JOB_SLOTS; i++) {
79 pfdev->devfreq.slot[i].busy_time = 0;
80 pfdev->devfreq.slot[i].idle_time = 0;
81 pfdev->devfreq.slot[i].time_last_update = now;
82 }
83}
84
85static int panfrost_devfreq_get_dev_status(struct device *dev,
86 struct devfreq_dev_status *status)
87{
88 struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
89 int i;
90
91 for (i = 0; i < NUM_JOB_SLOTS; i++) {
92 panfrost_devfreq_update_utilization(pfdev, i);
93 }
94
95 status->current_frequency = clk_get_rate(pfdev->clock);
96 status->total_time = ktime_to_ns(ktime_add(pfdev->devfreq.slot[0].busy_time,
97 pfdev->devfreq.slot[0].idle_time));
98
99 status->busy_time = 0;
100 for (i = 0; i < NUM_JOB_SLOTS; i++) {
101 status->busy_time += ktime_to_ns(pfdev->devfreq.slot[i].busy_time);
102 }
103
104 /* We're scheduling only to one core atm, so don't divide for now */
105 /* status->busy_time /= NUM_JOB_SLOTS; */
106
107 panfrost_devfreq_reset(pfdev);
108
109 dev_dbg(pfdev->dev, "busy %lu total %lu %lu %% freq %lu MHz\n", status->busy_time,
110 status->total_time,
111 status->busy_time / (status->total_time / 100),
112 status->current_frequency / 1000 / 1000);
113
114 return 0;
115}
116
117static int panfrost_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
118{
119 struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
120
121 *freq = pfdev->devfreq.cur_freq;
122
123 return 0;
124}
125
126static struct devfreq_dev_profile panfrost_devfreq_profile = {
127 .polling_ms = 50, /* ~3 frames */
128 .target = panfrost_devfreq_target,
129 .get_dev_status = panfrost_devfreq_get_dev_status,
130 .get_cur_freq = panfrost_devfreq_get_cur_freq,
131};
132
133int panfrost_devfreq_init(struct panfrost_device *pfdev)
134{
135 int ret;
136 struct dev_pm_opp *opp;
137
138 if (!pfdev->regulator)
139 return 0;
140
141 ret = dev_pm_opp_of_add_table(&pfdev->pdev->dev);
142 if (ret == -ENODEV) /* Optional, continue without devfreq */
143 return 0;
144
145 panfrost_devfreq_reset(pfdev);
146
147 pfdev->devfreq.cur_freq = clk_get_rate(pfdev->clock);
148
149 opp = devfreq_recommended_opp(&pfdev->pdev->dev, &pfdev->devfreq.cur_freq, 0);
150 if (IS_ERR(opp))
151 return PTR_ERR(opp);
152
153 panfrost_devfreq_profile.initial_freq = pfdev->devfreq.cur_freq;
154 dev_pm_opp_put(opp);
155
156 pfdev->devfreq.devfreq = devm_devfreq_add_device(&pfdev->pdev->dev,
157 &panfrost_devfreq_profile, "simple_ondemand", NULL);
158 if (IS_ERR(pfdev->devfreq.devfreq)) {
159 DRM_DEV_ERROR(&pfdev->pdev->dev, "Couldn't initialize GPU devfreq\n");
160 ret = PTR_ERR(pfdev->devfreq.devfreq);
161 pfdev->devfreq.devfreq = NULL;
162 return ret;
163 }
164
165 return 0;
166}
167
168void panfrost_devfreq_resume(struct panfrost_device *pfdev)
169{
170 int i;
171
172 if (!pfdev->devfreq.devfreq)
173 return;
174
175 panfrost_devfreq_reset(pfdev);
176 for (i = 0; i < NUM_JOB_SLOTS; i++)
177 pfdev->devfreq.slot[i].busy = false;
178
179 devfreq_resume_device(pfdev->devfreq.devfreq);
180}
181
182void panfrost_devfreq_suspend(struct panfrost_device *pfdev)
183{
184 if (!pfdev->devfreq.devfreq)
185 return;
186
187 devfreq_suspend_device(pfdev->devfreq.devfreq);
188}
189
190static void panfrost_devfreq_update_utilization(struct panfrost_device *pfdev, int slot)
191{
192 struct panfrost_devfreq_slot *devfreq_slot = &pfdev->devfreq.slot[slot];
193 ktime_t now;
194 ktime_t last;
195
196 if (!pfdev->devfreq.devfreq)
197 return;
198
199 now = ktime_get();
200 last = pfdev->devfreq.slot[slot].time_last_update;
201
202 /* If we last recorded a transition to busy, we have been idle since */
203 if (devfreq_slot->busy)
204 pfdev->devfreq.slot[slot].busy_time += ktime_sub(now, last);
205 else
206 pfdev->devfreq.slot[slot].idle_time += ktime_sub(now, last);
207
208 pfdev->devfreq.slot[slot].time_last_update = now;
209}
210
211/* The job scheduler is expected to call this at every transition busy <-> idle */
212void panfrost_devfreq_record_transition(struct panfrost_device *pfdev, int slot)
213{
214 struct panfrost_devfreq_slot *devfreq_slot = &pfdev->devfreq.slot[slot];
215
216 panfrost_devfreq_update_utilization(pfdev, slot);
217 devfreq_slot->busy = !devfreq_slot->busy;
218}
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.h b/drivers/gpu/drm/panfrost/panfrost_devfreq.h
new file mode 100644
index 000000000000..eb999531ed90
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.h
@@ -0,0 +1,14 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2019 Collabora ltd. */
3
4#ifndef __PANFROST_DEVFREQ_H__
5#define __PANFROST_DEVFREQ_H__
6
7int panfrost_devfreq_init(struct panfrost_device *pfdev);
8
9void panfrost_devfreq_resume(struct panfrost_device *pfdev);
10void panfrost_devfreq_suspend(struct panfrost_device *pfdev);
11
12void panfrost_devfreq_record_transition(struct panfrost_device *pfdev, int slot);
13
14#endif /* __PANFROST_DEVFREQ_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
new file mode 100644
index 000000000000..91e8fb0f2b25
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -0,0 +1,252 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4
5#include <linux/clk.h>
6#include <linux/reset.h>
7#include <linux/platform_device.h>
8#include <linux/pm_runtime.h>
9#include <linux/regulator/consumer.h>
10
11#include "panfrost_device.h"
12#include "panfrost_devfreq.h"
13#include "panfrost_features.h"
14#include "panfrost_gpu.h"
15#include "panfrost_job.h"
16#include "panfrost_mmu.h"
17
18static int panfrost_reset_init(struct panfrost_device *pfdev)
19{
20 int err;
21
22 pfdev->rstc = devm_reset_control_array_get(pfdev->dev, false, true);
23 if (IS_ERR(pfdev->rstc)) {
24 dev_err(pfdev->dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc));
25 return PTR_ERR(pfdev->rstc);
26 }
27
28 err = reset_control_deassert(pfdev->rstc);
29 if (err)
30 return err;
31
32 return 0;
33}
34
35static void panfrost_reset_fini(struct panfrost_device *pfdev)
36{
37 reset_control_assert(pfdev->rstc);
38}
39
40static int panfrost_clk_init(struct panfrost_device *pfdev)
41{
42 int err;
43 unsigned long rate;
44
45 pfdev->clock = devm_clk_get(pfdev->dev, NULL);
46 if (IS_ERR(pfdev->clock)) {
47 dev_err(pfdev->dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock));
48 return PTR_ERR(pfdev->clock);
49 }
50
51 rate = clk_get_rate(pfdev->clock);
52 dev_info(pfdev->dev, "clock rate = %lu\n", rate);
53
54 err = clk_prepare_enable(pfdev->clock);
55 if (err)
56 return err;
57
58 return 0;
59}
60
61static void panfrost_clk_fini(struct panfrost_device *pfdev)
62{
63 clk_disable_unprepare(pfdev->clock);
64}
65
66static int panfrost_regulator_init(struct panfrost_device *pfdev)
67{
68 int ret;
69
70 pfdev->regulator = devm_regulator_get_optional(pfdev->dev, "mali");
71 if (IS_ERR(pfdev->regulator)) {
72 ret = PTR_ERR(pfdev->regulator);
73 pfdev->regulator = NULL;
74 if (ret == -ENODEV)
75 return 0;
76 dev_err(pfdev->dev, "failed to get regulator: %d\n", ret);
77 return ret;
78 }
79
80 ret = regulator_enable(pfdev->regulator);
81 if (ret < 0) {
82 dev_err(pfdev->dev, "failed to enable regulator: %d\n", ret);
83 return ret;
84 }
85
86 return 0;
87}
88
89static void panfrost_regulator_fini(struct panfrost_device *pfdev)
90{
91 if (pfdev->regulator)
92 regulator_disable(pfdev->regulator);
93}
94
95int panfrost_device_init(struct panfrost_device *pfdev)
96{
97 int err;
98 struct resource *res;
99
100 mutex_init(&pfdev->sched_lock);
101 INIT_LIST_HEAD(&pfdev->scheduled_jobs);
102
103 spin_lock_init(&pfdev->hwaccess_lock);
104
105 err = panfrost_clk_init(pfdev);
106 if (err) {
107 dev_err(pfdev->dev, "clk init failed %d\n", err);
108 return err;
109 }
110
111 err = panfrost_regulator_init(pfdev);
112 if (err) {
113 dev_err(pfdev->dev, "regulator init failed %d\n", err);
114 goto err_out0;
115 }
116
117 err = panfrost_reset_init(pfdev);
118 if (err) {
119 dev_err(pfdev->dev, "reset init failed %d\n", err);
120 goto err_out1;
121 }
122
123 res = platform_get_resource(pfdev->pdev, IORESOURCE_MEM, 0);
124 pfdev->iomem = devm_ioremap_resource(pfdev->dev, res);
125 if (IS_ERR(pfdev->iomem)) {
126 dev_err(pfdev->dev, "failed to ioremap iomem\n");
127 err = PTR_ERR(pfdev->iomem);
128 goto err_out2;
129 }
130
131 err = panfrost_gpu_init(pfdev);
132 if (err)
133 goto err_out2;
134
135 err = panfrost_mmu_init(pfdev);
136 if (err)
137 goto err_out3;
138
139 err = panfrost_job_init(pfdev);
140 if (err)
141 goto err_out4;
142
143 /* runtime PM will wake us up later */
144 panfrost_gpu_power_off(pfdev);
145
146 pm_runtime_set_active(pfdev->dev);
147 pm_runtime_get_sync(pfdev->dev);
148 pm_runtime_mark_last_busy(pfdev->dev);
149 pm_runtime_put_autosuspend(pfdev->dev);
150
151 return 0;
152err_out4:
153 panfrost_mmu_fini(pfdev);
154err_out3:
155 panfrost_gpu_fini(pfdev);
156err_out2:
157 panfrost_reset_fini(pfdev);
158err_out1:
159 panfrost_regulator_fini(pfdev);
160err_out0:
161 panfrost_clk_fini(pfdev);
162 return err;
163}
164
165void panfrost_device_fini(struct panfrost_device *pfdev)
166{
167 panfrost_regulator_fini(pfdev);
168 panfrost_clk_fini(pfdev);
169}
170
171const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception_code)
172{
173 switch (exception_code) {
174 /* Non-Fault Status code */
175 case 0x00: return "NOT_STARTED/IDLE/OK";
176 case 0x01: return "DONE";
177 case 0x02: return "INTERRUPTED";
178 case 0x03: return "STOPPED";
179 case 0x04: return "TERMINATED";
180 case 0x08: return "ACTIVE";
181 /* Job exceptions */
182 case 0x40: return "JOB_CONFIG_FAULT";
183 case 0x41: return "JOB_POWER_FAULT";
184 case 0x42: return "JOB_READ_FAULT";
185 case 0x43: return "JOB_WRITE_FAULT";
186 case 0x44: return "JOB_AFFINITY_FAULT";
187 case 0x48: return "JOB_BUS_FAULT";
188 case 0x50: return "INSTR_INVALID_PC";
189 case 0x51: return "INSTR_INVALID_ENC";
190 case 0x52: return "INSTR_TYPE_MISMATCH";
191 case 0x53: return "INSTR_OPERAND_FAULT";
192 case 0x54: return "INSTR_TLS_FAULT";
193 case 0x55: return "INSTR_BARRIER_FAULT";
194 case 0x56: return "INSTR_ALIGN_FAULT";
195 case 0x58: return "DATA_INVALID_FAULT";
196 case 0x59: return "TILE_RANGE_FAULT";
197 case 0x5A: return "ADDR_RANGE_FAULT";
198 case 0x60: return "OUT_OF_MEMORY";
199 /* GPU exceptions */
200 case 0x80: return "DELAYED_BUS_FAULT";
201 case 0x88: return "SHAREABILITY_FAULT";
202 /* MMU exceptions */
203 case 0xC1: return "TRANSLATION_FAULT_LEVEL1";
204 case 0xC2: return "TRANSLATION_FAULT_LEVEL2";
205 case 0xC3: return "TRANSLATION_FAULT_LEVEL3";
206 case 0xC4: return "TRANSLATION_FAULT_LEVEL4";
207 case 0xC8: return "PERMISSION_FAULT";
208 case 0xC9 ... 0xCF: return "PERMISSION_FAULT";
209 case 0xD1: return "TRANSTAB_BUS_FAULT_LEVEL1";
210 case 0xD2: return "TRANSTAB_BUS_FAULT_LEVEL2";
211 case 0xD3: return "TRANSTAB_BUS_FAULT_LEVEL3";
212 case 0xD4: return "TRANSTAB_BUS_FAULT_LEVEL4";
213 case 0xD8: return "ACCESS_FLAG";
214 case 0xD9 ... 0xDF: return "ACCESS_FLAG";
215 case 0xE0 ... 0xE7: return "ADDRESS_SIZE_FAULT";
216 case 0xE8 ... 0xEF: return "MEMORY_ATTRIBUTES_FAULT";
217 }
218
219 return "UNKNOWN";
220}
221
222#ifdef CONFIG_PM
223int panfrost_device_resume(struct device *dev)
224{
225 struct platform_device *pdev = to_platform_device(dev);
226 struct panfrost_device *pfdev = platform_get_drvdata(pdev);
227
228 panfrost_gpu_soft_reset(pfdev);
229
230 /* TODO: Re-enable all other address spaces */
231 panfrost_gpu_power_on(pfdev);
232 panfrost_mmu_enable(pfdev, 0);
233 panfrost_job_enable_interrupts(pfdev);
234 panfrost_devfreq_resume(pfdev);
235
236 return 0;
237}
238
239int panfrost_device_suspend(struct device *dev)
240{
241 struct platform_device *pdev = to_platform_device(dev);
242 struct panfrost_device *pfdev = platform_get_drvdata(pdev);
243
244 if (!panfrost_job_is_idle(pfdev))
245 return -EBUSY;
246
247 panfrost_devfreq_suspend(pfdev);
248 panfrost_gpu_power_off(pfdev);
249
250 return 0;
251}
252#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
new file mode 100644
index 000000000000..1ba48d105763
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -0,0 +1,124 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4
5#ifndef __PANFROST_DEVICE_H__
6#define __PANFROST_DEVICE_H__
7
8#include <linux/spinlock.h>
9#include <drm/drm_device.h>
10#include <drm/drm_mm.h>
11#include <drm/gpu_scheduler.h>
12
13struct panfrost_device;
14struct panfrost_mmu;
15struct panfrost_job_slot;
16struct panfrost_job;
17
18#define NUM_JOB_SLOTS 3
19
20struct panfrost_features {
21 u16 id;
22 u16 revision;
23
24 u64 shader_present;
25 u64 tiler_present;
26 u64 l2_present;
27 u64 stack_present;
28 u32 as_present;
29 u32 js_present;
30
31 u32 l2_features;
32 u32 core_features;
33 u32 tiler_features;
34 u32 mem_features;
35 u32 mmu_features;
36 u32 thread_features;
37 u32 max_threads;
38 u32 thread_max_workgroup_sz;
39 u32 thread_max_barrier_sz;
40 u32 coherency_features;
41 u32 texture_features[4];
42 u32 js_features[16];
43
44 u32 nr_core_groups;
45
46 unsigned long hw_features[64 / BITS_PER_LONG];
47 unsigned long hw_issues[64 / BITS_PER_LONG];
48};
49
50struct panfrost_devfreq_slot {
51 ktime_t busy_time;
52 ktime_t idle_time;
53 ktime_t time_last_update;
54 bool busy;
55};
56
57struct panfrost_device {
58 struct device *dev;
59 struct drm_device *ddev;
60 struct platform_device *pdev;
61
62 spinlock_t hwaccess_lock;
63
64 struct drm_mm mm;
65 spinlock_t mm_lock;
66
67 void __iomem *iomem;
68 struct clk *clock;
69 struct regulator *regulator;
70 struct reset_control *rstc;
71
72 struct panfrost_features features;
73
74 struct panfrost_mmu *mmu;
75 struct panfrost_job_slot *js;
76
77 struct panfrost_job *jobs[NUM_JOB_SLOTS];
78 struct list_head scheduled_jobs;
79
80 struct mutex sched_lock;
81
82 struct {
83 struct devfreq *devfreq;
84 struct thermal_cooling_device *cooling;
85 unsigned long cur_freq;
86 unsigned long cur_volt;
87 struct panfrost_devfreq_slot slot[NUM_JOB_SLOTS];
88 } devfreq;
89};
90
91struct panfrost_file_priv {
92 struct panfrost_device *pfdev;
93
94 struct drm_sched_entity sched_entity[NUM_JOB_SLOTS];
95};
96
97static inline struct panfrost_device *to_panfrost_device(struct drm_device *ddev)
98{
99 return ddev->dev_private;
100}
101
102static inline int panfrost_model_cmp(struct panfrost_device *pfdev, s32 id)
103{
104 s32 match_id = pfdev->features.id;
105
106 if (match_id & 0xf000)
107 match_id &= 0xf00f;
108 return match_id - id;
109}
110
111static inline bool panfrost_model_eq(struct panfrost_device *pfdev, s32 id)
112{
113 return !panfrost_model_cmp(pfdev, id);
114}
115
116int panfrost_device_init(struct panfrost_device *pfdev);
117void panfrost_device_fini(struct panfrost_device *pfdev);
118
119int panfrost_device_resume(struct device *dev);
120int panfrost_device_suspend(struct device *dev);
121
122const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception_code);
123
124#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
new file mode 100644
index 000000000000..c06af78ab833
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -0,0 +1,463 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4/* Copyright 2019 Collabora ltd. */
5
6#include <linux/bitfield.h>
7#include <linux/dma-mapping.h>
8#include <linux/module.h>
9#include <linux/of_platform.h>
10#include <linux/pagemap.h>
11#include <linux/pm_runtime.h>
12#include <drm/panfrost_drm.h>
13#include <drm/drm_drv.h>
14#include <drm/drm_ioctl.h>
15#include <drm/drm_syncobj.h>
16#include <drm/drm_utils.h>
17
18#include "panfrost_device.h"
19#include "panfrost_devfreq.h"
20#include "panfrost_gem.h"
21#include "panfrost_mmu.h"
22#include "panfrost_job.h"
23#include "panfrost_gpu.h"
24
25static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file)
26{
27 struct drm_panfrost_get_param *param = data;
28 struct panfrost_device *pfdev = ddev->dev_private;
29
30 if (param->pad != 0)
31 return -EINVAL;
32
33 switch (param->param) {
34 case DRM_PANFROST_PARAM_GPU_PROD_ID:
35 param->value = pfdev->features.id;
36 break;
37 default:
38 return -EINVAL;
39 }
40
41 return 0;
42}
43
44static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
45 struct drm_file *file)
46{
47 int ret;
48 struct drm_gem_shmem_object *shmem;
49 struct drm_panfrost_create_bo *args = data;
50
51 if (!args->size || args->flags || args->pad)
52 return -EINVAL;
53
54 shmem = drm_gem_shmem_create_with_handle(file, dev, args->size,
55 &args->handle);
56 if (IS_ERR(shmem))
57 return PTR_ERR(shmem);
58
59 ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base));
60 if (ret)
61 goto err_free;
62
63 args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT;
64
65 return 0;
66
67err_free:
68 drm_gem_object_put_unlocked(&shmem->base);
69 return ret;
70}
71
72/**
73 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects
74 * referenced by the job.
75 * @dev: DRM device
76 * @file_priv: DRM file for this fd
77 * @args: IOCTL args
78 * @job: job being set up
79 *
80 * Resolve handles from userspace to BOs and attach them to job.
81 *
82 * Note that this function doesn't need to unreference the BOs on
83 * failure, because that will happen at panfrost_job_cleanup() time.
84 */
85static int
86panfrost_lookup_bos(struct drm_device *dev,
87 struct drm_file *file_priv,
88 struct drm_panfrost_submit *args,
89 struct panfrost_job *job)
90{
91 job->bo_count = args->bo_handle_count;
92
93 if (!job->bo_count)
94 return 0;
95
96 job->implicit_fences = kvmalloc_array(job->bo_count,
97 sizeof(struct dma_fence *),
98 GFP_KERNEL | __GFP_ZERO);
99 if (!job->implicit_fences)
100 return -ENOMEM;
101
102 return drm_gem_objects_lookup(file_priv,
103 (void __user *)(uintptr_t)args->bo_handles,
104 job->bo_count, &job->bos);
105}
106
107/**
108 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects
109 * referenced by the job.
110 * @dev: DRM device
111 * @file_priv: DRM file for this fd
112 * @args: IOCTL args
113 * @job: job being set up
114 *
115 * Resolve syncobjs from userspace to fences and attach them to job.
116 *
117 * Note that this function doesn't need to unreference the fences on
118 * failure, because that will happen at panfrost_job_cleanup() time.
119 */
120static int
121panfrost_copy_in_sync(struct drm_device *dev,
122 struct drm_file *file_priv,
123 struct drm_panfrost_submit *args,
124 struct panfrost_job *job)
125{
126 u32 *handles;
127 int ret = 0;
128 int i;
129
130 job->in_fence_count = args->in_sync_count;
131
132 if (!job->in_fence_count)
133 return 0;
134
135 job->in_fences = kvmalloc_array(job->in_fence_count,
136 sizeof(struct dma_fence *),
137 GFP_KERNEL | __GFP_ZERO);
138 if (!job->in_fences) {
139 DRM_DEBUG("Failed to allocate job in fences\n");
140 return -ENOMEM;
141 }
142
143 handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL);
144 if (!handles) {
145 ret = -ENOMEM;
146 DRM_DEBUG("Failed to allocate incoming syncobj handles\n");
147 goto fail;
148 }
149
150 if (copy_from_user(handles,
151 (void __user *)(uintptr_t)args->in_syncs,
152 job->in_fence_count * sizeof(u32))) {
153 ret = -EFAULT;
154 DRM_DEBUG("Failed to copy in syncobj handles\n");
155 goto fail;
156 }
157
158 for (i = 0; i < job->in_fence_count; i++) {
159 ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0,
160 &job->in_fences[i]);
161 if (ret == -EINVAL)
162 goto fail;
163 }
164
165fail:
166 kvfree(handles);
167 return ret;
168}
169
170static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
171 struct drm_file *file)
172{
173 struct panfrost_device *pfdev = dev->dev_private;
174 struct drm_panfrost_submit *args = data;
175 struct drm_syncobj *sync_out;
176 struct panfrost_job *job;
177 int ret = 0;
178
179 job = kzalloc(sizeof(*job), GFP_KERNEL);
180 if (!job)
181 return -ENOMEM;
182
183 kref_init(&job->refcount);
184
185 job->pfdev = pfdev;
186 job->jc = args->jc;
187 job->requirements = args->requirements;
188 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev);
189 job->file_priv = file->driver_priv;
190
191 ret = panfrost_copy_in_sync(dev, file, args, job);
192 if (ret)
193 goto fail;
194
195 ret = panfrost_lookup_bos(dev, file, args, job);
196 if (ret)
197 goto fail;
198
199 ret = panfrost_job_push(job);
200 if (ret)
201 goto fail;
202
203 /* Update the return sync object for the job */
204 sync_out = drm_syncobj_find(file, args->out_sync);
205 if (sync_out) {
206 drm_syncobj_replace_fence(sync_out, job->render_done_fence);
207 drm_syncobj_put(sync_out);
208 }
209
210fail:
211 panfrost_job_put(job);
212
213 return ret;
214}
215
216static int
217panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
218 struct drm_file *file_priv)
219{
220 long ret;
221 struct drm_panfrost_wait_bo *args = data;
222 struct drm_gem_object *gem_obj;
223 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
224
225 if (args->pad)
226 return -EINVAL;
227
228 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
229 if (!gem_obj)
230 return -ENOENT;
231
232 ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true,
233 true, timeout);
234 if (!ret)
235 ret = timeout ? -ETIMEDOUT : -EBUSY;
236
237 drm_gem_object_put_unlocked(gem_obj);
238
239 return ret;
240}
241
242static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data,
243 struct drm_file *file_priv)
244{
245 struct drm_panfrost_mmap_bo *args = data;
246 struct drm_gem_object *gem_obj;
247 int ret;
248
249 if (args->flags != 0) {
250 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
251 return -EINVAL;
252 }
253
254 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
255 if (!gem_obj) {
256 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
257 return -ENOENT;
258 }
259
260 ret = drm_gem_create_mmap_offset(gem_obj);
261 if (ret == 0)
262 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
263 drm_gem_object_put_unlocked(gem_obj);
264
265 return ret;
266}
267
268static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data,
269 struct drm_file *file_priv)
270{
271 struct drm_panfrost_get_bo_offset *args = data;
272 struct drm_gem_object *gem_obj;
273 struct panfrost_gem_object *bo;
274
275 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
276 if (!gem_obj) {
277 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
278 return -ENOENT;
279 }
280 bo = to_panfrost_bo(gem_obj);
281
282 args->offset = bo->node.start << PAGE_SHIFT;
283
284 drm_gem_object_put_unlocked(gem_obj);
285 return 0;
286}
287
288static int
289panfrost_open(struct drm_device *dev, struct drm_file *file)
290{
291 struct panfrost_device *pfdev = dev->dev_private;
292 struct panfrost_file_priv *panfrost_priv;
293
294 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL);
295 if (!panfrost_priv)
296 return -ENOMEM;
297
298 panfrost_priv->pfdev = pfdev;
299 file->driver_priv = panfrost_priv;
300
301 return panfrost_job_open(panfrost_priv);
302}
303
304static void
305panfrost_postclose(struct drm_device *dev, struct drm_file *file)
306{
307 struct panfrost_file_priv *panfrost_priv = file->driver_priv;
308
309 panfrost_job_close(panfrost_priv);
310
311 kfree(panfrost_priv);
312}
313
314/* DRM_AUTH is required on SUBMIT for now, while all clients share a single
315 * address space. Note that render nodes would be able to submit jobs that
316 * could access BOs from clients authenticated with the master node.
317 */
318static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
319#define PANFROST_IOCTL(n, func, flags) \
320 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags)
321
322 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW | DRM_AUTH),
323 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW),
324 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW),
325 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW),
326 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW),
327 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW),
328};
329
330DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops);
331
332static struct drm_driver panfrost_drm_driver = {
333 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME |
334 DRIVER_SYNCOBJ,
335 .open = panfrost_open,
336 .postclose = panfrost_postclose,
337 .ioctls = panfrost_drm_driver_ioctls,
338 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls),
339 .fops = &panfrost_drm_driver_fops,
340 .name = "panfrost",
341 .desc = "panfrost DRM",
342 .date = "20180908",
343 .major = 1,
344 .minor = 0,
345
346 .gem_create_object = panfrost_gem_create_object,
347 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
348 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
349 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
350 .gem_prime_mmap = drm_gem_prime_mmap,
351};
352
353static int panfrost_probe(struct platform_device *pdev)
354{
355 struct panfrost_device *pfdev;
356 struct drm_device *ddev;
357 int err;
358
359 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL);
360 if (!pfdev)
361 return -ENOMEM;
362
363 pfdev->pdev = pdev;
364 pfdev->dev = &pdev->dev;
365
366 platform_set_drvdata(pdev, pfdev);
367
368 /* Allocate and initialze the DRM device. */
369 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev);
370 if (IS_ERR(ddev))
371 return PTR_ERR(ddev);
372
373 ddev->dev_private = pfdev;
374 pfdev->ddev = ddev;
375
376 spin_lock_init(&pfdev->mm_lock);
377
378 /* 4G enough for now. can be 48-bit */
379 drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT);
380
381 pm_runtime_use_autosuspend(pfdev->dev);
382 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */
383 pm_runtime_enable(pfdev->dev);
384
385 err = panfrost_device_init(pfdev);
386 if (err) {
387 dev_err(&pdev->dev, "Fatal error during GPU init\n");
388 goto err_out0;
389 }
390
391 dma_set_mask_and_coherent(pfdev->dev,
392 DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features)));
393
394 err = panfrost_devfreq_init(pfdev);
395 if (err) {
396 dev_err(&pdev->dev, "Fatal error during devfreq init\n");
397 goto err_out1;
398 }
399
400 /*
401 * Register the DRM device with the core and the connectors with
402 * sysfs
403 */
404 err = drm_dev_register(ddev, 0);
405 if (err < 0)
406 goto err_out1;
407
408 return 0;
409
410err_out1:
411 panfrost_device_fini(pfdev);
412err_out0:
413 drm_dev_put(ddev);
414 return err;
415}
416
417static int panfrost_remove(struct platform_device *pdev)
418{
419 struct panfrost_device *pfdev = platform_get_drvdata(pdev);
420 struct drm_device *ddev = pfdev->ddev;
421
422 drm_dev_unregister(ddev);
423 pm_runtime_get_sync(pfdev->dev);
424 pm_runtime_put_sync_autosuspend(pfdev->dev);
425 pm_runtime_disable(pfdev->dev);
426 panfrost_device_fini(pfdev);
427 drm_dev_put(ddev);
428 return 0;
429}
430
431static const struct of_device_id dt_match[] = {
432 { .compatible = "arm,mali-t604" },
433 { .compatible = "arm,mali-t624" },
434 { .compatible = "arm,mali-t628" },
435 { .compatible = "arm,mali-t720" },
436 { .compatible = "arm,mali-t760" },
437 { .compatible = "arm,mali-t820" },
438 { .compatible = "arm,mali-t830" },
439 { .compatible = "arm,mali-t860" },
440 { .compatible = "arm,mali-t880" },
441 {}
442};
443MODULE_DEVICE_TABLE(of, dt_match);
444
445static const struct dev_pm_ops panfrost_pm_ops = {
446 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
447 SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL)
448};
449
450static struct platform_driver panfrost_driver = {
451 .probe = panfrost_probe,
452 .remove = panfrost_remove,
453 .driver = {
454 .name = "panfrost",
455 .pm = &panfrost_pm_ops,
456 .of_match_table = dt_match,
457 },
458};
459module_platform_driver(panfrost_driver);
460
461MODULE_AUTHOR("Panfrost Project Developers");
462MODULE_DESCRIPTION("Panfrost DRM Driver");
463MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panfrost/panfrost_features.h b/drivers/gpu/drm/panfrost/panfrost_features.h
new file mode 100644
index 000000000000..5056777c7744
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_features.h
@@ -0,0 +1,309 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. */
3/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4#ifndef __PANFROST_FEATURES_H__
5#define __PANFROST_FEATURES_H__
6
7#include <linux/bitops.h>
8
9#include "panfrost_device.h"
10
11enum panfrost_hw_feature {
12 HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
13 HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
14 HW_FEATURE_XAFFINITY,
15 HW_FEATURE_OUT_OF_ORDER_EXEC,
16 HW_FEATURE_MRT,
17 HW_FEATURE_BRNDOUT_CC,
18 HW_FEATURE_INTERPIPE_REG_ALIASING,
19 HW_FEATURE_LD_ST_TILEBUFFER,
20 HW_FEATURE_MSAA_16X,
21 HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
22 HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
23 HW_FEATURE_OPTIMIZED_COVERAGE_MASK,
24 HW_FEATURE_T7XX_PAIRING_RULES,
25 HW_FEATURE_LD_ST_LEA_TEX,
26 HW_FEATURE_LINEAR_FILTER_FLOAT,
27 HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4,
28 HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS,
29 HW_FEATURE_TEST4_DATUM_MODE,
30 HW_FEATURE_NEXT_INSTRUCTION_TYPE,
31 HW_FEATURE_BRNDOUT_KILL,
32 HW_FEATURE_WARPING,
33 HW_FEATURE_V4,
34 HW_FEATURE_FLUSH_REDUCTION,
35 HW_FEATURE_PROTECTED_MODE,
36 HW_FEATURE_COHERENCY_REG,
37 HW_FEATURE_PROTECTED_DEBUG_MODE,
38 HW_FEATURE_AARCH64_MMU,
39 HW_FEATURE_TLS_HASHING,
40 HW_FEATURE_THREAD_GROUP_SPLIT,
41 HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG,
42};
43
44#define hw_features_t600 (\
45 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
46 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
47 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
48 BIT_ULL(HW_FEATURE_V4))
49
50#define hw_features_t620 (\
51 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
52 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
53 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
54 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
55 BIT_ULL(HW_FEATURE_V4))
56
57#define hw_features_t720 (\
58 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
59 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
60 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
61 BIT_ULL(HW_FEATURE_OPTIMIZED_COVERAGE_MASK) | \
62 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
63 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
64 BIT_ULL(HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4) | \
65 BIT_ULL(HW_FEATURE_WARPING) | \
66 BIT_ULL(HW_FEATURE_V4))
67
68
69#define hw_features_t760 (\
70 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
71 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
72 BIT_ULL(HW_FEATURE_XAFFINITY) | \
73 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
74 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
75 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
76 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
77 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
78 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
79 BIT_ULL(HW_FEATURE_MRT) | \
80 BIT_ULL(HW_FEATURE_MSAA_16X) | \
81 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
82 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
83 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
84 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
85
86// T860
87#define hw_features_t860 (\
88 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
89 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
90 BIT_ULL(HW_FEATURE_XAFFINITY) | \
91 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
92 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
93 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
94 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
95 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
96 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
97 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
98 BIT_ULL(HW_FEATURE_MRT) | \
99 BIT_ULL(HW_FEATURE_MSAA_16X) | \
100 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
101 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
102 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
103 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
104 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
105
106#define hw_features_t880 hw_features_t860
107
108#define hw_features_t830 (\
109 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
110 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
111 BIT_ULL(HW_FEATURE_XAFFINITY) | \
112 BIT_ULL(HW_FEATURE_WARPING) | \
113 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
114 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
115 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
116 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
117 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
118 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
119 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
120 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
121 BIT_ULL(HW_FEATURE_MRT) | \
122 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
123 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
124 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
125 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
126 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
127
128#define hw_features_t820 (\
129 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
130 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
131 BIT_ULL(HW_FEATURE_XAFFINITY) | \
132 BIT_ULL(HW_FEATURE_WARPING) | \
133 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
134 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
135 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
136 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
137 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
138 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
139 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
140 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
141 BIT_ULL(HW_FEATURE_MRT) | \
142 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
143 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
144 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
145 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
146 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
147
148#define hw_features_g71 (\
149 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
150 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
151 BIT_ULL(HW_FEATURE_XAFFINITY) | \
152 BIT_ULL(HW_FEATURE_WARPING) | \
153 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
154 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
155 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
156 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
157 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
158 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
159 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
160 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
161 BIT_ULL(HW_FEATURE_MRT) | \
162 BIT_ULL(HW_FEATURE_MSAA_16X) | \
163 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
164 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
165 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
166 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
167 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
168 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
169 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
170 BIT_ULL(HW_FEATURE_COHERENCY_REG))
171
172#define hw_features_g72 (\
173 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
174 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
175 BIT_ULL(HW_FEATURE_XAFFINITY) | \
176 BIT_ULL(HW_FEATURE_WARPING) | \
177 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
178 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
179 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
180 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
181 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
182 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
183 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
184 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
185 BIT_ULL(HW_FEATURE_MRT) | \
186 BIT_ULL(HW_FEATURE_MSAA_16X) | \
187 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
188 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
189 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
190 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
191 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
192 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
193 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
194 BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
195 BIT_ULL(HW_FEATURE_COHERENCY_REG))
196
197#define hw_features_g51 (\
198 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
199 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
200 BIT_ULL(HW_FEATURE_XAFFINITY) | \
201 BIT_ULL(HW_FEATURE_WARPING) | \
202 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
203 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
204 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
205 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
206 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
207 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
208 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
209 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
210 BIT_ULL(HW_FEATURE_MRT) | \
211 BIT_ULL(HW_FEATURE_MSAA_16X) | \
212 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
213 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
214 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
215 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
216 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
217 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
218 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
219 BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
220 BIT_ULL(HW_FEATURE_COHERENCY_REG))
221
222#define hw_features_g52 (\
223 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
224 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
225 BIT_ULL(HW_FEATURE_XAFFINITY) | \
226 BIT_ULL(HW_FEATURE_WARPING) | \
227 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
228 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
229 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
230 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
231 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
232 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
233 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
234 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
235 BIT_ULL(HW_FEATURE_MRT) | \
236 BIT_ULL(HW_FEATURE_MSAA_16X) | \
237 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
238 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
239 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
240 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
241 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
242 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
243 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
244 BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
245 BIT_ULL(HW_FEATURE_COHERENCY_REG))
246
247#define hw_features_g76 (\
248 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
249 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
250 BIT_ULL(HW_FEATURE_XAFFINITY) | \
251 BIT_ULL(HW_FEATURE_WARPING) | \
252 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
253 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
254 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
255 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
256 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
257 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
258 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
259 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
260 BIT_ULL(HW_FEATURE_MRT) | \
261 BIT_ULL(HW_FEATURE_MSAA_16X) | \
262 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
263 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
264 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
265 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
266 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
267 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
268 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
269 BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
270 BIT_ULL(HW_FEATURE_COHERENCY_REG) | \
271 BIT_ULL(HW_FEATURE_AARCH64_MMU) | \
272 BIT_ULL(HW_FEATURE_TLS_HASHING) | \
273 BIT_ULL(HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
274
275#define hw_features_g31 (\
276 BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
277 BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
278 BIT_ULL(HW_FEATURE_XAFFINITY) | \
279 BIT_ULL(HW_FEATURE_WARPING) | \
280 BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
281 BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
282 BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
283 BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
284 BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
285 BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
286 BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
287 BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
288 BIT_ULL(HW_FEATURE_MRT) | \
289 BIT_ULL(HW_FEATURE_MSAA_16X) | \
290 BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
291 BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
292 BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
293 BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
294 BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
295 BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
296 BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
297 BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
298 BIT_ULL(HW_FEATURE_COHERENCY_REG) | \
299 BIT_ULL(HW_FEATURE_AARCH64_MMU) | \
300 BIT_ULL(HW_FEATURE_TLS_HASHING) | \
301 BIT_ULL(HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
302
303static inline bool panfrost_has_hw_feature(struct panfrost_device *pfdev,
304 enum panfrost_hw_feature feat)
305{
306 return test_bit(feat, pfdev->features.hw_features);
307}
308
309#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
new file mode 100644
index 000000000000..8a0376283a21
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -0,0 +1,95 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3
4#include <linux/err.h>
5#include <linux/slab.h>
6#include <linux/dma-buf.h>
7#include <linux/dma-mapping.h>
8
9#include <drm/panfrost_drm.h>
10#include "panfrost_device.h"
11#include "panfrost_gem.h"
12#include "panfrost_mmu.h"
13
14/* Called DRM core on the last userspace/kernel unreference of the
15 * BO.
16 */
17void panfrost_gem_free_object(struct drm_gem_object *obj)
18{
19 struct panfrost_gem_object *bo = to_panfrost_bo(obj);
20 struct panfrost_device *pfdev = obj->dev->dev_private;
21
22 panfrost_mmu_unmap(bo);
23
24 spin_lock(&pfdev->mm_lock);
25 drm_mm_remove_node(&bo->node);
26 spin_unlock(&pfdev->mm_lock);
27
28 drm_gem_shmem_free_object(obj);
29}
30
31static const struct drm_gem_object_funcs panfrost_gem_funcs = {
32 .free = panfrost_gem_free_object,
33 .print_info = drm_gem_shmem_print_info,
34 .pin = drm_gem_shmem_pin,
35 .unpin = drm_gem_shmem_unpin,
36 .get_sg_table = drm_gem_shmem_get_sg_table,
37 .vmap = drm_gem_shmem_vmap,
38 .vunmap = drm_gem_shmem_vunmap,
39 .vm_ops = &drm_gem_shmem_vm_ops,
40};
41
42/**
43 * panfrost_gem_create_object - Implementation of driver->gem_create_object.
44 * @dev: DRM device
45 * @size: Size in bytes of the memory the object will reference
46 *
47 * This lets the GEM helpers allocate object structs for us, and keep
48 * our BO stats correct.
49 */
50struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
51{
52 int ret;
53 struct panfrost_device *pfdev = dev->dev_private;
54 struct panfrost_gem_object *obj;
55
56 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
57 if (!obj)
58 return NULL;
59
60 obj->base.base.funcs = &panfrost_gem_funcs;
61
62 spin_lock(&pfdev->mm_lock);
63 ret = drm_mm_insert_node(&pfdev->mm, &obj->node,
64 roundup(size, PAGE_SIZE) >> PAGE_SHIFT);
65 spin_unlock(&pfdev->mm_lock);
66 if (ret)
67 goto free_obj;
68
69 return &obj->base.base;
70
71free_obj:
72 kfree(obj);
73 return ERR_PTR(ret);
74}
75
76struct drm_gem_object *
77panfrost_gem_prime_import_sg_table(struct drm_device *dev,
78 struct dma_buf_attachment *attach,
79 struct sg_table *sgt)
80{
81 struct drm_gem_object *obj;
82 struct panfrost_gem_object *pobj;
83
84 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
85 if (IS_ERR(obj))
86 return ERR_CAST(obj);
87
88 pobj = to_panfrost_bo(obj);
89
90 obj->resv = attach->dmabuf->resv;
91
92 panfrost_mmu_map(pobj);
93
94 return obj;
95}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h
new file mode 100644
index 000000000000..045000eb5fcf
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.h
@@ -0,0 +1,29 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3
4#ifndef __PANFROST_GEM_H__
5#define __PANFROST_GEM_H__
6
7#include <drm/drm_gem_shmem_helper.h>
8#include <drm/drm_mm.h>
9
10struct panfrost_gem_object {
11 struct drm_gem_shmem_object base;
12
13 struct drm_mm_node node;
14};
15
16static inline
17struct panfrost_gem_object *to_panfrost_bo(struct drm_gem_object *obj)
18{
19 return container_of(to_drm_gem_shmem_obj(obj), struct panfrost_gem_object, base);
20}
21
22struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size);
23
24struct drm_gem_object *
25panfrost_gem_prime_import_sg_table(struct drm_device *dev,
26 struct dma_buf_attachment *attach,
27 struct sg_table *sgt);
28
29#endif /* __PANFROST_GEM_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
new file mode 100644
index 000000000000..aceaf6e44a09
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -0,0 +1,362 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4/* Copyright 2019 Collabora ltd. */
5#include <linux/bitmap.h>
6#include <linux/delay.h>
7#include <linux/interrupt.h>
8#include <linux/io.h>
9#include <linux/iopoll.h>
10#include <linux/platform_device.h>
11
12#include "panfrost_device.h"
13#include "panfrost_features.h"
14#include "panfrost_issues.h"
15#include "panfrost_gpu.h"
16#include "panfrost_regs.h"
17
18#define gpu_write(dev, reg, data) writel(data, dev->iomem + reg)
19#define gpu_read(dev, reg) readl(dev->iomem + reg)
20
21static irqreturn_t panfrost_gpu_irq_handler(int irq, void *data)
22{
23 struct panfrost_device *pfdev = data;
24 u32 state = gpu_read(pfdev, GPU_INT_STAT);
25 u32 fault_status = gpu_read(pfdev, GPU_FAULT_STATUS);
26
27 if (!state)
28 return IRQ_NONE;
29
30 if (state & GPU_IRQ_MASK_ERROR) {
31 u64 address = (u64) gpu_read(pfdev, GPU_FAULT_ADDRESS_HI) << 32;
32 address |= gpu_read(pfdev, GPU_FAULT_ADDRESS_LO);
33
34 dev_warn(pfdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n",
35 fault_status & 0xFF, panfrost_exception_name(pfdev, fault_status),
36 address);
37
38 if (state & GPU_IRQ_MULTIPLE_FAULT)
39 dev_warn(pfdev->dev, "There were multiple GPU faults - some have not been reported\n");
40
41 gpu_write(pfdev, GPU_INT_MASK, 0);
42 }
43
44 gpu_write(pfdev, GPU_INT_CLEAR, state);
45
46 return IRQ_HANDLED;
47}
48
49int panfrost_gpu_soft_reset(struct panfrost_device *pfdev)
50{
51 int ret;
52 u32 val;
53
54 gpu_write(pfdev, GPU_INT_MASK, 0);
55 gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
56 gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
57
58 ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,
59 val, val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
60
61 if (ret) {
62 dev_err(pfdev->dev, "gpu soft reset timed out\n");
63 return ret;
64 }
65
66 gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_MASK_ALL);
67 gpu_write(pfdev, GPU_INT_MASK, GPU_IRQ_MASK_ALL);
68
69 return 0;
70}
71
72static void panfrost_gpu_init_quirks(struct panfrost_device *pfdev)
73{
74 u32 quirks = 0;
75
76 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8443) ||
77 panfrost_has_hw_issue(pfdev, HW_ISSUE_11035))
78 quirks |= SC_LS_PAUSEBUFFER_DISABLE;
79
80 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10327))
81 quirks |= SC_SDC_DISABLE_OQ_DISCARD;
82
83 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10797))
84 quirks |= SC_ENABLE_TEXGRD_FLAGS;
85
86 if (!panfrost_has_hw_issue(pfdev, GPUCORE_1619)) {
87 if (panfrost_model_cmp(pfdev, 0x750) < 0) /* T60x, T62x, T72x */
88 quirks |= SC_LS_ATTR_CHECK_DISABLE;
89 else if (panfrost_model_cmp(pfdev, 0x880) <= 0) /* T76x, T8xx */
90 quirks |= SC_LS_ALLOW_ATTR_TYPES;
91 }
92
93 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_TLS_HASHING))
94 quirks |= SC_TLS_HASH_ENABLE;
95
96 if (quirks)
97 gpu_write(pfdev, GPU_SHADER_CONFIG, quirks);
98
99
100 quirks = gpu_read(pfdev, GPU_TILER_CONFIG);
101
102 /* Set tiler clock gate override if required */
103 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_T76X_3953))
104 quirks |= TC_CLOCK_GATE_OVERRIDE;
105
106 gpu_write(pfdev, GPU_TILER_CONFIG, quirks);
107
108
109 quirks = gpu_read(pfdev, GPU_L2_MMU_CONFIG);
110
111 /* Limit read & write ID width for AXI */
112 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
113 quirks &= ~(L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS |
114 L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES);
115 else
116 quirks &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS |
117 L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
118
119 gpu_write(pfdev, GPU_L2_MMU_CONFIG, quirks);
120
121 quirks = 0;
122 if ((panfrost_model_eq(pfdev, 0x860) || panfrost_model_eq(pfdev, 0x880)) &&
123 pfdev->features.revision >= 0x2000)
124 quirks |= JM_MAX_JOB_THROTTLE_LIMIT << JM_JOB_THROTTLE_LIMIT_SHIFT;
125 else if (panfrost_model_eq(pfdev, 0x6000) &&
126 pfdev->features.coherency_features == COHERENCY_ACE)
127 quirks |= (COHERENCY_ACE_LITE | COHERENCY_ACE) <<
128 JM_FORCE_COHERENCY_FEATURES_SHIFT;
129
130 if (quirks)
131 gpu_write(pfdev, GPU_JM_CONFIG, quirks);
132}
133
134#define MAX_HW_REVS 6
135
136struct panfrost_model {
137 const char *name;
138 u32 id;
139 u32 id_mask;
140 u64 features;
141 u64 issues;
142 struct {
143 u32 revision;
144 u64 issues;
145 } revs[MAX_HW_REVS];
146};
147
148#define GPU_MODEL(_name, _id, ...) \
149{\
150 .name = __stringify(_name), \
151 .id = _id, \
152 .features = hw_features_##_name, \
153 .issues = hw_issues_##_name, \
154 .revs = { __VA_ARGS__ }, \
155}
156
157#define GPU_REV_EXT(name, _rev, _p, _s, stat) \
158{\
159 .revision = (_rev) << 12 | (_p) << 4 | (_s), \
160 .issues = hw_issues_##name##_r##_rev##p##_p##stat, \
161}
162#define GPU_REV(name, r, p) GPU_REV_EXT(name, r, p, 0, )
163
164static const struct panfrost_model gpu_models[] = {
165 /* T60x has an oddball version */
166 GPU_MODEL(t600, 0x600,
167 GPU_REV_EXT(t600, 0, 0, 1, _15dev0)),
168 GPU_MODEL(t620, 0x620,
169 GPU_REV(t620, 0, 1), GPU_REV(t620, 1, 0)),
170 GPU_MODEL(t720, 0x720),
171 GPU_MODEL(t760, 0x750,
172 GPU_REV(t760, 0, 0), GPU_REV(t760, 0, 1),
173 GPU_REV_EXT(t760, 0, 1, 0, _50rel0),
174 GPU_REV(t760, 0, 2), GPU_REV(t760, 0, 3)),
175 GPU_MODEL(t820, 0x820),
176 GPU_MODEL(t830, 0x830),
177 GPU_MODEL(t860, 0x860),
178 GPU_MODEL(t880, 0x880),
179
180 GPU_MODEL(g71, 0x6000,
181 GPU_REV_EXT(g71, 0, 0, 1, _05dev0)),
182 GPU_MODEL(g72, 0x6001),
183 GPU_MODEL(g51, 0x7000),
184 GPU_MODEL(g76, 0x7001),
185 GPU_MODEL(g52, 0x7002),
186 GPU_MODEL(g31, 0x7003,
187 GPU_REV(g31, 1, 0)),
188};
189
190static void panfrost_gpu_init_features(struct panfrost_device *pfdev)
191{
192 u32 gpu_id, num_js, major, minor, status, rev;
193 const char *name = "unknown";
194 u64 hw_feat = 0;
195 u64 hw_issues = hw_issues_all;
196 const struct panfrost_model *model;
197 int i;
198
199 pfdev->features.l2_features = gpu_read(pfdev, GPU_L2_FEATURES);
200 pfdev->features.core_features = gpu_read(pfdev, GPU_CORE_FEATURES);
201 pfdev->features.tiler_features = gpu_read(pfdev, GPU_TILER_FEATURES);
202 pfdev->features.mem_features = gpu_read(pfdev, GPU_MEM_FEATURES);
203 pfdev->features.mmu_features = gpu_read(pfdev, GPU_MMU_FEATURES);
204 pfdev->features.thread_features = gpu_read(pfdev, GPU_THREAD_FEATURES);
205 pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);
206 for (i = 0; i < 4; i++)
207 pfdev->features.texture_features[i] = gpu_read(pfdev, GPU_TEXTURE_FEATURES(i));
208
209 pfdev->features.as_present = gpu_read(pfdev, GPU_AS_PRESENT);
210
211 pfdev->features.js_present = gpu_read(pfdev, GPU_JS_PRESENT);
212 num_js = hweight32(pfdev->features.js_present);
213 for (i = 0; i < num_js; i++)
214 pfdev->features.js_features[i] = gpu_read(pfdev, GPU_JS_FEATURES(i));
215
216 pfdev->features.shader_present = gpu_read(pfdev, GPU_SHADER_PRESENT_LO);
217 pfdev->features.shader_present |= (u64)gpu_read(pfdev, GPU_SHADER_PRESENT_HI) << 32;
218
219 pfdev->features.tiler_present = gpu_read(pfdev, GPU_TILER_PRESENT_LO);
220 pfdev->features.tiler_present |= (u64)gpu_read(pfdev, GPU_TILER_PRESENT_HI) << 32;
221
222 pfdev->features.l2_present = gpu_read(pfdev, GPU_L2_PRESENT_LO);
223 pfdev->features.l2_present |= (u64)gpu_read(pfdev, GPU_L2_PRESENT_HI) << 32;
224 pfdev->features.nr_core_groups = hweight64(pfdev->features.l2_present);
225
226 pfdev->features.stack_present = gpu_read(pfdev, GPU_STACK_PRESENT_LO);
227 pfdev->features.stack_present |= (u64)gpu_read(pfdev, GPU_STACK_PRESENT_HI) << 32;
228
229 gpu_id = gpu_read(pfdev, GPU_ID);
230 pfdev->features.revision = gpu_id & 0xffff;
231 pfdev->features.id = gpu_id >> 16;
232
233 /* The T60x has an oddball ID value. Fix it up to the standard Midgard
234 * format so we (and userspace) don't have to special case it.
235 */
236 if (pfdev->features.id == 0x6956)
237 pfdev->features.id = 0x0600;
238
239 major = (pfdev->features.revision >> 12) & 0xf;
240 minor = (pfdev->features.revision >> 4) & 0xff;
241 status = pfdev->features.revision & 0xf;
242 rev = pfdev->features.revision;
243
244 gpu_id = pfdev->features.id;
245
246 for (model = gpu_models; model->name; model++) {
247 int best = -1;
248
249 if (!panfrost_model_eq(pfdev, model->id))
250 continue;
251
252 name = model->name;
253 hw_feat = model->features;
254 hw_issues |= model->issues;
255 for (i = 0; i < MAX_HW_REVS; i++) {
256 if (model->revs[i].revision == rev) {
257 best = i;
258 break;
259 } else if (model->revs[i].revision == (rev & ~0xf))
260 best = i;
261 }
262
263 if (best >= 0)
264 hw_issues |= model->revs[best].issues;
265
266 break;
267 }
268
269 bitmap_from_u64(pfdev->features.hw_features, hw_feat);
270 bitmap_from_u64(pfdev->features.hw_issues, hw_issues);
271
272 dev_info(pfdev->dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x",
273 name, gpu_id, major, minor, status);
274 dev_info(pfdev->dev, "features: %64pb, issues: %64pb",
275 pfdev->features.hw_features,
276 pfdev->features.hw_issues);
277
278 dev_info(pfdev->dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x JS:0x%x",
279 gpu_read(pfdev, GPU_L2_FEATURES),
280 gpu_read(pfdev, GPU_CORE_FEATURES),
281 gpu_read(pfdev, GPU_TILER_FEATURES),
282 gpu_read(pfdev, GPU_MEM_FEATURES),
283 gpu_read(pfdev, GPU_MMU_FEATURES),
284 gpu_read(pfdev, GPU_AS_PRESENT),
285 gpu_read(pfdev, GPU_JS_PRESENT));
286
287 dev_info(pfdev->dev, "shader_present=0x%0llx l2_present=0x%0llx",
288 pfdev->features.shader_present, pfdev->features.l2_present);
289}
290
291void panfrost_gpu_power_on(struct panfrost_device *pfdev)
292{
293 int ret;
294 u32 val;
295
296 /* Just turn on everything for now */
297 gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present);
298 ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,
299 val, val == pfdev->features.l2_present, 100, 1000);
300
301 gpu_write(pfdev, STACK_PWRON_LO, pfdev->features.stack_present);
302 ret |= readl_relaxed_poll_timeout(pfdev->iomem + STACK_READY_LO,
303 val, val == pfdev->features.stack_present, 100, 1000);
304
305 gpu_write(pfdev, SHADER_PWRON_LO, pfdev->features.shader_present);
306 ret |= readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,
307 val, val == pfdev->features.shader_present, 100, 1000);
308
309 gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);
310 ret |= readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,
311 val, val == pfdev->features.tiler_present, 100, 1000);
312
313 if (ret)
314 dev_err(pfdev->dev, "error powering up gpu");
315}
316
317void panfrost_gpu_power_off(struct panfrost_device *pfdev)
318{
319 gpu_write(pfdev, TILER_PWROFF_LO, 0);
320 gpu_write(pfdev, SHADER_PWROFF_LO, 0);
321 gpu_write(pfdev, STACK_PWROFF_LO, 0);
322 gpu_write(pfdev, L2_PWROFF_LO, 0);
323}
324
325int panfrost_gpu_init(struct panfrost_device *pfdev)
326{
327 int err, irq;
328
329 err = panfrost_gpu_soft_reset(pfdev);
330 if (err)
331 return err;
332
333 panfrost_gpu_init_features(pfdev);
334
335 irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu");
336 if (irq <= 0)
337 return -ENODEV;
338
339 err = devm_request_irq(pfdev->dev, irq, panfrost_gpu_irq_handler,
340 IRQF_SHARED, "gpu", pfdev);
341 if (err) {
342 dev_err(pfdev->dev, "failed to request gpu irq");
343 return err;
344 }
345
346 panfrost_gpu_init_quirks(pfdev);
347 panfrost_gpu_power_on(pfdev);
348
349 return 0;
350}
351
352void panfrost_gpu_fini(struct panfrost_device *pfdev)
353{
354 panfrost_gpu_power_off(pfdev);
355}
356
357u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev)
358{
359 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
360 return gpu_read(pfdev, GPU_LATEST_FLUSH_ID);
361 return 0;
362}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.h b/drivers/gpu/drm/panfrost/panfrost_gpu.h
new file mode 100644
index 000000000000..4112412087b2
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.h
@@ -0,0 +1,19 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Collabora ltd. */
4
5#ifndef __PANFROST_GPU_H__
6#define __PANFROST_GPU_H__
7
8struct panfrost_device;
9
10int panfrost_gpu_init(struct panfrost_device *pfdev);
11void panfrost_gpu_fini(struct panfrost_device *pfdev);
12
13u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev);
14
15int panfrost_gpu_soft_reset(struct panfrost_device *pfdev);
16void panfrost_gpu_power_on(struct panfrost_device *pfdev);
17void panfrost_gpu_power_off(struct panfrost_device *pfdev);
18
19#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_issues.h b/drivers/gpu/drm/panfrost/panfrost_issues.h
new file mode 100644
index 000000000000..cec6dcdadb5c
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_issues.h
@@ -0,0 +1,176 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. */
3/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4#ifndef __PANFROST_ISSUES_H__
5#define __PANFROST_ISSUES_H__
6
7#include <linux/bitops.h>
8
9#include "panfrost_device.h"
10
11/*
12 * This is not a complete list of issues, but only the ones the driver needs
13 * to care about.
14 */
15enum panfrost_hw_issue {
16 HW_ISSUE_6367,
17 HW_ISSUE_6787,
18 HW_ISSUE_8186,
19 HW_ISSUE_8245,
20 HW_ISSUE_8316,
21 HW_ISSUE_8394,
22 HW_ISSUE_8401,
23 HW_ISSUE_8408,
24 HW_ISSUE_8443,
25 HW_ISSUE_8987,
26 HW_ISSUE_9435,
27 HW_ISSUE_9510,
28 HW_ISSUE_9630,
29 HW_ISSUE_10327,
30 HW_ISSUE_10649,
31 HW_ISSUE_10676,
32 HW_ISSUE_10797,
33 HW_ISSUE_10817,
34 HW_ISSUE_10883,
35 HW_ISSUE_10959,
36 HW_ISSUE_10969,
37 HW_ISSUE_11020,
38 HW_ISSUE_11024,
39 HW_ISSUE_11035,
40 HW_ISSUE_11056,
41 HW_ISSUE_T76X_3542,
42 HW_ISSUE_T76X_3953,
43 HW_ISSUE_TMIX_8463,
44 GPUCORE_1619,
45 HW_ISSUE_TMIX_8438,
46 HW_ISSUE_TGOX_R1_1234,
47 HW_ISSUE_END
48};
49
50#define hw_issues_all (\
51 BIT_ULL(HW_ISSUE_9435))
52
53#define hw_issues_t600 (\
54 BIT_ULL(HW_ISSUE_6367) | \
55 BIT_ULL(HW_ISSUE_6787) | \
56 BIT_ULL(HW_ISSUE_8408) | \
57 BIT_ULL(HW_ISSUE_9510) | \
58 BIT_ULL(HW_ISSUE_10649) | \
59 BIT_ULL(HW_ISSUE_10676) | \
60 BIT_ULL(HW_ISSUE_10883) | \
61 BIT_ULL(HW_ISSUE_11020) | \
62 BIT_ULL(HW_ISSUE_11035) | \
63 BIT_ULL(HW_ISSUE_11056) | \
64 BIT_ULL(HW_ISSUE_TMIX_8438))
65
66#define hw_issues_t600_r0p0_15dev0 (\
67 BIT_ULL(HW_ISSUE_8186) | \
68 BIT_ULL(HW_ISSUE_8245) | \
69 BIT_ULL(HW_ISSUE_8316) | \
70 BIT_ULL(HW_ISSUE_8394) | \
71 BIT_ULL(HW_ISSUE_8401) | \
72 BIT_ULL(HW_ISSUE_8443) | \
73 BIT_ULL(HW_ISSUE_8987) | \
74 BIT_ULL(HW_ISSUE_9630) | \
75 BIT_ULL(HW_ISSUE_10969) | \
76 BIT_ULL(GPUCORE_1619))
77
78#define hw_issues_t620 (\
79 BIT_ULL(HW_ISSUE_10649) | \
80 BIT_ULL(HW_ISSUE_10883) | \
81 BIT_ULL(HW_ISSUE_10959) | \
82 BIT_ULL(HW_ISSUE_11056) | \
83 BIT_ULL(HW_ISSUE_TMIX_8438))
84
85#define hw_issues_t620_r0p1 (\
86 BIT_ULL(HW_ISSUE_10327) | \
87 BIT_ULL(HW_ISSUE_10676) | \
88 BIT_ULL(HW_ISSUE_10817) | \
89 BIT_ULL(HW_ISSUE_11020) | \
90 BIT_ULL(HW_ISSUE_11024) | \
91 BIT_ULL(HW_ISSUE_11035))
92
93#define hw_issues_t620_r1p0 (\
94 BIT_ULL(HW_ISSUE_11020) | \
95 BIT_ULL(HW_ISSUE_11024))
96
97#define hw_issues_t720 (\
98 BIT_ULL(HW_ISSUE_10649) | \
99 BIT_ULL(HW_ISSUE_10797) | \
100 BIT_ULL(HW_ISSUE_10883) | \
101 BIT_ULL(HW_ISSUE_11056) | \
102 BIT_ULL(HW_ISSUE_TMIX_8438))
103
104#define hw_issues_t760 (\
105 BIT_ULL(HW_ISSUE_10883) | \
106 BIT_ULL(HW_ISSUE_T76X_3953) | \
107 BIT_ULL(HW_ISSUE_TMIX_8438))
108
109#define hw_issues_t760_r0p0 (\
110 BIT_ULL(HW_ISSUE_11020) | \
111 BIT_ULL(HW_ISSUE_11024) | \
112 BIT_ULL(HW_ISSUE_T76X_3542))
113
114#define hw_issues_t760_r0p1 (\
115 BIT_ULL(HW_ISSUE_11020) | \
116 BIT_ULL(HW_ISSUE_11024) | \
117 BIT_ULL(HW_ISSUE_T76X_3542))
118
119#define hw_issues_t760_r0p1_50rel0 (\
120 BIT_ULL(HW_ISSUE_T76X_3542))
121
122#define hw_issues_t760_r0p2 (\
123 BIT_ULL(HW_ISSUE_11020) | \
124 BIT_ULL(HW_ISSUE_11024) | \
125 BIT_ULL(HW_ISSUE_T76X_3542))
126
127#define hw_issues_t760_r0p3 (\
128 BIT_ULL(HW_ISSUE_T76X_3542))
129
130#define hw_issues_t820 (\
131 BIT_ULL(HW_ISSUE_10883) | \
132 BIT_ULL(HW_ISSUE_T76X_3953) | \
133 BIT_ULL(HW_ISSUE_TMIX_8438))
134
135#define hw_issues_t830 (\
136 BIT_ULL(HW_ISSUE_10883) | \
137 BIT_ULL(HW_ISSUE_T76X_3953) | \
138 BIT_ULL(HW_ISSUE_TMIX_8438))
139
140#define hw_issues_t860 (\
141 BIT_ULL(HW_ISSUE_10883) | \
142 BIT_ULL(HW_ISSUE_T76X_3953) | \
143 BIT_ULL(HW_ISSUE_TMIX_8438))
144
145#define hw_issues_t880 (\
146 BIT_ULL(HW_ISSUE_10883) | \
147 BIT_ULL(HW_ISSUE_T76X_3953) | \
148 BIT_ULL(HW_ISSUE_TMIX_8438))
149
150#define hw_issues_g31 0
151
152#define hw_issues_g31_r1p0 (\
153 BIT_ULL(HW_ISSUE_TGOX_R1_1234))
154
155#define hw_issues_g51 0
156
157#define hw_issues_g52 0
158
159#define hw_issues_g71 (\
160 BIT_ULL(HW_ISSUE_TMIX_8463) | \
161 BIT_ULL(HW_ISSUE_TMIX_8438))
162
163#define hw_issues_g71_r0p0_05dev0 (\
164 BIT_ULL(HW_ISSUE_T76X_3953))
165
166#define hw_issues_g72 0
167
168#define hw_issues_g76 0
169
170static inline bool panfrost_has_hw_issue(struct panfrost_device *pfdev,
171 enum panfrost_hw_issue issue)
172{
173 return test_bit(issue, pfdev->features.hw_issues);
174}
175
176#endif /* __PANFROST_ISSUES_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
new file mode 100644
index 000000000000..0a7ed04f7d52
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -0,0 +1,560 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3/* Copyright 2019 Collabora ltd. */
4#include <linux/delay.h>
5#include <linux/interrupt.h>
6#include <linux/io.h>
7#include <linux/platform_device.h>
8#include <linux/pm_runtime.h>
9#include <linux/reservation.h>
10#include <drm/gpu_scheduler.h>
11#include <drm/panfrost_drm.h>
12
13#include "panfrost_device.h"
14#include "panfrost_devfreq.h"
15#include "panfrost_job.h"
16#include "panfrost_features.h"
17#include "panfrost_issues.h"
18#include "panfrost_gem.h"
19#include "panfrost_regs.h"
20#include "panfrost_gpu.h"
21#include "panfrost_mmu.h"
22
23#define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
24#define job_read(dev, reg) readl(dev->iomem + (reg))
25
26struct panfrost_queue_state {
27 struct drm_gpu_scheduler sched;
28
29 u64 fence_context;
30 u64 emit_seqno;
31};
32
33struct panfrost_job_slot {
34 struct panfrost_queue_state queue[NUM_JOB_SLOTS];
35 spinlock_t job_lock;
36};
37
38static struct panfrost_job *
39to_panfrost_job(struct drm_sched_job *sched_job)
40{
41 return container_of(sched_job, struct panfrost_job, base);
42}
43
44struct panfrost_fence {
45 struct dma_fence base;
46 struct drm_device *dev;
47 /* panfrost seqno for signaled() test */
48 u64 seqno;
49 int queue;
50};
51
52static inline struct panfrost_fence *
53to_panfrost_fence(struct dma_fence *fence)
54{
55 return (struct panfrost_fence *)fence;
56}
57
58static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)
59{
60 return "panfrost";
61}
62
63static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)
64{
65 struct panfrost_fence *f = to_panfrost_fence(fence);
66
67 switch (f->queue) {
68 case 0:
69 return "panfrost-js-0";
70 case 1:
71 return "panfrost-js-1";
72 case 2:
73 return "panfrost-js-2";
74 default:
75 return NULL;
76 }
77}
78
79static const struct dma_fence_ops panfrost_fence_ops = {
80 .get_driver_name = panfrost_fence_get_driver_name,
81 .get_timeline_name = panfrost_fence_get_timeline_name,
82};
83
84static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
85{
86 struct panfrost_fence *fence;
87 struct panfrost_job_slot *js = pfdev->js;
88
89 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
90 if (!fence)
91 return ERR_PTR(-ENOMEM);
92
93 fence->dev = pfdev->ddev;
94 fence->queue = js_num;
95 fence->seqno = ++js->queue[js_num].emit_seqno;
96 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
97 js->queue[js_num].fence_context, fence->seqno);
98
99 return &fence->base;
100}
101
102static int panfrost_job_get_slot(struct panfrost_job *job)
103{
104 /* JS0: fragment jobs.
105 * JS1: vertex/tiler jobs
106 * JS2: compute jobs
107 */
108 if (job->requirements & PANFROST_JD_REQ_FS)
109 return 0;
110
111/* Not exposed to userspace yet */
112#if 0
113 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
114 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
115 (job->pfdev->features.nr_core_groups == 2))
116 return 2;
117 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
118 return 2;
119 }
120#endif
121 return 1;
122}
123
124static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
125 u32 requirements,
126 int js)
127{
128 u64 affinity;
129
130 /*
131 * Use all cores for now.
132 * Eventually we may need to support tiler only jobs and h/w with
133 * multiple (2) coherent core groups
134 */
135 affinity = pfdev->features.shader_present;
136
137 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), affinity & 0xFFFFFFFF);
138 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32);
139}
140
141static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
142{
143 struct panfrost_device *pfdev = job->pfdev;
144 unsigned long flags;
145 u32 cfg;
146 u64 jc_head = job->jc;
147 int ret;
148
149 ret = pm_runtime_get_sync(pfdev->dev);
150 if (ret < 0)
151 return;
152
153 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js))))
154 goto end;
155
156 panfrost_devfreq_record_transition(pfdev, js);
157 spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
158
159 job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);
160 job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32);
161
162 panfrost_job_write_affinity(pfdev, job->requirements, js);
163
164 /* start MMU, medium priority, cache clean/flush on end, clean/flush on
165 * start */
166 /* TODO: different address spaces */
167 cfg = JS_CONFIG_THREAD_PRI(8) |
168 JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
169 JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
170
171 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
172 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
173
174 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
175 cfg |= JS_CONFIG_START_MMU;
176
177 job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
178
179 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
180 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
181
182 /* GO ! */
183 dev_dbg(pfdev->dev, "JS: Submitting atom %p to js[%d] with head=0x%llx",
184 job, js, jc_head);
185
186 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
187
188 spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
189
190end:
191 pm_runtime_mark_last_busy(pfdev->dev);
192 pm_runtime_put_autosuspend(pfdev->dev);
193}
194
195static void panfrost_acquire_object_fences(struct drm_gem_object **bos,
196 int bo_count,
197 struct dma_fence **implicit_fences)
198{
199 int i;
200
201 for (i = 0; i < bo_count; i++)
202 implicit_fences[i] = reservation_object_get_excl_rcu(bos[i]->resv);
203}
204
205static void panfrost_attach_object_fences(struct drm_gem_object **bos,
206 int bo_count,
207 struct dma_fence *fence)
208{
209 int i;
210
211 for (i = 0; i < bo_count; i++)
212 reservation_object_add_excl_fence(bos[i]->resv, fence);
213}
214
215int panfrost_job_push(struct panfrost_job *job)
216{
217 struct panfrost_device *pfdev = job->pfdev;
218 int slot = panfrost_job_get_slot(job);
219 struct drm_sched_entity *entity = &job->file_priv->sched_entity[slot];
220 struct ww_acquire_ctx acquire_ctx;
221 int ret = 0;
222
223 mutex_lock(&pfdev->sched_lock);
224
225 ret = drm_gem_lock_reservations(job->bos, job->bo_count,
226 &acquire_ctx);
227 if (ret) {
228 mutex_unlock(&pfdev->sched_lock);
229 return ret;
230 }
231
232 ret = drm_sched_job_init(&job->base, entity, NULL);
233 if (ret) {
234 mutex_unlock(&pfdev->sched_lock);
235 goto unlock;
236 }
237
238 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
239
240 kref_get(&job->refcount); /* put by scheduler job completion */
241
242 panfrost_acquire_object_fences(job->bos, job->bo_count,
243 job->implicit_fences);
244
245 drm_sched_entity_push_job(&job->base, entity);
246
247 mutex_unlock(&pfdev->sched_lock);
248
249 panfrost_attach_object_fences(job->bos, job->bo_count,
250 job->render_done_fence);
251
252unlock:
253 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
254
255 return ret;
256}
257
258static void panfrost_job_cleanup(struct kref *ref)
259{
260 struct panfrost_job *job = container_of(ref, struct panfrost_job,
261 refcount);
262 unsigned int i;
263
264 if (job->in_fences) {
265 for (i = 0; i < job->in_fence_count; i++)
266 dma_fence_put(job->in_fences[i]);
267 kvfree(job->in_fences);
268 }
269 if (job->implicit_fences) {
270 for (i = 0; i < job->bo_count; i++)
271 dma_fence_put(job->implicit_fences[i]);
272 kvfree(job->implicit_fences);
273 }
274 dma_fence_put(job->done_fence);
275 dma_fence_put(job->render_done_fence);
276
277 if (job->bos) {
278 for (i = 0; i < job->bo_count; i++)
279 drm_gem_object_put_unlocked(job->bos[i]);
280 kvfree(job->bos);
281 }
282
283 kfree(job);
284}
285
286void panfrost_job_put(struct panfrost_job *job)
287{
288 kref_put(&job->refcount, panfrost_job_cleanup);
289}
290
291static void panfrost_job_free(struct drm_sched_job *sched_job)
292{
293 struct panfrost_job *job = to_panfrost_job(sched_job);
294
295 drm_sched_job_cleanup(sched_job);
296
297 panfrost_job_put(job);
298}
299
300static struct dma_fence *panfrost_job_dependency(struct drm_sched_job *sched_job,
301 struct drm_sched_entity *s_entity)
302{
303 struct panfrost_job *job = to_panfrost_job(sched_job);
304 struct dma_fence *fence;
305 unsigned int i;
306
307 /* Explicit fences */
308 for (i = 0; i < job->in_fence_count; i++) {
309 if (job->in_fences[i]) {
310 fence = job->in_fences[i];
311 job->in_fences[i] = NULL;
312 return fence;
313 }
314 }
315
316 /* Implicit fences, max. one per BO */
317 for (i = 0; i < job->bo_count; i++) {
318 if (job->implicit_fences[i]) {
319 fence = job->implicit_fences[i];
320 job->implicit_fences[i] = NULL;
321 return fence;
322 }
323 }
324
325 return NULL;
326}
327
328static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)
329{
330 struct panfrost_job *job = to_panfrost_job(sched_job);
331 struct panfrost_device *pfdev = job->pfdev;
332 int slot = panfrost_job_get_slot(job);
333 struct dma_fence *fence = NULL;
334
335 if (unlikely(job->base.s_fence->finished.error))
336 return NULL;
337
338 pfdev->jobs[slot] = job;
339
340 fence = panfrost_fence_create(pfdev, slot);
341 if (IS_ERR(fence))
342 return NULL;
343
344 if (job->done_fence)
345 dma_fence_put(job->done_fence);
346 job->done_fence = dma_fence_get(fence);
347
348 panfrost_job_hw_submit(job, slot);
349
350 return fence;
351}
352
353void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
354{
355 int j;
356 u32 irq_mask = 0;
357
358 for (j = 0; j < NUM_JOB_SLOTS; j++) {
359 irq_mask |= MK_JS_MASK(j);
360 }
361
362 job_write(pfdev, JOB_INT_CLEAR, irq_mask);
363 job_write(pfdev, JOB_INT_MASK, irq_mask);
364}
365
366static void panfrost_job_timedout(struct drm_sched_job *sched_job)
367{
368 struct panfrost_job *job = to_panfrost_job(sched_job);
369 struct panfrost_device *pfdev = job->pfdev;
370 int js = panfrost_job_get_slot(job);
371 int i;
372
373 /*
374 * If the GPU managed to complete this jobs fence, the timeout is
375 * spurious. Bail out.
376 */
377 if (dma_fence_is_signaled(job->done_fence))
378 return;
379
380 dev_err(pfdev->dev, "gpu sched timeout, js=%d, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
381 js,
382 job_read(pfdev, JS_STATUS(js)),
383 job_read(pfdev, JS_HEAD_LO(js)),
384 job_read(pfdev, JS_TAIL_LO(js)),
385 sched_job);
386
387 for (i = 0; i < NUM_JOB_SLOTS; i++)
388 drm_sched_stop(&pfdev->js->queue[i].sched);
389
390 if (sched_job)
391 drm_sched_increase_karma(sched_job);
392
393 /* panfrost_core_dump(pfdev); */
394
395 panfrost_devfreq_record_transition(pfdev, js);
396 panfrost_gpu_soft_reset(pfdev);
397
398 /* TODO: Re-enable all other address spaces */
399 panfrost_mmu_enable(pfdev, 0);
400 panfrost_gpu_power_on(pfdev);
401 panfrost_job_enable_interrupts(pfdev);
402
403 for (i = 0; i < NUM_JOB_SLOTS; i++)
404 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
405
406 /* restart scheduler after GPU is usable again */
407 for (i = 0; i < NUM_JOB_SLOTS; i++)
408 drm_sched_start(&pfdev->js->queue[i].sched, true);
409}
410
411static const struct drm_sched_backend_ops panfrost_sched_ops = {
412 .dependency = panfrost_job_dependency,
413 .run_job = panfrost_job_run,
414 .timedout_job = panfrost_job_timedout,
415 .free_job = panfrost_job_free
416};
417
418static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
419{
420 struct panfrost_device *pfdev = data;
421 u32 status = job_read(pfdev, JOB_INT_STAT);
422 int j;
423
424 dev_dbg(pfdev->dev, "jobslot irq status=%x\n", status);
425
426 if (!status)
427 return IRQ_NONE;
428
429 pm_runtime_mark_last_busy(pfdev->dev);
430
431 for (j = 0; status; j++) {
432 u32 mask = MK_JS_MASK(j);
433
434 if (!(status & mask))
435 continue;
436
437 job_write(pfdev, JOB_INT_CLEAR, mask);
438
439 if (status & JOB_INT_MASK_ERR(j)) {
440 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
441
442 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
443 j,
444 panfrost_exception_name(pfdev, job_read(pfdev, JS_STATUS(j))),
445 job_read(pfdev, JS_HEAD_LO(j)),
446 job_read(pfdev, JS_TAIL_LO(j)));
447
448 drm_sched_fault(&pfdev->js->queue[j].sched);
449 }
450
451 if (status & JOB_INT_MASK_DONE(j)) {
452 panfrost_devfreq_record_transition(pfdev, j);
453 dma_fence_signal(pfdev->jobs[j]->done_fence);
454 }
455
456 status &= ~mask;
457 }
458
459 return IRQ_HANDLED;
460}
461
462int panfrost_job_init(struct panfrost_device *pfdev)
463{
464 struct panfrost_job_slot *js;
465 int ret, j, irq;
466
467 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
468 if (!js)
469 return -ENOMEM;
470
471 spin_lock_init(&js->job_lock);
472
473 irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
474 if (irq <= 0)
475 return -ENODEV;
476
477 ret = devm_request_irq(pfdev->dev, irq, panfrost_job_irq_handler,
478 IRQF_SHARED, "job", pfdev);
479 if (ret) {
480 dev_err(pfdev->dev, "failed to request job irq");
481 return ret;
482 }
483
484 for (j = 0; j < NUM_JOB_SLOTS; j++) {
485 js->queue[j].fence_context = dma_fence_context_alloc(1);
486
487 ret = drm_sched_init(&js->queue[j].sched,
488 &panfrost_sched_ops,
489 1, 0, msecs_to_jiffies(500),
490 "pan_js");
491 if (ret) {
492 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
493 goto err_sched;
494 }
495 }
496
497 panfrost_job_enable_interrupts(pfdev);
498
499 return 0;
500
501err_sched:
502 for (j--; j >= 0; j--)
503 drm_sched_fini(&js->queue[j].sched);
504
505 return ret;
506}
507
508void panfrost_job_fini(struct panfrost_device *pfdev)
509{
510 struct panfrost_job_slot *js = pfdev->js;
511 int j;
512
513 job_write(pfdev, JOB_INT_MASK, 0);
514
515 for (j = 0; j < NUM_JOB_SLOTS; j++)
516 drm_sched_fini(&js->queue[j].sched);
517
518}
519
520int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)
521{
522 struct panfrost_device *pfdev = panfrost_priv->pfdev;
523 struct panfrost_job_slot *js = pfdev->js;
524 struct drm_sched_rq *rq;
525 int ret, i;
526
527 for (i = 0; i < NUM_JOB_SLOTS; i++) {
528 rq = &js->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
529 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i], &rq, 1, NULL);
530 if (WARN_ON(ret))
531 return ret;
532 }
533 return 0;
534}
535
536void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)
537{
538 int i;
539
540 for (i = 0; i < NUM_JOB_SLOTS; i++)
541 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
542}
543
544int panfrost_job_is_idle(struct panfrost_device *pfdev)
545{
546 struct panfrost_job_slot *js = pfdev->js;
547 int i;
548
549 for (i = 0; i < NUM_JOB_SLOTS; i++) {
550 /* If there are any jobs in the HW queue, we're not idle */
551 if (atomic_read(&js->queue[i].sched.hw_rq_count))
552 return false;
553
554 /* Check whether the hardware is idle */
555 if (pfdev->devfreq.slot[i].busy)
556 return false;
557 }
558
559 return true;
560}
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.h b/drivers/gpu/drm/panfrost/panfrost_job.h
new file mode 100644
index 000000000000..62454128a792
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_job.h
@@ -0,0 +1,51 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2019 Collabora ltd. */
3
4#ifndef __PANFROST_JOB_H__
5#define __PANFROST_JOB_H__
6
7#include <uapi/drm/panfrost_drm.h>
8#include <drm/gpu_scheduler.h>
9
10struct panfrost_device;
11struct panfrost_gem_object;
12struct panfrost_file_priv;
13
14struct panfrost_job {
15 struct drm_sched_job base;
16
17 struct kref refcount;
18
19 struct panfrost_device *pfdev;
20 struct panfrost_file_priv *file_priv;
21
22 /* Optional fences userspace can pass in for the job to depend on. */
23 struct dma_fence **in_fences;
24 u32 in_fence_count;
25
26 /* Fence to be signaled by IRQ handler when the job is complete. */
27 struct dma_fence *done_fence;
28
29 __u64 jc;
30 __u32 requirements;
31 __u32 flush_id;
32
33 /* Exclusive fences we have taken from the BOs to wait for */
34 struct dma_fence **implicit_fences;
35 struct drm_gem_object **bos;
36 u32 bo_count;
37
38 /* Fence to be signaled by drm-sched once its done with the job */
39 struct dma_fence *render_done_fence;
40};
41
42int panfrost_job_init(struct panfrost_device *pfdev);
43void panfrost_job_fini(struct panfrost_device *pfdev);
44int panfrost_job_open(struct panfrost_file_priv *panfrost_priv);
45void panfrost_job_close(struct panfrost_file_priv *panfrost_priv);
46int panfrost_job_push(struct panfrost_job *job);
47void panfrost_job_put(struct panfrost_job *job);
48void panfrost_job_enable_interrupts(struct panfrost_device *pfdev);
49int panfrost_job_is_idle(struct panfrost_device *pfdev);
50
51#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
new file mode 100644
index 000000000000..762b1bd2a8c2
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
@@ -0,0 +1,386 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3#include <linux/bitfield.h>
4#include <linux/delay.h>
5#include <linux/interrupt.h>
6#include <linux/io.h>
7#include <linux/iopoll.h>
8#include <linux/io-pgtable.h>
9#include <linux/iommu.h>
10#include <linux/platform_device.h>
11#include <linux/pm_runtime.h>
12#include <linux/sizes.h>
13
14#include "panfrost_device.h"
15#include "panfrost_mmu.h"
16#include "panfrost_gem.h"
17#include "panfrost_features.h"
18#include "panfrost_regs.h"
19
20#define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
21#define mmu_read(dev, reg) readl(dev->iomem + reg)
22
23struct panfrost_mmu {
24 struct io_pgtable_cfg pgtbl_cfg;
25 struct io_pgtable_ops *pgtbl_ops;
26 struct mutex lock;
27};
28
29static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
30{
31 int ret;
32 u32 val;
33
34 /* Wait for the MMU status to indicate there is no active command, in
35 * case one is pending. */
36 ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
37 val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000);
38
39 if (ret)
40 dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
41
42 return ret;
43}
44
45static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
46{
47 int status;
48
49 /* write AS_COMMAND when MMU is ready to accept another command */
50 status = wait_ready(pfdev, as_nr);
51 if (!status)
52 mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
53
54 return status;
55}
56
57static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
58 u64 iova, size_t size)
59{
60 u8 region_width;
61 u64 region = iova & PAGE_MASK;
62 /*
63 * fls returns:
64 * 1 .. 32
65 *
66 * 10 + fls(num_pages)
67 * results in the range (11 .. 42)
68 */
69
70 size = round_up(size, PAGE_SIZE);
71
72 region_width = 10 + fls(size >> PAGE_SHIFT);
73 if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) {
74 /* not pow2, so must go up to the next pow2 */
75 region_width += 1;
76 }
77 region |= region_width;
78
79 /* Lock the region that needs to be updated */
80 mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL);
81 mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL);
82 write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
83}
84
85
86static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr,
87 u64 iova, size_t size, u32 op)
88{
89 unsigned long flags;
90 int ret;
91
92 spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
93
94 if (op != AS_COMMAND_UNLOCK)
95 lock_region(pfdev, as_nr, iova, size);
96
97 /* Run the MMU operation */
98 write_cmd(pfdev, as_nr, op);
99
100 /* Wait for the flush to complete */
101 ret = wait_ready(pfdev, as_nr);
102
103 spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
104
105 return ret;
106}
107
108void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr)
109{
110 struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg;
111 u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
112 u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
113
114 mmu_write(pfdev, MMU_INT_CLEAR, ~0);
115 mmu_write(pfdev, MMU_INT_MASK, ~0);
116
117 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
118 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
119
120 /* Need to revisit mem attrs.
121 * NC is the default, Mali driver is inner WT.
122 */
123 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL);
124 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32);
125
126 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
127}
128
129static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
130{
131 mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
132 mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
133
134 mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
135 mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
136
137 write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
138}
139
140static size_t get_pgsize(u64 addr, size_t size)
141{
142 if (addr & (SZ_2M - 1) || size < SZ_2M)
143 return SZ_4K;
144
145 return SZ_2M;
146}
147
148int panfrost_mmu_map(struct panfrost_gem_object *bo)
149{
150 struct drm_gem_object *obj = &bo->base.base;
151 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
152 struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
153 u64 iova = bo->node.start << PAGE_SHIFT;
154 unsigned int count;
155 struct scatterlist *sgl;
156 struct sg_table *sgt;
157 int ret;
158
159 sgt = drm_gem_shmem_get_pages_sgt(obj);
160 if (WARN_ON(IS_ERR(sgt)))
161 return PTR_ERR(sgt);
162
163 ret = pm_runtime_get_sync(pfdev->dev);
164 if (ret < 0)
165 return ret;
166
167 mutex_lock(&pfdev->mmu->lock);
168
169 for_each_sg(sgt->sgl, sgl, sgt->nents, count) {
170 unsigned long paddr = sg_dma_address(sgl);
171 size_t len = sg_dma_len(sgl);
172
173 dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len);
174
175 while (len) {
176 size_t pgsize = get_pgsize(iova | paddr, len);
177
178 ops->map(ops, iova, paddr, pgsize, IOMMU_WRITE | IOMMU_READ);
179 iova += pgsize;
180 paddr += pgsize;
181 len -= pgsize;
182 }
183 }
184
185 mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
186 bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
187
188 mutex_unlock(&pfdev->mmu->lock);
189
190 pm_runtime_mark_last_busy(pfdev->dev);
191 pm_runtime_put_autosuspend(pfdev->dev);
192
193 return 0;
194}
195
196void panfrost_mmu_unmap(struct panfrost_gem_object *bo)
197{
198 struct drm_gem_object *obj = &bo->base.base;
199 struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
200 struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
201 u64 iova = bo->node.start << PAGE_SHIFT;
202 size_t len = bo->node.size << PAGE_SHIFT;
203 size_t unmapped_len = 0;
204 int ret;
205
206 dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len);
207
208 ret = pm_runtime_get_sync(pfdev->dev);
209 if (ret < 0)
210 return;
211
212 mutex_lock(&pfdev->mmu->lock);
213
214 while (unmapped_len < len) {
215 size_t unmapped_page;
216 size_t pgsize = get_pgsize(iova, len - unmapped_len);
217
218 unmapped_page = ops->unmap(ops, iova, pgsize);
219 if (!unmapped_page)
220 break;
221
222 iova += unmapped_page;
223 unmapped_len += unmapped_page;
224 }
225
226 mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
227 bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
228
229 mutex_unlock(&pfdev->mmu->lock);
230
231 pm_runtime_mark_last_busy(pfdev->dev);
232 pm_runtime_put_autosuspend(pfdev->dev);
233}
234
235static void mmu_tlb_inv_context_s1(void *cookie)
236{
237 struct panfrost_device *pfdev = cookie;
238
239 mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
240}
241
242static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
243 size_t granule, bool leaf, void *cookie)
244{}
245
246static void mmu_tlb_sync_context(void *cookie)
247{
248 //struct panfrost_device *pfdev = cookie;
249 // TODO: Wait 1000 GPU cycles for HW_ISSUE_6367/T60X
250}
251
252static const struct iommu_gather_ops mmu_tlb_ops = {
253 .tlb_flush_all = mmu_tlb_inv_context_s1,
254 .tlb_add_flush = mmu_tlb_inv_range_nosync,
255 .tlb_sync = mmu_tlb_sync_context,
256};
257
258static const char *access_type_name(struct panfrost_device *pfdev,
259 u32 fault_status)
260{
261 switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
262 case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
263 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
264 return "ATOMIC";
265 else
266 return "UNKNOWN";
267 case AS_FAULTSTATUS_ACCESS_TYPE_READ:
268 return "READ";
269 case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
270 return "WRITE";
271 case AS_FAULTSTATUS_ACCESS_TYPE_EX:
272 return "EXECUTE";
273 default:
274 WARN_ON(1);
275 return NULL;
276 }
277}
278
279static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
280{
281 struct panfrost_device *pfdev = data;
282 u32 status = mmu_read(pfdev, MMU_INT_STAT);
283 int i;
284
285 if (!status)
286 return IRQ_NONE;
287
288 dev_err(pfdev->dev, "mmu irq status=%x\n", status);
289
290 for (i = 0; status; i++) {
291 u32 mask = BIT(i) | BIT(i + 16);
292 u64 addr;
293 u32 fault_status;
294 u32 exception_type;
295 u32 access_type;
296 u32 source_id;
297
298 if (!(status & mask))
299 continue;
300
301 fault_status = mmu_read(pfdev, AS_FAULTSTATUS(i));
302 addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(i));
303 addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(i)) << 32;
304
305 /* decode the fault status */
306 exception_type = fault_status & 0xFF;
307 access_type = (fault_status >> 8) & 0x3;
308 source_id = (fault_status >> 16);
309
310 /* terminal fault, print info about the fault */
311 dev_err(pfdev->dev,
312 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
313 "Reason: %s\n"
314 "raw fault status: 0x%X\n"
315 "decoded fault status: %s\n"
316 "exception type 0x%X: %s\n"
317 "access type 0x%X: %s\n"
318 "source id 0x%X\n",
319 i, addr,
320 "TODO",
321 fault_status,
322 (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
323 exception_type, panfrost_exception_name(pfdev, exception_type),
324 access_type, access_type_name(pfdev, fault_status),
325 source_id);
326
327 mmu_write(pfdev, MMU_INT_CLEAR, mask);
328
329 status &= ~mask;
330 }
331
332 return IRQ_HANDLED;
333};
334
335int panfrost_mmu_init(struct panfrost_device *pfdev)
336{
337 struct io_pgtable_ops *pgtbl_ops;
338 int err, irq;
339
340 pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL);
341 if (!pfdev->mmu)
342 return -ENOMEM;
343
344 mutex_init(&pfdev->mmu->lock);
345
346 irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
347 if (irq <= 0)
348 return -ENODEV;
349
350 err = devm_request_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
351 IRQF_SHARED, "mmu", pfdev);
352
353 if (err) {
354 dev_err(pfdev->dev, "failed to request mmu irq");
355 return err;
356 }
357 mmu_write(pfdev, MMU_INT_CLEAR, ~0);
358 mmu_write(pfdev, MMU_INT_MASK, ~0);
359
360 pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
361 .pgsize_bitmap = SZ_4K | SZ_2M,
362 .ias = FIELD_GET(0xff, pfdev->features.mmu_features),
363 .oas = FIELD_GET(0xff00, pfdev->features.mmu_features),
364 .tlb = &mmu_tlb_ops,
365 .iommu_dev = pfdev->dev,
366 };
367
368 pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg,
369 pfdev);
370 if (!pgtbl_ops)
371 return -ENOMEM;
372
373 pfdev->mmu->pgtbl_ops = pgtbl_ops;
374
375 panfrost_mmu_enable(pfdev, 0);
376
377 return 0;
378}
379
380void panfrost_mmu_fini(struct panfrost_device *pfdev)
381{
382 mmu_write(pfdev, MMU_INT_MASK, 0);
383 mmu_disable(pfdev, 0);
384
385 free_io_pgtable_ops(pfdev->mmu->pgtbl_ops);
386}
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h
new file mode 100644
index 000000000000..f5878d86a5ce
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h
@@ -0,0 +1,17 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3
4#ifndef __PANFROST_MMU_H__
5#define __PANFROST_MMU_H__
6
7struct panfrost_gem_object;
8
9int panfrost_mmu_map(struct panfrost_gem_object *bo);
10void panfrost_mmu_unmap(struct panfrost_gem_object *bo);
11
12int panfrost_mmu_init(struct panfrost_device *pfdev);
13void panfrost_mmu_fini(struct panfrost_device *pfdev);
14
15void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr);
16
17#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
new file mode 100644
index 000000000000..578c5fc2188b
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -0,0 +1,298 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4/*
5 * Register definitions based on mali_midg_regmap.h
6 * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved.
7 */
8#ifndef __PANFROST_REGS_H__
9#define __PANFROST_REGS_H__
10
11#define GPU_ID 0x00
12#define GPU_L2_FEATURES 0x004 /* (RO) Level 2 cache features */
13#define GPU_CORE_FEATURES 0x008 /* (RO) Shader Core Features */
14#define GPU_TILER_FEATURES 0x00C /* (RO) Tiler Features */
15#define GPU_MEM_FEATURES 0x010 /* (RO) Memory system features */
16#define GROUPS_L2_COHERENT BIT(0) /* Cores groups are l2 coherent */
17
18#define GPU_MMU_FEATURES 0x014 /* (RO) MMU features */
19#define GPU_AS_PRESENT 0x018 /* (RO) Address space slots present */
20#define GPU_JS_PRESENT 0x01C /* (RO) Job slots present */
21
22#define GPU_INT_RAWSTAT 0x20
23#define GPU_INT_CLEAR 0x24
24#define GPU_INT_MASK 0x28
25#define GPU_INT_STAT 0x2c
26#define GPU_IRQ_FAULT BIT(0)
27#define GPU_IRQ_MULTIPLE_FAULT BIT(7)
28#define GPU_IRQ_RESET_COMPLETED BIT(8)
29#define GPU_IRQ_POWER_CHANGED BIT(9)
30#define GPU_IRQ_POWER_CHANGED_ALL BIT(10)
31#define GPU_IRQ_PERFCNT_SAMPLE_COMPLETED BIT(16)
32#define GPU_IRQ_CLEAN_CACHES_COMPLETED BIT(17)
33#define GPU_IRQ_MASK_ALL \
34 (GPU_IRQ_FAULT |\
35 GPU_IRQ_MULTIPLE_FAULT |\
36 GPU_IRQ_RESET_COMPLETED |\
37 GPU_IRQ_POWER_CHANGED |\
38 GPU_IRQ_POWER_CHANGED_ALL |\
39 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED |\
40 GPU_IRQ_CLEAN_CACHES_COMPLETED)
41#define GPU_IRQ_MASK_ERROR \
42 ( \
43 GPU_IRQ_FAULT |\
44 GPU_IRQ_MULTIPLE_FAULT)
45#define GPU_CMD 0x30
46#define GPU_CMD_SOFT_RESET 0x01
47#define GPU_STATUS 0x34
48#define GPU_LATEST_FLUSH_ID 0x38
49#define GPU_FAULT_STATUS 0x3C
50#define GPU_FAULT_ADDRESS_LO 0x40
51#define GPU_FAULT_ADDRESS_HI 0x44
52
53#define GPU_THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */
54#define GPU_THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */
55#define GPU_THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */
56#define GPU_THREAD_FEATURES 0x0AC /* (RO) Thread features */
57#define GPU_THREAD_TLS_ALLOC 0x310 /* (RO) Number of threads per core that
58 * TLS must be allocated for */
59
60#define GPU_TEXTURE_FEATURES(n) (0x0B0 + ((n) * 4))
61#define GPU_JS_FEATURES(n) (0x0C0 + ((n) * 4))
62
63#define GPU_SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */
64#define GPU_SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */
65#define GPU_TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */
66#define GPU_TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */
67
68#define GPU_L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */
69#define GPU_L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */
70
71#define GPU_COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */
72#define COHERENCY_ACE_LITE BIT(0)
73#define COHERENCY_ACE BIT(1)
74
75#define GPU_STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */
76#define GPU_STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */
77
78#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */
79#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */
80
81#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */
82#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */
83
84#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */
85#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */
86
87#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */
88#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */
89
90
91#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */
92#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */
93
94#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */
95#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */
96
97#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */
98#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */
99
100#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */
101#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */
102
103
104#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */
105#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */
106
107#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */
108#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */
109
110#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */
111#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */
112
113#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */
114#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */
115
116
117#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */
118#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */
119
120#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */
121#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */
122
123#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */
124#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */
125
126#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */
127#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */
128
129
130#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */
131#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */
132
133#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */
134#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */
135
136#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */
137#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */
138
139#define GPU_JM_CONFIG 0xF00 /* (RW) Job Manager configuration register (Implementation specific register) */
140#define GPU_SHADER_CONFIG 0xF04 /* (RW) Shader core configuration settings (Implementation specific register) */
141#define GPU_TILER_CONFIG 0xF08 /* (RW) Tiler core configuration settings (Implementation specific register) */
142#define GPU_L2_MMU_CONFIG 0xF0C /* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */
143
144/* L2_MMU_CONFIG register */
145#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT 23
146#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT)
147#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT 24
148#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
149#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
150#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
151#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
152
153#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT 26
154#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
155#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT (0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
156#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER (0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
157#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF (0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
158
159#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS_SHIFT 12
160#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS (0x7 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
161
162#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES_SHIFT 15
163#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES (0x7 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
164
165/* SHADER_CONFIG register */
166#define SC_ALT_COUNTERS BIT(3)
167#define SC_OVERRIDE_FWD_PIXEL_KILL BIT(4)
168#define SC_SDC_DISABLE_OQ_DISCARD BIT(6)
169#define SC_LS_ALLOW_ATTR_TYPES BIT(16)
170#define SC_LS_PAUSEBUFFER_DISABLE BIT(16)
171#define SC_TLS_HASH_ENABLE BIT(17)
172#define SC_LS_ATTR_CHECK_DISABLE BIT(18)
173#define SC_ENABLE_TEXGRD_FLAGS BIT(25)
174/* End SHADER_CONFIG register */
175
176/* TILER_CONFIG register */
177#define TC_CLOCK_GATE_OVERRIDE BIT(0)
178
179/* JM_CONFIG register */
180#define JM_TIMESTAMP_OVERRIDE BIT(0)
181#define JM_CLOCK_GATE_OVERRIDE BIT(1)
182#define JM_JOB_THROTTLE_ENABLE BIT(2)
183#define JM_JOB_THROTTLE_LIMIT_SHIFT 3
184#define JM_MAX_JOB_THROTTLE_LIMIT 0x3F
185#define JM_FORCE_COHERENCY_FEATURES_SHIFT 2
186#define JM_IDVS_GROUP_SIZE_SHIFT 16
187#define JM_MAX_IDVS_GROUP_SIZE 0x3F
188
189
190/* Job Control regs */
191#define JOB_INT_RAWSTAT 0x1000
192#define JOB_INT_CLEAR 0x1004
193#define JOB_INT_MASK 0x1008
194#define JOB_INT_STAT 0x100c
195#define JOB_INT_JS_STATE 0x1010
196#define JOB_INT_THROTTLE 0x1014
197
198#define MK_JS_MASK(j) (0x10001 << (j))
199#define JOB_INT_MASK_ERR(j) BIT((j) + 16)
200#define JOB_INT_MASK_DONE(j) BIT(j)
201
202#define JS_BASE 0x1800
203#define JS_HEAD_LO(n) (JS_BASE + ((n) * 0x80) + 0x00)
204#define JS_HEAD_HI(n) (JS_BASE + ((n) * 0x80) + 0x04)
205#define JS_TAIL_LO(n) (JS_BASE + ((n) * 0x80) + 0x08)
206#define JS_TAIL_HI(n) (JS_BASE + ((n) * 0x80) + 0x0c)
207#define JS_AFFINITY_LO(n) (JS_BASE + ((n) * 0x80) + 0x10)
208#define JS_AFFINITY_HI(n) (JS_BASE + ((n) * 0x80) + 0x14)
209#define JS_CONFIG(n) (JS_BASE + ((n) * 0x80) + 0x18)
210#define JS_XAFFINITY(n) (JS_BASE + ((n) * 0x80) + 0x1c)
211#define JS_COMMAND(n) (JS_BASE + ((n) * 0x80) + 0x20)
212#define JS_STATUS(n) (JS_BASE + ((n) * 0x80) + 0x24)
213#define JS_HEAD_NEXT_LO(n) (JS_BASE + ((n) * 0x80) + 0x40)
214#define JS_HEAD_NEXT_HI(n) (JS_BASE + ((n) * 0x80) + 0x44)
215#define JS_AFFINITY_NEXT_LO(n) (JS_BASE + ((n) * 0x80) + 0x50)
216#define JS_AFFINITY_NEXT_HI(n) (JS_BASE + ((n) * 0x80) + 0x54)
217#define JS_CONFIG_NEXT(n) (JS_BASE + ((n) * 0x80) + 0x58)
218#define JS_COMMAND_NEXT(n) (JS_BASE + ((n) * 0x80) + 0x60)
219#define JS_FLUSH_ID_NEXT(n) (JS_BASE + ((n) * 0x80) + 0x70)
220
221/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */
222#define JS_CONFIG_START_FLUSH_CLEAN BIT(8)
223#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8)
224#define JS_CONFIG_START_MMU BIT(10)
225#define JS_CONFIG_JOB_CHAIN_FLAG BIT(11)
226#define JS_CONFIG_END_FLUSH_CLEAN BIT(12)
227#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12)
228#define JS_CONFIG_ENABLE_FLUSH_REDUCTION BIT(14)
229#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK BIT(15)
230#define JS_CONFIG_THREAD_PRI(n) ((n) << 16)
231
232#define JS_COMMAND_NOP 0x00
233#define JS_COMMAND_START 0x01
234#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */
235#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */
236#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */
237#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */
238#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */
239#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */
240
241#define JS_STATUS_EVENT_ACTIVE 0x08
242
243
244/* MMU regs */
245#define MMU_INT_RAWSTAT 0x2000
246#define MMU_INT_CLEAR 0x2004
247#define MMU_INT_MASK 0x2008
248#define MMU_INT_STAT 0x200c
249
250/* AS_COMMAND register commands */
251#define AS_COMMAND_NOP 0x00 /* NOP Operation */
252#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */
253#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */
254#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */
255#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs
256 (deprecated - only for use with T60x) */
257#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */
258#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then
259 flush all L2 caches then issue a flush region command to all MMUs */
260
261#define MMU_AS(as) (0x2400 + ((as) << 6))
262
263#define AS_TRANSTAB_LO(as) (MMU_AS(as) + 0x00) /* (RW) Translation Table Base Address for address space n, low word */
264#define AS_TRANSTAB_HI(as) (MMU_AS(as) + 0x04) /* (RW) Translation Table Base Address for address space n, high word */
265#define AS_MEMATTR_LO(as) (MMU_AS(as) + 0x08) /* (RW) Memory attributes for address space n, low word. */
266#define AS_MEMATTR_HI(as) (MMU_AS(as) + 0x0C) /* (RW) Memory attributes for address space n, high word. */
267#define AS_LOCKADDR_LO(as) (MMU_AS(as) + 0x10) /* (RW) Lock region address for address space n, low word */
268#define AS_LOCKADDR_HI(as) (MMU_AS(as) + 0x14) /* (RW) Lock region address for address space n, high word */
269#define AS_COMMAND(as) (MMU_AS(as) + 0x18) /* (WO) MMU command register for address space n */
270#define AS_FAULTSTATUS(as) (MMU_AS(as) + 0x1C) /* (RO) MMU fault status register for address space n */
271#define AS_FAULTADDRESS_LO(as) (MMU_AS(as) + 0x20) /* (RO) Fault Address for address space n, low word */
272#define AS_FAULTADDRESS_HI(as) (MMU_AS(as) + 0x24) /* (RO) Fault Address for address space n, high word */
273#define AS_STATUS(as) (MMU_AS(as) + 0x28) /* (RO) Status flags for address space n */
274/* Additional Bifrost AS regsiters */
275#define AS_TRANSCFG_LO(as) (MMU_AS(as) + 0x30) /* (RW) Translation table configuration for address space n, low word */
276#define AS_TRANSCFG_HI(as) (MMU_AS(as) + 0x34) /* (RW) Translation table configuration for address space n, high word */
277#define AS_FAULTEXTRA_LO(as) (MMU_AS(as) + 0x38) /* (RO) Secondary fault address for address space n, low word */
278#define AS_FAULTEXTRA_HI(as) (MMU_AS(as) + 0x3C) /* (RO) Secondary fault address for address space n, high word */
279
280/*
281 * Begin LPAE MMU TRANSTAB register values
282 */
283#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffffffffffff000
284#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY 0x2
285#define AS_TRANSTAB_LPAE_ADRMODE_TABLE 0x3
286#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x3
287#define AS_TRANSTAB_LPAE_READ_INNER BIT(2)
288#define AS_TRANSTAB_LPAE_SHARE_OUTER BIT(4)
289
290#define AS_STATUS_AS_ACTIVE 0x01
291
292#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << 8)
293#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0 << 8)
294#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1 << 8)
295#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2 << 8)
296#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3 << 8)
297
298#endif
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index ee59da4a0172..4e5922c89d7b 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -361,13 +361,6 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend,
361 paddr = drm_fb_cma_get_gem_addr(fb, state, 0); 361 paddr = drm_fb_cma_get_gem_addr(fb, state, 0);
362 DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr); 362 DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
363 363
364 /*
365 * backend DMA accesses DRAM directly, bypassing the system
366 * bus. As such, the address range is different and the buffer
367 * address needs to be corrected.
368 */
369 paddr -= PHYS_OFFSET;
370
371 if (fb->format->is_yuv) 364 if (fb->format->is_yuv)
372 return sun4i_backend_update_yuv_buffer(backend, fb, paddr); 365 return sun4i_backend_update_yuv_buffer(backend, fb, paddr);
373 366
@@ -803,6 +796,27 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
803 dev_set_drvdata(dev, backend); 796 dev_set_drvdata(dev, backend);
804 spin_lock_init(&backend->frontend_lock); 797 spin_lock_init(&backend->frontend_lock);
805 798
799 if (of_find_property(dev->of_node, "interconnects", NULL)) {
800 /*
801 * This assume we have the same DMA constraints for all our the
802 * devices in our pipeline (all the backends, but also the
803 * frontends). This sounds bad, but it has always been the case
804 * for us, and DRM doesn't do per-device allocation either, so
805 * we would need to fix DRM first...
806 */
807 ret = of_dma_configure(drm->dev, dev->of_node, true);
808 if (ret)
809 return ret;
810 } else {
811 /*
812 * If we don't have the interconnect property, most likely
813 * because of an old DT, we need to set the DMA offset by hand
814 * on our device since the RAM mapping is at 0 for the DMA bus,
815 * unlike the CPU.
816 */
817 drm->dev->dma_pfn_offset = PHYS_PFN_OFFSET;
818 }
819
806 backend->engine.node = dev->of_node; 820 backend->engine.node = dev->of_node;
807 backend->engine.ops = &sun4i_backend_engine_ops; 821 backend->engine.ops = &sun4i_backend_engine_ops;
808 backend->engine.id = sun4i_backend_of_get_id(dev->of_node); 822 backend->engine.id = sun4i_backend_of_get_id(dev->of_node);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index fa92e992a282..9d8d8124b1f6 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -236,8 +236,8 @@ static struct sun4i_tcon *sun4i_get_tcon0(struct drm_device *drm)
236 return NULL; 236 return NULL;
237} 237}
238 238
239void sun4i_tcon_set_mux(struct sun4i_tcon *tcon, int channel, 239static void sun4i_tcon_set_mux(struct sun4i_tcon *tcon, int channel,
240 const struct drm_encoder *encoder) 240 const struct drm_encoder *encoder)
241{ 241{
242 int ret = -ENOTSUPP; 242 int ret = -ENOTSUPP;
243 243
diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
index b1e7c76e9c17..3267d0f9b9b2 100644
--- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
+++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
@@ -269,12 +269,12 @@ static int sun8i_tcon_top_remove(struct platform_device *pdev)
269 return 0; 269 return 0;
270} 270}
271 271
272const struct sun8i_tcon_top_quirks sun8i_r40_tcon_top_quirks = { 272static const struct sun8i_tcon_top_quirks sun8i_r40_tcon_top_quirks = {
273 .has_tcon_tv1 = true, 273 .has_tcon_tv1 = true,
274 .has_dsi = true, 274 .has_dsi = true,
275}; 275};
276 276
277const struct sun8i_tcon_top_quirks sun50i_h6_tcon_top_quirks = { 277static const struct sun8i_tcon_top_quirks sun50i_h6_tcon_top_quirks = {
278 /* Nothing special */ 278 /* Nothing special */
279}; 279};
280 280
diff --git a/drivers/gpu/drm/tinydrm/hx8357d.c b/drivers/gpu/drm/tinydrm/hx8357d.c
index fab961dded87..5773d0fb6ca1 100644
--- a/drivers/gpu/drm/tinydrm/hx8357d.c
+++ b/drivers/gpu/drm/tinydrm/hx8357d.c
@@ -267,7 +267,7 @@ static int hx8357d_probe(struct spi_device *spi)
267 267
268 spi_set_drvdata(spi, drm); 268 spi_set_drvdata(spi, drm);
269 269
270 drm_fbdev_generic_setup(drm, 32); 270 drm_fbdev_generic_setup(drm, 0);
271 271
272 return 0; 272 return 0;
273} 273}
diff --git a/drivers/gpu/drm/tinydrm/ili9225.c b/drivers/gpu/drm/tinydrm/ili9225.c
index e9116ef4b5bc..4b1a587c0134 100644
--- a/drivers/gpu/drm/tinydrm/ili9225.c
+++ b/drivers/gpu/drm/tinydrm/ili9225.c
@@ -433,7 +433,7 @@ static int ili9225_probe(struct spi_device *spi)
433 433
434 spi_set_drvdata(spi, drm); 434 spi_set_drvdata(spi, drm);
435 435
436 drm_fbdev_generic_setup(drm, 32); 436 drm_fbdev_generic_setup(drm, 0);
437 437
438 return 0; 438 return 0;
439} 439}
diff --git a/drivers/gpu/drm/tinydrm/ili9341.c b/drivers/gpu/drm/tinydrm/ili9341.c
index d15f85e837ae..4ade9e4b924f 100644
--- a/drivers/gpu/drm/tinydrm/ili9341.c
+++ b/drivers/gpu/drm/tinydrm/ili9341.c
@@ -229,7 +229,7 @@ static int ili9341_probe(struct spi_device *spi)
229 229
230 spi_set_drvdata(spi, drm); 230 spi_set_drvdata(spi, drm);
231 231
232 drm_fbdev_generic_setup(drm, 32); 232 drm_fbdev_generic_setup(drm, 0);
233 233
234 return 0; 234 return 0;
235} 235}
diff --git a/drivers/gpu/drm/tinydrm/mi0283qt.c b/drivers/gpu/drm/tinydrm/mi0283qt.c
index c6dc31084a4e..8e169846fbd8 100644
--- a/drivers/gpu/drm/tinydrm/mi0283qt.c
+++ b/drivers/gpu/drm/tinydrm/mi0283qt.c
@@ -242,7 +242,7 @@ static int mi0283qt_probe(struct spi_device *spi)
242 242
243 spi_set_drvdata(spi, drm); 243 spi_set_drvdata(spi, drm);
244 244
245 drm_fbdev_generic_setup(drm, 32); 245 drm_fbdev_generic_setup(drm, 0);
246 246
247 return 0; 247 return 0;
248} 248}
diff --git a/drivers/gpu/drm/tinydrm/repaper.c b/drivers/gpu/drm/tinydrm/repaper.c
index a29b8278324b..370629e2de94 100644
--- a/drivers/gpu/drm/tinydrm/repaper.c
+++ b/drivers/gpu/drm/tinydrm/repaper.c
@@ -1131,7 +1131,7 @@ static int repaper_probe(struct spi_device *spi)
1131 1131
1132 DRM_DEBUG_DRIVER("SPI speed: %uMHz\n", spi->max_speed_hz / 1000000); 1132 DRM_DEBUG_DRIVER("SPI speed: %uMHz\n", spi->max_speed_hz / 1000000);
1133 1133
1134 drm_fbdev_generic_setup(drm, 32); 1134 drm_fbdev_generic_setup(drm, 0);
1135 1135
1136 return 0; 1136 return 0;
1137} 1137}
diff --git a/drivers/gpu/drm/tinydrm/st7586.c b/drivers/gpu/drm/tinydrm/st7586.c
index 560d7ac0cadc..36bb16a15f7e 100644
--- a/drivers/gpu/drm/tinydrm/st7586.c
+++ b/drivers/gpu/drm/tinydrm/st7586.c
@@ -408,7 +408,7 @@ static int st7586_probe(struct spi_device *spi)
408 DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n", 408 DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
409 drm->mode_config.preferred_depth, rotation); 409 drm->mode_config.preferred_depth, rotation);
410 410
411 drm_fbdev_generic_setup(drm, 32); 411 drm_fbdev_generic_setup(drm, 0);
412 412
413 return 0; 413 return 0;
414} 414}
diff --git a/drivers/gpu/drm/tinydrm/st7735r.c b/drivers/gpu/drm/tinydrm/st7735r.c
index 022e9849b95b..ce9109e613e0 100644
--- a/drivers/gpu/drm/tinydrm/st7735r.c
+++ b/drivers/gpu/drm/tinydrm/st7735r.c
@@ -207,7 +207,7 @@ static int st7735r_probe(struct spi_device *spi)
207 207
208 spi_set_drvdata(spi, drm); 208 spi_set_drvdata(spi, drm);
209 209
210 drm_fbdev_generic_setup(drm, 32); 210 drm_fbdev_generic_setup(drm, 0);
211 211
212 return 0; 212 return 0;
213} 213}
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index d3700ec15cbd..4e21efbc4459 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -172,6 +172,10 @@
172#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 172#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
173#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 173#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
174 174
175#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
176#define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
177#define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
178
175/* IOPTE accessors */ 179/* IOPTE accessors */
176#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) 180#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
177 181
@@ -180,11 +184,6 @@
180 184
181#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) 185#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
182 186
183#define iopte_leaf(pte,l) \
184 (l == (ARM_LPAE_MAX_LEVELS - 1) ? \
185 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \
186 (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK))
187
188struct arm_lpae_io_pgtable { 187struct arm_lpae_io_pgtable {
189 struct io_pgtable iop; 188 struct io_pgtable iop;
190 189
@@ -198,6 +197,15 @@ struct arm_lpae_io_pgtable {
198 197
199typedef u64 arm_lpae_iopte; 198typedef u64 arm_lpae_iopte;
200 199
200static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
201 enum io_pgtable_fmt fmt)
202{
203 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
204 return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE;
205
206 return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK;
207}
208
201static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, 209static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
202 struct arm_lpae_io_pgtable *data) 210 struct arm_lpae_io_pgtable *data)
203{ 211{
@@ -303,12 +311,14 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
303 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 311 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
304 pte |= ARM_LPAE_PTE_NS; 312 pte |= ARM_LPAE_PTE_NS;
305 313
306 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 314 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
307 pte |= ARM_LPAE_PTE_TYPE_PAGE; 315 pte |= ARM_LPAE_PTE_TYPE_PAGE;
308 else 316 else
309 pte |= ARM_LPAE_PTE_TYPE_BLOCK; 317 pte |= ARM_LPAE_PTE_TYPE_BLOCK;
310 318
311 pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; 319 if (data->iop.fmt != ARM_MALI_LPAE)
320 pte |= ARM_LPAE_PTE_AF;
321 pte |= ARM_LPAE_PTE_SH_IS;
312 pte |= paddr_to_iopte(paddr, data); 322 pte |= paddr_to_iopte(paddr, data);
313 323
314 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); 324 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg);
@@ -321,7 +331,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
321{ 331{
322 arm_lpae_iopte pte = *ptep; 332 arm_lpae_iopte pte = *ptep;
323 333
324 if (iopte_leaf(pte, lvl)) { 334 if (iopte_leaf(pte, lvl, data->iop.fmt)) {
325 /* We require an unmap first */ 335 /* We require an unmap first */
326 WARN_ON(!selftest_running); 336 WARN_ON(!selftest_running);
327 return -EEXIST; 337 return -EEXIST;
@@ -409,7 +419,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
409 __arm_lpae_sync_pte(ptep, cfg); 419 __arm_lpae_sync_pte(ptep, cfg);
410 } 420 }
411 421
412 if (pte && !iopte_leaf(pte, lvl)) { 422 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
413 cptep = iopte_deref(pte, data); 423 cptep = iopte_deref(pte, data);
414 } else if (pte) { 424 } else if (pte) {
415 /* We require an unmap first */ 425 /* We require an unmap first */
@@ -429,31 +439,37 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
429 if (data->iop.fmt == ARM_64_LPAE_S1 || 439 if (data->iop.fmt == ARM_64_LPAE_S1 ||
430 data->iop.fmt == ARM_32_LPAE_S1) { 440 data->iop.fmt == ARM_32_LPAE_S1) {
431 pte = ARM_LPAE_PTE_nG; 441 pte = ARM_LPAE_PTE_nG;
432
433 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 442 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
434 pte |= ARM_LPAE_PTE_AP_RDONLY; 443 pte |= ARM_LPAE_PTE_AP_RDONLY;
435
436 if (!(prot & IOMMU_PRIV)) 444 if (!(prot & IOMMU_PRIV))
437 pte |= ARM_LPAE_PTE_AP_UNPRIV; 445 pte |= ARM_LPAE_PTE_AP_UNPRIV;
438
439 if (prot & IOMMU_MMIO)
440 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
441 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
442 else if (prot & IOMMU_CACHE)
443 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
444 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
445 } else { 446 } else {
446 pte = ARM_LPAE_PTE_HAP_FAULT; 447 pte = ARM_LPAE_PTE_HAP_FAULT;
447 if (prot & IOMMU_READ) 448 if (prot & IOMMU_READ)
448 pte |= ARM_LPAE_PTE_HAP_READ; 449 pte |= ARM_LPAE_PTE_HAP_READ;
449 if (prot & IOMMU_WRITE) 450 if (prot & IOMMU_WRITE)
450 pte |= ARM_LPAE_PTE_HAP_WRITE; 451 pte |= ARM_LPAE_PTE_HAP_WRITE;
452 }
453
454 /*
455 * Note that this logic is structured to accommodate Mali LPAE
456 * having stage-1-like attributes but stage-2-like permissions.
457 */
458 if (data->iop.fmt == ARM_64_LPAE_S2 ||
459 data->iop.fmt == ARM_32_LPAE_S2) {
451 if (prot & IOMMU_MMIO) 460 if (prot & IOMMU_MMIO)
452 pte |= ARM_LPAE_PTE_MEMATTR_DEV; 461 pte |= ARM_LPAE_PTE_MEMATTR_DEV;
453 else if (prot & IOMMU_CACHE) 462 else if (prot & IOMMU_CACHE)
454 pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 463 pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
455 else 464 else
456 pte |= ARM_LPAE_PTE_MEMATTR_NC; 465 pte |= ARM_LPAE_PTE_MEMATTR_NC;
466 } else {
467 if (prot & IOMMU_MMIO)
468 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
469 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
470 else if (prot & IOMMU_CACHE)
471 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
472 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
457 } 473 }
458 474
459 if (prot & IOMMU_NOEXEC) 475 if (prot & IOMMU_NOEXEC)
@@ -511,7 +527,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
511 while (ptep != end) { 527 while (ptep != end) {
512 arm_lpae_iopte pte = *ptep++; 528 arm_lpae_iopte pte = *ptep++;
513 529
514 if (!pte || iopte_leaf(pte, lvl)) 530 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
515 continue; 531 continue;
516 532
517 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 533 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
@@ -602,7 +618,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
602 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 618 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
603 __arm_lpae_set_pte(ptep, 0, &iop->cfg); 619 __arm_lpae_set_pte(ptep, 0, &iop->cfg);
604 620
605 if (!iopte_leaf(pte, lvl)) { 621 if (!iopte_leaf(pte, lvl, iop->fmt)) {
606 /* Also flush any partial walks */ 622 /* Also flush any partial walks */
607 io_pgtable_tlb_add_flush(iop, iova, size, 623 io_pgtable_tlb_add_flush(iop, iova, size,
608 ARM_LPAE_GRANULE(data), false); 624 ARM_LPAE_GRANULE(data), false);
@@ -621,7 +637,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
621 } 637 }
622 638
623 return size; 639 return size;
624 } else if (iopte_leaf(pte, lvl)) { 640 } else if (iopte_leaf(pte, lvl, iop->fmt)) {
625 /* 641 /*
626 * Insert a table at the next level to map the old region, 642 * Insert a table at the next level to map the old region,
627 * minus the part we want to unmap 643 * minus the part we want to unmap
@@ -669,7 +685,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
669 return 0; 685 return 0;
670 686
671 /* Leaf entry? */ 687 /* Leaf entry? */
672 if (iopte_leaf(pte,lvl)) 688 if (iopte_leaf(pte, lvl, data->iop.fmt))
673 goto found_translation; 689 goto found_translation;
674 690
675 /* Take it to the next level */ 691 /* Take it to the next level */
@@ -995,6 +1011,32 @@ arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
995 return iop; 1011 return iop;
996} 1012}
997 1013
1014static struct io_pgtable *
1015arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
1016{
1017 struct io_pgtable *iop;
1018
1019 if (cfg->ias != 48 || cfg->oas > 40)
1020 return NULL;
1021
1022 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1023 iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
1024 if (iop) {
1025 u64 mair, ttbr;
1026
1027 /* Copy values as union fields overlap */
1028 mair = cfg->arm_lpae_s1_cfg.mair[0];
1029 ttbr = cfg->arm_lpae_s1_cfg.ttbr[0];
1030
1031 cfg->arm_mali_lpae_cfg.memattr = mair;
1032 cfg->arm_mali_lpae_cfg.transtab = ttbr |
1033 ARM_MALI_LPAE_TTBR_READ_INNER |
1034 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
1035 }
1036
1037 return iop;
1038}
1039
998struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { 1040struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
999 .alloc = arm_64_lpae_alloc_pgtable_s1, 1041 .alloc = arm_64_lpae_alloc_pgtable_s1,
1000 .free = arm_lpae_free_pgtable, 1042 .free = arm_lpae_free_pgtable,
@@ -1015,6 +1057,11 @@ struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
1015 .free = arm_lpae_free_pgtable, 1057 .free = arm_lpae_free_pgtable,
1016}; 1058};
1017 1059
1060struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
1061 .alloc = arm_mali_lpae_alloc_pgtable,
1062 .free = arm_lpae_free_pgtable,
1063};
1064
1018#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST 1065#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
1019 1066
1020static struct io_pgtable_cfg *cfg_cookie; 1067static struct io_pgtable_cfg *cfg_cookie;
diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 93f2880be6c6..5227cfdbb65b 100644
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -30,6 +30,7 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
30 [ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns, 30 [ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns,
31 [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, 31 [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns,
32 [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, 32 [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns,
33 [ARM_MALI_LPAE] = &io_pgtable_arm_mali_lpae_init_fns,
33#endif 34#endif
34#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S 35#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S
35 [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, 36 [ARM_V7S] = &io_pgtable_arm_v7s_init_fns,
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-29 18:49:52 -0500