Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Copyright (C) 2013--2024 Intel Corporation
4 : */
5 : #include <asm/barrier.h>
6 :
7 : #include <linux/align.h>
8 : #include <linux/atomic.h>
9 : #include <linux/bitops.h>
10 : #include <linux/bits.h>
11 : #include <linux/bug.h>
12 : #include <linux/cacheflush.h>
13 : #include <linux/dma-mapping.h>
14 : #include <linux/err.h>
15 : #include <linux/gfp.h>
16 : #include <linux/io.h>
17 : #include <linux/iova.h>
18 : #include <linux/math.h>
19 : #include <linux/minmax.h>
20 : #include <linux/mm.h>
21 : #include <linux/pfn.h>
22 : #include <linux/slab.h>
23 : #include <linux/spinlock.h>
24 : #include <linux/types.h>
25 : #include <linux/vmalloc.h>
26 :
27 : #include "ipu6.h"
28 : #include "ipu6-dma.h"
29 : #include "ipu6-mmu.h"
30 : #include "ipu6-platform-regs.h"
31 :
32 : #define ISP_PAGE_SHIFT 12
33 : #define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT)
34 : #define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1))
35 :
36 : #define ISP_L1PT_SHIFT 22
37 : #define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1))
38 :
39 : #define ISP_L2PT_SHIFT 12
40 : #define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK))))
41 :
42 : #define ISP_L1PT_PTES 1024
43 : #define ISP_L2PT_PTES 1024
44 :
45 : #define ISP_PADDR_SHIFT 12
46 :
47 : #define REG_TLB_INVALIDATE 0x0000
48 : #define MMU0_TLB_INVALIDATE 1
49 : #define MMU1_TLB_INVALIDATE 0xffff
50 :
51 : #define REG_L1_PHYS 0x0004 /* 27-bit pfn */
52 : #define REG_INFO 0x0008
53 :
54 : /* ZLW Enable for each stream in L1 MMU AT where i : 0..15 */
55 : #define MMUV2_AT_REG_L1_ZLW_EN_SID(i) (0x100 + ((i) * 0x20))
56 :
57 : /* ZLW 1D mode Enable for each stream in L1 MMU AT where i : 0..15 */
58 : #define MMUV2_AT_REG_L1_ZLW_1DMODE_SID(i) (0x100 + ((i) * 0x20) + 0x0004)
59 :
60 : /* Set ZLW insertion N pages ahead per stream 1D where i : 0..15 */
61 : #define MMUV2_AT_REG_L1_ZLW_INS_N_AHEAD_SID(i) (0x100 + ((i) * 0x20) + 0x0008)
62 :
63 : /* ZLW 2D mode Enable for each stream in L1 MMU AT where i : 0..15 */
64 : #define MMUV2_AT_REG_L1_ZLW_2DMODE_SID(i) (0x100 + ((i) * 0x20) + 0x0010)
65 :
66 : /* ZLW Insertion for each stream in L1 MMU AT where i : 0..15 */
67 : #define MMUV2_AT_REG_L1_ZLW_INSERTION(i) (0x100 + ((i) * 0x20) + 0x000c)
68 :
69 : #define MMUV2_AT_REG_L1_FW_ZLW_FIFO (0x100 + \
70 : (IPU4_MMU_MAX_TLB_L1_STREAMS * 0x20) + 0x003c)
71 :
72 : /* FW ZLW has prioty - needed for ZLW invalidations */
73 : #define MMUV2_AT_REG_L1_FW_ZLW_PRIO (0x100 + \
74 : (IPU4_MMU_MAX_TLB_L1_STREAMS * 0x20))
75 :
76 : #define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)
77 :
78 0 : static void zlw_invalidate(struct ipu6_mmu *mmu, struct ipu6_mmu_hw *mmu_hw)
79 : {
80 : unsigned int retry = 0;
81 : unsigned int i, j;
82 : int ret;
83 :
84 0 : for (i = 0; i < mmu_hw->nr_l1streams; i++) {
85 : /* We need to invalidate only the zlw enabled stream IDs */
86 0 : if (mmu_hw->l1_zlw_en[i]) {
87 : /*
88 : * Maximum 16 blocks per L1 stream
89 : * Write trash buffer iova offset to the FW_ZLW
90 : * register. This will trigger pre-fetching of next 16
91 : * pages from the page table. So we need to increment
92 : * iova address by 16 * 4K to trigger the next 16 pages.
93 : * Once this loop is completed, the L1 cache will be
94 : * filled with trash buffer translation.
95 : *
96 : * TODO: Instead of maximum 16 blocks, use the allocated
97 : * block size
98 : */
99 0 : for (j = 0; j < mmu_hw->l1_block_sz[i]; j++)
100 0 : writel(mmu->iova_trash_page +
101 0 : j * MMUV2_TRASH_L1_BLOCK_OFFSET,
102 0 : mmu_hw->base +
103 0 : MMUV2_AT_REG_L1_ZLW_INSERTION(i));
104 :
105 : /*
106 : * Now we need to fill the L2 cache entry. L2 cache
107 : * entries will be automatically updated, based on the
108 : * L1 entry. The above loop for L1 will update only one
109 : * of the two entries in L2 as the L1 is under 4MB
110 : * range. To force the other entry in L2 to update, we
111 : * just need to trigger another pre-fetch which is
112 : * outside the above 4MB range.
113 : */
114 0 : writel(mmu->iova_trash_page +
115 : MMUV2_TRASH_L2_BLOCK_OFFSET,
116 0 : mmu_hw->base +
117 : MMUV2_AT_REG_L1_ZLW_INSERTION(0));
118 : }
119 : }
120 :
121 : /*
122 : * Wait until AT is ready. FIFO read should return 2 when AT is ready.
123 : * Retry value of 1000 is just by guess work to avoid the forever loop.
124 : */
125 : do {
126 0 : if (retry > 1000) {
127 0 : dev_err(mmu->dev, "zlw invalidation failed\n");
128 0 : return;
129 : }
130 0 : ret = readl(mmu_hw->base + MMUV2_AT_REG_L1_FW_ZLW_FIFO);
131 0 : retry++;
132 0 : } while (ret != 2);
133 : }
134 :
135 3 : static void tlb_invalidate(struct ipu6_mmu *mmu)
136 : {
137 : unsigned long flags;
138 : unsigned int i;
139 :
140 3 : spin_lock_irqsave(&mmu->ready_lock, flags);
141 3 : if (!mmu->ready) {
142 : spin_unlock_irqrestore(&mmu->ready_lock, flags);
143 0 : return;
144 : }
145 :
146 9 : for (i = 0; i < mmu->nr_mmus; i++) {
147 : /*
148 : * To avoid the HW bug induced dead lock in some of the IPU6
149 : * MMUs on successive invalidate calls, we need to first do a
150 : * read to the page table base before writing the invalidate
151 : * register. MMUs which need to implement this WA, will have
152 : * the insert_read_before_invalidate flags set as true.
153 : * Disregard the return value of the read.
154 : */
155 6 : if (mmu->mmu_hw[i].insert_read_before_invalidate)
156 3 : readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
157 :
158 : /* Normal invalidate or zlw invalidate */
159 6 : if (mmu->mmu_hw[i].zlw_invalidate) {
160 : /* trash buffer must be mapped by now, just in case! */
161 0 : WARN_ON(!mmu->iova_trash_page);
162 :
163 0 : zlw_invalidate(mmu, &mmu->mmu_hw[i]);
164 : } else {
165 : u32 inv;
166 :
167 6 : if (mmu->mmu_hw[i].nr_l1streams == 32)
168 : inv = 0xffffffff;
169 6 : else if (mmu->mmu_hw[i].nr_l1streams == 0)
170 : inv = MMU0_TLB_INVALIDATE;
171 : else
172 : inv = MMU1_TLB_INVALIDATE;
173 6 : writel(inv, mmu->mmu_hw[i].base +
174 : REG_TLB_INVALIDATE);
175 : }
176 : /*
177 : * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
178 : * When the actual MMIO write reaches the IPU6 TLB Invalidate
179 : * register, wmb() will force the TLB invalidate out if the CPU
180 : * attempts to update the IOMMU page table (or sooner).
181 : */
182 6 : wmb();
183 : }
184 : spin_unlock_irqrestore(&mmu->ready_lock, flags);
185 : }
186 :
187 : #ifdef DEBUG
188 : static void page_table_dump(struct ipu6_mmu_info *mmu_info)
189 : {
190 : u32 l1_idx;
191 :
192 : dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");
193 :
194 : for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
195 : u32 l2_idx;
196 : u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;
197 : phys_addr_t l2_phys;
198 :
199 : if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)
200 : continue;
201 :
202 : l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx];)
203 : dev_dbg(mmu_info->dev,
204 : "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pap\n",
205 : l1_idx, iova, iova + ISP_PAGE_SIZE, &l2_phys);
206 :
207 : for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {
208 : u32 *l2_pt = mmu_info->l2_pts[l1_idx];
209 : u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);
210 :
211 : if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)
212 : continue;
213 :
214 : dev_dbg(mmu_info->dev,
215 : "\tl2 entry %u; iova 0x%8.8x, phys %pa\n",
216 : l2_idx, iova2,
217 : TBL_PHYS_ADDR(l2_pt[l2_idx]));
218 : }
219 : }
220 :
221 : dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");
222 : }
223 : #endif /* DEBUG */
224 :
225 12 : static dma_addr_t map_single(struct ipu6_mmu_info *mmu_info, void *ptr)
226 : {
227 : dma_addr_t dma;
228 :
229 12 : dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL);
230 : if (dma_mapping_error(mmu_info->dev, dma))
231 0 : return 0;
232 :
233 : return dma;
234 : }
235 :
236 2 : static int get_dummy_page(struct ipu6_mmu_info *mmu_info)
237 : {
238 2 : void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
239 : dma_addr_t dma;
240 :
241 2 : if (!pt)
242 : return -ENOMEM;
243 :
244 2 : dev_dbg(mmu_info->dev, "dummy_page: get_zeroed_page() == %p\n", pt);
245 :
246 2 : dma = map_single(mmu_info, pt);
247 2 : if (!dma) {
248 0 : dev_err(mmu_info->dev, "Failed to map dummy page\n");
249 0 : goto err_free_page;
250 : }
251 :
252 2 : mmu_info->dummy_page = pt;
253 2 : mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT;
254 :
255 2 : return 0;
256 :
257 : err_free_page:
258 0 : free_page((unsigned long)pt);
259 0 : return -ENOMEM;
260 : }
261 :
262 0 : static void free_dummy_page(struct ipu6_mmu_info *mmu_info)
263 : {
264 0 : dma_unmap_single(mmu_info->dev,
265 : TBL_PHYS_ADDR(mmu_info->dummy_page_pteval),
266 : PAGE_SIZE, DMA_BIDIRECTIONAL);
267 0 : free_page((unsigned long)mmu_info->dummy_page);
268 0 : }
269 :
270 2 : static int alloc_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
271 : {
272 2 : u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
273 : dma_addr_t dma;
274 : unsigned int i;
275 :
276 2 : if (!pt)
277 : return -ENOMEM;
278 :
279 2 : dev_dbg(mmu_info->dev, "dummy_l2: get_zeroed_page() = %p\n", pt);
280 :
281 2 : dma = map_single(mmu_info, pt);
282 2 : if (!dma) {
283 0 : dev_err(mmu_info->dev, "Failed to map l2pt page\n");
284 0 : goto err_free_page;
285 : }
286 :
287 2050 : for (i = 0; i < ISP_L2PT_PTES; i++)
288 2048 : pt[i] = mmu_info->dummy_page_pteval;
289 :
290 2 : mmu_info->dummy_l2_pt = pt;
291 2 : mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT;
292 :
293 2 : return 0;
294 :
295 : err_free_page:
296 0 : free_page((unsigned long)pt);
297 0 : return -ENOMEM;
298 : }
299 :
300 0 : static void free_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
301 : {
302 0 : dma_unmap_single(mmu_info->dev,
303 : TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval),
304 : PAGE_SIZE, DMA_BIDIRECTIONAL);
305 0 : free_page((unsigned long)mmu_info->dummy_l2_pt);
306 0 : }
307 :
308 2 : static u32 *alloc_l1_pt(struct ipu6_mmu_info *mmu_info)
309 : {
310 2 : u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
311 2 : dma_addr_t dma;
312 : unsigned int i;
313 :
314 2 : if (!pt)
315 : return NULL;
316 :
317 2 : dev_dbg(mmu_info->dev, "alloc_l1: get_zeroed_page() = %p\n", pt);
318 :
319 2050 : for (i = 0; i < ISP_L1PT_PTES; i++)
320 2048 : pt[i] = mmu_info->dummy_l2_pteval;
321 :
322 2 : dma = map_single(mmu_info, pt);
323 2 : if (!dma) {
324 0 : dev_err(mmu_info->dev, "Failed to map l1pt page\n");
325 0 : goto err_free_page;
326 : }
327 :
328 2 : mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT;
329 2 : dev_dbg(mmu_info->dev, "l1 pt %p mapped at %pad\n", pt, &dma);
330 :
331 : return pt;
332 :
333 : err_free_page:
334 0 : free_page((unsigned long)pt);
335 0 : return NULL;
336 : }
337 :
338 6 : static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info)
339 : {
340 6 : u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
341 : unsigned int i;
342 :
343 6 : if (!pt)
344 : return NULL;
345 :
346 6 : dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt);
347 :
348 6150 : for (i = 0; i < ISP_L1PT_PTES; i++)
349 6144 : pt[i] = mmu_info->dummy_page_pteval;
350 :
351 : return pt;
352 : }
353 :
354 3 : static void l2_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
355 : phys_addr_t dummy, size_t size)
356 : {
357 : unsigned int l2_entries;
358 : unsigned int l2_idx;
359 : unsigned long flags;
360 : u32 l1_idx;
361 : u32 *l2_pt;
362 :
363 3 : spin_lock_irqsave(&mmu_info->lock, flags);
364 3 : for (l1_idx = iova >> ISP_L1PT_SHIFT;
365 6 : size > 0 && l1_idx < ISP_L1PT_PTES; l1_idx++) {
366 3 : dev_dbg(mmu_info->dev,
367 : "unmapping l2 pgtable (l1 index %u (iova 0x%8.8lx))\n",
368 : l1_idx, iova);
369 :
370 3 : if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) {
371 0 : dev_err(mmu_info->dev,
372 : "unmap not mapped iova 0x%8.8lx l1 index %u\n",
373 : iova, l1_idx);
374 0 : continue;
375 : }
376 3 : l2_pt = mmu_info->l2_pts[l1_idx];
377 :
378 : l2_entries = 0;
379 3 : for (l2_idx = (iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
380 958 : size > 0 && l2_idx < ISP_L2PT_PTES; l2_idx++) {
381 955 : phys_addr_t pteval = TBL_PHYS_ADDR(l2_pt[l2_idx]);
382 :
383 955 : dev_dbg(mmu_info->dev,
384 : "unmap l2 index %u with pteval 0x%p\n",
385 : l2_idx, &pteval);
386 955 : l2_pt[l2_idx] = mmu_info->dummy_page_pteval;
387 :
388 955 : iova += ISP_PAGE_SIZE;
389 955 : size -= ISP_PAGE_SIZE;
390 :
391 955 : l2_entries++;
392 : }
393 :
394 3 : WARN_ON_ONCE(!l2_entries);
395 3 : clflush_cache_range(&l2_pt[l2_idx - l2_entries],
396 : sizeof(l2_pt[0]) * l2_entries);
397 : }
398 :
399 3 : WARN_ON_ONCE(size);
400 : spin_unlock_irqrestore(&mmu_info->lock, flags);
401 3 : }
402 :
403 4132 : static int l2_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
404 : phys_addr_t paddr, size_t size)
405 : {
406 4132 : struct device *dev = mmu_info->dev;
407 : unsigned int l2_entries;
408 : u32 *l2_pt, *l2_virt;
409 : unsigned int l2_idx;
410 : unsigned long flags;
411 : size_t mapped = 0;
412 : dma_addr_t dma;
413 : u32 l1_entry;
414 : u32 l1_idx;
415 : int err = 0;
416 :
417 4132 : spin_lock_irqsave(&mmu_info->lock, flags);
418 :
419 4132 : paddr = ALIGN(paddr, ISP_PAGE_SIZE);
420 4132 : for (l1_idx = iova >> ISP_L1PT_SHIFT;
421 8264 : size > 0 && l1_idx < ISP_L1PT_PTES; l1_idx++) {
422 4132 : dev_dbg(dev,
423 : "mapping l2 page table for l1 index %u (iova %8.8x)\n",
424 : l1_idx, (u32)iova);
425 :
426 4132 : l1_entry = mmu_info->l1_pt[l1_idx];
427 4132 : if (l1_entry == mmu_info->dummy_l2_pteval) {
428 6 : l2_virt = mmu_info->l2_pts[l1_idx];
429 6 : if (likely(!l2_virt)) {
430 6 : l2_virt = alloc_l2_pt(mmu_info);
431 6 : if (!l2_virt) {
432 : err = -ENOMEM;
433 0 : goto error;
434 : }
435 : }
436 :
437 6 : dma = map_single(mmu_info, l2_virt);
438 6 : if (!dma) {
439 0 : dev_err(dev, "Failed to map l2pt page\n");
440 0 : free_page((unsigned long)l2_virt);
441 : err = -EINVAL;
442 0 : goto error;
443 : }
444 :
445 6 : l1_entry = dma >> ISP_PADDR_SHIFT;
446 :
447 6 : dev_dbg(dev, "page for l1_idx %u %p allocated\n",
448 : l1_idx, l2_virt);
449 6 : mmu_info->l1_pt[l1_idx] = l1_entry;
450 6 : mmu_info->l2_pts[l1_idx] = l2_virt;
451 :
452 6 : clflush_cache_range(&mmu_info->l1_pt[l1_idx],
453 : sizeof(mmu_info->l1_pt[l1_idx]));
454 : }
455 :
456 4132 : l2_pt = mmu_info->l2_pts[l1_idx];
457 : l2_entries = 0;
458 :
459 4132 : for (l2_idx = (iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
460 9205 : size > 0 && l2_idx < ISP_L2PT_PTES; l2_idx++) {
461 5073 : l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT;
462 :
463 5073 : dev_dbg(dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx,
464 : l2_pt[l2_idx]);
465 :
466 5073 : iova += ISP_PAGE_SIZE;
467 5073 : paddr += ISP_PAGE_SIZE;
468 5073 : mapped += ISP_PAGE_SIZE;
469 5073 : size -= ISP_PAGE_SIZE;
470 :
471 5073 : l2_entries++;
472 : }
473 :
474 4132 : WARN_ON_ONCE(!l2_entries);
475 4132 : clflush_cache_range(&l2_pt[l2_idx - l2_entries],
476 : sizeof(l2_pt[0]) * l2_entries);
477 : }
478 :
479 : spin_unlock_irqrestore(&mmu_info->lock, flags);
480 :
481 4132 : return 0;
482 :
483 0 : error:
484 : spin_unlock_irqrestore(&mmu_info->lock, flags);
485 : /* unroll mapping in case something went wrong */
486 0 : if (mapped)
487 0 : l2_unmap(mmu_info, iova - mapped, paddr - mapped, mapped);
488 :
489 : return err;
490 : }
491 :
492 4132 : static int __ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
493 : phys_addr_t paddr, size_t size)
494 : {
495 4132 : u32 iova_start = round_down(iova, ISP_PAGE_SIZE);
496 4132 : u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE);
497 :
498 4132 : dev_dbg(mmu_info->dev,
499 : "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr %pap\n",
500 : iova_start, iova_end, size, &paddr);
501 :
502 4132 : return l2_map(mmu_info, iova_start, paddr, size);
503 : }
504 :
505 : static void __ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info,
506 : unsigned long iova, size_t size)
507 : {
508 3 : l2_unmap(mmu_info, iova, 0, size);
509 3 : }
510 :
511 2 : static int allocate_trash_buffer(struct ipu6_mmu *mmu)
512 : {
513 : unsigned int n_pages = PFN_UP(IPU6_MMUV2_TRASH_RANGE);
514 : struct iova *iova;
515 : unsigned int i;
516 : dma_addr_t dma;
517 : unsigned long iova_addr;
518 : int ret;
519 :
520 : /* Allocate 8MB in iova range */
521 2 : iova = alloc_iova(&mmu->dmap->iovad, n_pages,
522 2 : PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
523 2 : if (!iova) {
524 0 : dev_err(mmu->dev, "cannot allocate iova range for trash\n");
525 0 : return -ENOMEM;
526 : }
527 :
528 2 : dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,
529 : PAGE_SIZE, DMA_BIDIRECTIONAL);
530 2 : if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {
531 0 : dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");
532 : ret = -ENOMEM;
533 0 : goto out_free_iova;
534 : }
535 :
536 2 : mmu->pci_trash_page = dma;
537 :
538 : /*
539 : * Map the 8MB iova address range to the same physical trash page
540 : * mmu->trash_page which is already reserved at the probe
541 : */
542 2 : iova_addr = iova->pfn_lo;
543 4098 : for (i = 0; i < n_pages; i++) {
544 4096 : ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),
545 : mmu->pci_trash_page, PAGE_SIZE);
546 4096 : if (ret) {
547 0 : dev_err(mmu->dev,
548 : "mapping trash buffer range failed\n");
549 0 : goto out_unmap;
550 : }
551 :
552 4096 : iova_addr++;
553 : }
554 :
555 2 : mmu->iova_trash_page = PFN_PHYS(iova->pfn_lo);
556 2 : dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",
557 : mmu->mmid, (unsigned int)mmu->iova_trash_page);
558 : return 0;
559 :
560 : out_unmap:
561 0 : ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
562 : PFN_PHYS(iova_size(iova)));
563 0 : dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,
564 : PAGE_SIZE, DMA_BIDIRECTIONAL);
565 0 : out_free_iova:
566 0 : __free_iova(&mmu->dmap->iovad, iova);
567 0 : return ret;
568 : }
569 :
570 3 : int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
571 : {
572 : struct ipu6_mmu_info *mmu_info;
573 : unsigned long flags;
574 : unsigned int i;
575 :
576 3 : mmu_info = mmu->dmap->mmu_info;
577 :
578 : /* Initialise the each MMU HW block */
579 10 : for (i = 0; i < mmu->nr_mmus; i++) {
580 7 : struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];
581 : unsigned int j;
582 : u16 block_addr;
583 : bool zlw_invalidate_in_any = false;
584 :
585 : /* Write page table address per MMU */
586 7 : writel((phys_addr_t)mmu_info->l1_pt_dma,
587 7 : mmu->mmu_hw[i].base + REG_L1_PHYS);
588 :
589 : /* Set info bits per MMU */
590 7 : writel(mmu->mmu_hw[i].info_bits,
591 7 : mmu->mmu_hw[i].base + REG_INFO);
592 :
593 : /* Configure MMU TLB stream configuration for L1 */
594 71 : for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams;
595 64 : block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {
596 64 : if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
597 0 : dev_err(mmu->dev, "invalid L1 configuration\n");
598 0 : return -EINVAL;
599 : }
600 :
601 : /* Write block start address for each streams */
602 64 : writel(block_addr, mmu_hw->base +
603 64 : mmu_hw->l1_stream_id_reg_offset + 4 * j);
604 :
605 : /* Enable ZLW for streams based on the init table */
606 64 : writel(mmu->mmu_hw[i].l1_zlw_en[j],
607 64 : mmu_hw->base +
608 64 : MMUV2_AT_REG_L1_ZLW_EN_SID(j));
609 :
610 : /* To track if zlw is enabled in any streams */
611 64 : zlw_invalidate_in_any |= mmu->mmu_hw[i].l1_zlw_en[j];
612 :
613 : /* Enable ZLW 1D mode for streams from the init table */
614 64 : writel(mmu->mmu_hw[i].l1_zlw_1d_mode[j],
615 64 : mmu_hw->base +
616 64 : MMUV2_AT_REG_L1_ZLW_1DMODE_SID(j));
617 :
618 : /* Set when the ZLW insertion will happen */
619 64 : writel(mmu->mmu_hw[i].l1_ins_zlw_ahead_pages[j],
620 64 : mmu_hw->base +
621 64 : MMUV2_AT_REG_L1_ZLW_INS_N_AHEAD_SID(j));
622 :
623 : /* Set if ZLW 2D mode active for each streams */
624 64 : writel(mmu->mmu_hw[i].l1_zlw_2d_mode[j],
625 64 : mmu_hw->base +
626 64 : MMUV2_AT_REG_L1_ZLW_2DMODE_SID(j));
627 : }
628 :
629 7 : if (zlw_invalidate_in_any)
630 4 : writel(1, mmu_hw->base +
631 : MMUV2_AT_REG_L1_FW_ZLW_PRIO);
632 :
633 : /* Configure MMU TLB stream configuration for L2 */
634 71 : for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams;
635 64 : block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {
636 64 : if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
637 0 : dev_err(mmu->dev, "invalid L2 configuration\n");
638 0 : return -EINVAL;
639 : }
640 :
641 64 : writel(block_addr, mmu_hw->base +
642 64 : mmu_hw->l2_stream_id_reg_offset + 4 * j);
643 : }
644 : }
645 :
646 3 : if (!mmu->trash_page) {
647 : int ret;
648 :
649 2 : mmu->trash_page = alloc_page(GFP_KERNEL);
650 2 : if (!mmu->trash_page) {
651 0 : dev_err(mmu->dev, "insufficient memory for trash buffer\n");
652 0 : return -ENOMEM;
653 : }
654 :
655 2 : ret = allocate_trash_buffer(mmu);
656 2 : if (ret) {
657 0 : __free_page(mmu->trash_page);
658 0 : mmu->trash_page = NULL;
659 0 : dev_err(mmu->dev, "trash buffer allocation failed\n");
660 0 : return ret;
661 : }
662 : }
663 :
664 3 : spin_lock_irqsave(&mmu->ready_lock, flags);
665 3 : mmu->ready = true;
666 : spin_unlock_irqrestore(&mmu->ready_lock, flags);
667 :
668 3 : return 0;
669 : }
670 : EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_init, INTEL_IPU6);
671 :
672 2 : static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp)
673 : {
674 : struct ipu6_mmu_info *mmu_info;
675 : int ret;
676 :
677 2 : mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL);
678 2 : if (!mmu_info)
679 : return NULL;
680 :
681 2 : mmu_info->aperture_start = 0;
682 2 : mmu_info->aperture_end =
683 2 : (dma_addr_t)DMA_BIT_MASK(isp->secure_mode ?
684 : IPU6_MMU_ADDR_BITS :
685 : IPU6_MMU_ADDR_BITS_NON_SECURE);
686 2 : mmu_info->pgsize_bitmap = SZ_4K;
687 2 : mmu_info->dev = &isp->pdev->dev;
688 :
689 2 : ret = get_dummy_page(mmu_info);
690 2 : if (ret)
691 0 : goto err_free_info;
692 :
693 2 : ret = alloc_dummy_l2_pt(mmu_info);
694 2 : if (ret)
695 0 : goto err_free_dummy_page;
696 :
697 2 : mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts));
698 2 : if (!mmu_info->l2_pts)
699 0 : goto err_free_dummy_l2_pt;
700 :
701 : /*
702 : * We always map the L1 page table (a single page as well as
703 : * the L2 page tables).
704 : */
705 2 : mmu_info->l1_pt = alloc_l1_pt(mmu_info);
706 2 : if (!mmu_info->l1_pt)
707 0 : goto err_free_l2_pts;
708 :
709 2 : spin_lock_init(&mmu_info->lock);
710 :
711 2 : dev_dbg(mmu_info->dev, "domain initialised\n");
712 :
713 : return mmu_info;
714 :
715 : err_free_l2_pts:
716 0 : vfree(mmu_info->l2_pts);
717 0 : err_free_dummy_l2_pt:
718 0 : free_dummy_l2_pt(mmu_info);
719 0 : err_free_dummy_page:
720 0 : free_dummy_page(mmu_info);
721 0 : err_free_info:
722 0 : kfree(mmu_info);
723 :
724 0 : return NULL;
725 : }
726 :
727 3 : void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu)
728 : {
729 : unsigned long flags;
730 :
731 3 : spin_lock_irqsave(&mmu->ready_lock, flags);
732 3 : mmu->ready = false;
733 : spin_unlock_irqrestore(&mmu->ready_lock, flags);
734 3 : }
735 : EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_cleanup, INTEL_IPU6);
736 :
737 2 : static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)
738 : {
739 : struct ipu6_dma_mapping *dmap;
740 :
741 2 : dmap = kzalloc(sizeof(*dmap), GFP_KERNEL);
742 2 : if (!dmap)
743 : return NULL;
744 :
745 2 : dmap->mmu_info = ipu6_mmu_alloc(isp);
746 2 : if (!dmap->mmu_info) {
747 0 : kfree(dmap);
748 0 : return NULL;
749 : }
750 :
751 2 : init_iova_domain(&dmap->iovad, SZ_4K, 1);
752 2 : dmap->mmu_info->dmap = dmap;
753 :
754 2 : dev_dbg(&isp->pdev->dev, "alloc mapping\n");
755 :
756 2 : iova_cache_get();
757 :
758 2 : return dmap;
759 : }
760 :
761 14 : phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info,
762 : dma_addr_t iova)
763 : {
764 : phys_addr_t phy_addr;
765 : unsigned long flags;
766 : u32 *l2_pt;
767 :
768 14 : spin_lock_irqsave(&mmu_info->lock, flags);
769 14 : l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT];
770 14 : phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT];
771 14 : phy_addr <<= ISP_PAGE_SHIFT;
772 : spin_unlock_irqrestore(&mmu_info->lock, flags);
773 :
774 14 : return phy_addr;
775 : }
776 :
777 3 : void ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
778 : size_t size)
779 : {
780 : unsigned int min_pagesz;
781 :
782 3 : dev_dbg(mmu_info->dev, "unmapping iova 0x%lx size 0x%zx\n", iova, size);
783 :
784 : /* find out the minimum page size supported */
785 3 : min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);
786 :
787 : /*
788 : * The virtual address and the size of the mapping must be
789 : * aligned (at least) to the size of the smallest page supported
790 : * by the hardware
791 : */
792 3 : if (!IS_ALIGNED(iova | size, min_pagesz)) {
793 0 : dev_err(NULL, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
794 : iova, size, min_pagesz);
795 0 : return;
796 : }
797 :
798 : __ipu6_mmu_unmap(mmu_info, iova, size);
799 : }
800 :
801 4132 : int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
802 : phys_addr_t paddr, size_t size)
803 : {
804 : unsigned int min_pagesz;
805 :
806 4132 : if (mmu_info->pgsize_bitmap == 0UL)
807 : return -ENODEV;
808 :
809 : /* find out the minimum page size supported */
810 4132 : min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);
811 :
812 : /*
813 : * both the virtual address and the physical one, as well as
814 : * the size of the mapping, must be aligned (at least) to the
815 : * size of the smallest page supported by the hardware
816 : */
817 4132 : if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
818 0 : dev_err(mmu_info->dev,
819 : "unaligned: iova %lx pa %pa size %zx min_pagesz %x\n",
820 : iova, &paddr, size, min_pagesz);
821 0 : return -EINVAL;
822 : }
823 :
824 4132 : dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
825 : iova, &paddr, size);
826 :
827 4132 : return __ipu6_mmu_map(mmu_info, iova, paddr, size);
828 : }
829 :
830 0 : static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)
831 : {
832 0 : struct ipu6_dma_mapping *dmap = mmu->dmap;
833 0 : struct ipu6_mmu_info *mmu_info = dmap->mmu_info;
834 : struct iova *iova;
835 : u32 l1_idx;
836 :
837 0 : if (mmu->iova_trash_page) {
838 0 : iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page));
839 0 : if (iova) {
840 : /* unmap and free the trash buffer iova */
841 0 : ipu6_mmu_unmap(mmu_info, PFN_PHYS(iova->pfn_lo),
842 : PFN_PHYS(iova_size(iova)));
843 0 : __free_iova(&dmap->iovad, iova);
844 : } else {
845 0 : dev_err(mmu->dev, "trash buffer iova not found.\n");
846 : }
847 :
848 0 : mmu->iova_trash_page = 0;
849 0 : dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,
850 : PAGE_SIZE, DMA_BIDIRECTIONAL);
851 0 : mmu->pci_trash_page = 0;
852 0 : __free_page(mmu->trash_page);
853 : }
854 :
855 0 : for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
856 0 : if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) {
857 0 : dma_unmap_single(mmu_info->dev,
858 : TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]),
859 : PAGE_SIZE, DMA_BIDIRECTIONAL);
860 0 : free_page((unsigned long)mmu_info->l2_pts[l1_idx]);
861 : }
862 : }
863 :
864 0 : vfree(mmu_info->l2_pts);
865 0 : free_dummy_page(mmu_info);
866 0 : dma_unmap_single(mmu_info->dev, TBL_PHYS_ADDR(mmu_info->l1_pt_dma),
867 : PAGE_SIZE, DMA_BIDIRECTIONAL);
868 0 : free_page((unsigned long)mmu_info->dummy_l2_pt);
869 0 : free_page((unsigned long)mmu_info->l1_pt);
870 0 : kfree(mmu_info);
871 0 : }
872 :
873 2 : struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
874 : void __iomem *base, int mmid,
875 : const struct ipu6_hw_variants *hw)
876 : {
877 : struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));
878 : struct ipu6_mmu_pdata *pdata;
879 : struct ipu6_mmu *mmu;
880 : unsigned int i;
881 :
882 2 : if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES)
883 : return ERR_PTR(-EINVAL);
884 :
885 : pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
886 2 : if (!pdata)
887 : return ERR_PTR(-ENOMEM);
888 :
889 7 : for (i = 0; i < hw->nr_mmus; i++) {
890 : struct ipu6_mmu_hw *pdata_mmu = &pdata->mmu_hw[i];
891 : const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i];
892 :
893 5 : if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
894 5 : src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
895 : return ERR_PTR(-EINVAL);
896 :
897 5 : *pdata_mmu = *src_mmu;
898 5 : pdata_mmu->base = base + src_mmu->offset;
899 : }
900 :
901 : mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
902 2 : if (!mmu)
903 : return ERR_PTR(-ENOMEM);
904 :
905 2 : mmu->mmid = mmid;
906 2 : mmu->mmu_hw = pdata->mmu_hw;
907 2 : mmu->nr_mmus = hw->nr_mmus;
908 2 : mmu->tlb_invalidate = tlb_invalidate;
909 2 : mmu->ready = false;
910 2 : INIT_LIST_HEAD(&mmu->vma_list);
911 2 : spin_lock_init(&mmu->ready_lock);
912 :
913 2 : mmu->dmap = alloc_dma_mapping(isp);
914 2 : if (!mmu->dmap) {
915 0 : dev_err(dev, "can't alloc dma mapping\n");
916 0 : return ERR_PTR(-ENOMEM);
917 : }
918 :
919 : return mmu;
920 : }
921 :
922 0 : void ipu6_mmu_cleanup(struct ipu6_mmu *mmu)
923 : {
924 0 : struct ipu6_dma_mapping *dmap = mmu->dmap;
925 :
926 0 : ipu6_mmu_destroy(mmu);
927 0 : mmu->dmap = NULL;
928 0 : iova_cache_put();
929 0 : put_iova_domain(&dmap->iovad);
930 0 : kfree(dmap);
931 0 : }
|
