Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Copyright (C) 2013--2024 Intel Corporation
4 : */
5 :
6 : #include <linux/cacheflush.h>
7 : #include <linux/dma-mapping.h>
8 : #include <linux/iova.h>
9 : #include <linux/list.h>
10 : #include <linux/mm.h>
11 : #include <linux/vmalloc.h>
12 : #include <linux/scatterlist.h>
13 : #include <linux/slab.h>
14 : #include <linux/types.h>
15 :
16 : #include "ipu6.h"
17 : #include "ipu6-bus.h"
18 : #include "ipu6-dma.h"
19 : #include "ipu6-mmu.h"
20 :
21 : struct vm_info {
22 : struct list_head list;
23 : struct page **pages;
24 : dma_addr_t ipu6_iova;
25 : void *vaddr;
26 : unsigned long size;
27 : };
28 :
29 3 : static struct vm_info *get_vm_info(struct ipu6_mmu *mmu, dma_addr_t iova)
30 : {
31 : struct vm_info *info, *save;
32 :
33 3 : list_for_each_entry_safe(info, save, &mmu->vma_list, list) {
34 3 : if (iova >= info->ipu6_iova &&
35 3 : iova < (info->ipu6_iova + info->size))
36 3 : return info;
37 : }
38 :
39 : return NULL;
40 : }
41 :
42 26 : static void __clear_buffer(struct page *page, size_t size, unsigned long attrs)
43 : {
44 : void *ptr;
45 :
46 26 : if (!page)
47 : return;
48 : /*
49 : * Ensure that the allocated pages are zeroed, and that any data
50 : * lurking in the kernel direct-mapped region is invalidated.
51 : */
52 : ptr = page_address(page);
53 26 : memset(ptr, 0, size);
54 26 : if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
55 26 : clflush_cache_range(ptr, size);
56 : }
57 :
58 22 : static struct page **__alloc_buffer(size_t size, gfp_t gfp, unsigned long attrs)
59 : {
60 22 : int count = PHYS_PFN(size);
61 22 : int array_size = count * sizeof(struct page *);
62 : struct page **pages;
63 : int i = 0;
64 :
65 22 : pages = kvzalloc(array_size, GFP_KERNEL);
66 22 : if (!pages)
67 : return NULL;
68 :
69 22 : gfp |= __GFP_NOWARN;
70 :
71 45 : while (count) {
72 23 : int j, order = __fls(count);
73 :
74 23 : pages[i] = alloc_pages(gfp, order);
75 23 : while (!pages[i] && order)
76 0 : pages[i] = alloc_pages(gfp, --order);
77 23 : if (!pages[i])
78 0 : goto error;
79 :
80 23 : if (order) {
81 1 : split_page(pages[i], order);
82 1 : j = 1 << order;
83 3 : while (j--)
84 2 : pages[i + j] = pages[i] + j;
85 : }
86 :
87 23 : __clear_buffer(pages[i], PAGE_SIZE << order, attrs);
88 23 : i += 1 << order;
89 23 : count -= 1 << order;
90 : }
91 :
92 : return pages;
93 : error:
94 0 : while (i--)
95 0 : if (pages[i])
96 0 : __free_pages(pages[i], 0);
97 0 : kvfree(pages);
98 0 : return NULL;
99 : }
100 :
101 1 : static void __free_buffer(struct page **pages, size_t size, unsigned long attrs)
102 : {
103 1 : int count = PHYS_PFN(size);
104 : unsigned int i;
105 :
106 4 : for (i = 0; i < count && pages[i]; i++) {
107 3 : __clear_buffer(pages[i], PAGE_SIZE, attrs);
108 3 : __free_pages(pages[i], 0);
109 : }
110 :
111 1 : kvfree(pages);
112 1 : }
113 :
114 2 : void ipu6_dma_sync_single(struct ipu6_bus_device *sys, dma_addr_t dma_handle,
115 : size_t size)
116 : {
117 : void *vaddr;
118 : u32 offset;
119 : struct vm_info *info;
120 2 : struct ipu6_mmu *mmu = sys->mmu;
121 :
122 2 : info = get_vm_info(mmu, dma_handle);
123 2 : if (WARN_ON(!info))
124 0 : return;
125 :
126 2 : offset = dma_handle - info->ipu6_iova;
127 2 : if (WARN_ON(size > (info->size - offset)))
128 : return;
129 :
130 2 : vaddr = info->vaddr + offset;
131 2 : clflush_cache_range(vaddr, size);
132 : }
133 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_single, INTEL_IPU6);
134 :
135 1 : void ipu6_dma_sync_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
136 : int nents)
137 : {
138 : struct scatterlist *sg;
139 : int i;
140 :
141 2 : for_each_sg(sglist, sg, nents, i)
142 1 : clflush_cache_range(sg_virt(sg), sg->length);
143 1 : }
144 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sg, INTEL_IPU6);
145 :
146 1 : void ipu6_dma_sync_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt)
147 : {
148 1 : ipu6_dma_sync_sg(sys, sgt->sgl, sgt->orig_nents);
149 1 : }
150 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sgtable, INTEL_IPU6);
151 :
152 22 : void *ipu6_dma_alloc(struct ipu6_bus_device *sys, size_t size,
153 : dma_addr_t *dma_handle, gfp_t gfp,
154 : unsigned long attrs)
155 : {
156 22 : struct device *dev = &sys->auxdev.dev;
157 22 : struct pci_dev *pdev = sys->isp->pdev;
158 22 : dma_addr_t pci_dma_addr, ipu6_iova;
159 22 : struct ipu6_mmu *mmu = sys->mmu;
160 : struct vm_info *info;
161 : unsigned long count;
162 : struct page **pages;
163 : struct iova *iova;
164 : unsigned int i;
165 : int ret;
166 :
167 22 : info = kzalloc(sizeof(*info), GFP_KERNEL);
168 22 : if (!info)
169 : return NULL;
170 :
171 22 : size = PAGE_ALIGN(size);
172 22 : count = PHYS_PFN(size);
173 :
174 22 : iova = alloc_iova(&mmu->dmap->iovad, count,
175 22 : PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
176 22 : if (!iova)
177 0 : goto out_kfree;
178 :
179 22 : pages = __alloc_buffer(size, gfp, attrs);
180 22 : if (!pages)
181 0 : goto out_free_iova;
182 :
183 22 : dev_dbg(dev, "dma_alloc: size %zu iova low pfn %lu, high pfn %lu\n",
184 : size, iova->pfn_lo, iova->pfn_hi);
185 46 : for (i = 0; iova->pfn_lo + i <= iova->pfn_hi; i++) {
186 24 : pci_dma_addr = dma_map_page_attrs(&pdev->dev, pages[i], 0,
187 : PAGE_SIZE, DMA_BIDIRECTIONAL,
188 : attrs);
189 24 : dev_dbg(dev, "dma_alloc: mapped pci_dma_addr %pad\n",
190 : &pci_dma_addr);
191 24 : if (dma_mapping_error(&pdev->dev, pci_dma_addr)) {
192 0 : dev_err(dev, "pci_dma_mapping for page[%d] failed", i);
193 0 : goto out_unmap;
194 : }
195 :
196 24 : ret = ipu6_mmu_map(mmu->dmap->mmu_info,
197 24 : PFN_PHYS(iova->pfn_lo + i), pci_dma_addr,
198 : PAGE_SIZE);
199 24 : if (ret) {
200 0 : dev_err(dev, "ipu6_mmu_map for pci_dma[%d] %pad failed",
201 : i, &pci_dma_addr);
202 0 : dma_unmap_page_attrs(&pdev->dev, pci_dma_addr,
203 : PAGE_SIZE, DMA_BIDIRECTIONAL,
204 : attrs);
205 0 : goto out_unmap;
206 : }
207 : }
208 :
209 22 : info->vaddr = vmap(pages, count, VM_USERMAP, PAGE_KERNEL);
210 22 : if (!info->vaddr)
211 0 : goto out_unmap;
212 :
213 22 : *dma_handle = PFN_PHYS(iova->pfn_lo);
214 :
215 22 : info->pages = pages;
216 22 : info->ipu6_iova = *dma_handle;
217 22 : info->size = size;
218 22 : list_add(&info->list, &mmu->vma_list);
219 :
220 22 : return info->vaddr;
221 :
222 : out_unmap:
223 0 : while (i--) {
224 0 : ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
225 0 : pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
226 : ipu6_iova);
227 0 : dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
228 : DMA_BIDIRECTIONAL, attrs);
229 :
230 0 : ipu6_mmu_unmap(mmu->dmap->mmu_info, ipu6_iova, PAGE_SIZE);
231 : }
232 :
233 0 : __free_buffer(pages, size, attrs);
234 :
235 0 : out_free_iova:
236 0 : __free_iova(&mmu->dmap->iovad, iova);
237 0 : out_kfree:
238 0 : kfree(info);
239 :
240 0 : return NULL;
241 : }
242 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_alloc, INTEL_IPU6);
243 :
244 1 : void ipu6_dma_free(struct ipu6_bus_device *sys, size_t size, void *vaddr,
245 : dma_addr_t dma_handle, unsigned long attrs)
246 : {
247 1 : struct ipu6_mmu *mmu = sys->mmu;
248 1 : struct pci_dev *pdev = sys->isp->pdev;
249 1 : struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle));
250 : dma_addr_t pci_dma_addr, ipu6_iova;
251 : struct vm_info *info;
252 : struct page **pages;
253 : unsigned int i;
254 :
255 1 : if (WARN_ON(!iova))
256 0 : return;
257 :
258 1 : info = get_vm_info(mmu, dma_handle);
259 1 : if (WARN_ON(!info))
260 0 : return;
261 :
262 1 : if (WARN_ON(!info->vaddr))
263 0 : return;
264 :
265 1 : if (WARN_ON(!info->pages))
266 0 : return;
267 :
268 : list_del(&info->list);
269 :
270 1 : size = PAGE_ALIGN(size);
271 :
272 1 : pages = info->pages;
273 :
274 1 : vunmap(vaddr);
275 :
276 4 : for (i = 0; i < PHYS_PFN(size); i++) {
277 3 : ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
278 3 : pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
279 : ipu6_iova);
280 3 : dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
281 : DMA_BIDIRECTIONAL, attrs);
282 : }
283 :
284 1 : ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
285 : PFN_PHYS(iova_size(iova)));
286 :
287 1 : __free_buffer(pages, size, attrs);
288 :
289 1 : mmu->tlb_invalidate(mmu);
290 :
291 1 : __free_iova(&mmu->dmap->iovad, iova);
292 :
293 1 : kfree(info);
294 : }
295 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_free, INTEL_IPU6);
296 :
297 0 : int ipu6_dma_mmap(struct ipu6_bus_device *sys, struct vm_area_struct *vma,
298 : void *addr, dma_addr_t iova, size_t size,
299 : unsigned long attrs)
300 : {
301 0 : struct ipu6_mmu *mmu = sys->mmu;
302 0 : size_t count = PFN_UP(size);
303 : struct vm_info *info;
304 : size_t i;
305 : int ret;
306 :
307 0 : info = get_vm_info(mmu, iova);
308 0 : if (!info)
309 : return -EFAULT;
310 :
311 0 : if (!info->vaddr)
312 : return -EFAULT;
313 :
314 0 : if (vma->vm_start & ~PAGE_MASK)
315 : return -EINVAL;
316 :
317 0 : if (size > info->size)
318 : return -EFAULT;
319 :
320 0 : for (i = 0; i < count; i++) {
321 0 : ret = vm_insert_page(vma, vma->vm_start + PFN_PHYS(i),
322 0 : info->pages[i]);
323 0 : if (ret < 0)
324 0 : return ret;
325 : }
326 :
327 : return 0;
328 : }
329 :
330 2 : void ipu6_dma_unmap_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
331 : int nents, enum dma_data_direction dir,
332 : unsigned long attrs)
333 : {
334 2 : struct device *dev = &sys->auxdev.dev;
335 2 : struct ipu6_mmu *mmu = sys->mmu;
336 2 : struct iova *iova = find_iova(&mmu->dmap->iovad,
337 2 : PHYS_PFN(sg_dma_address(sglist)));
338 : struct scatterlist *sg;
339 2 : dma_addr_t pci_dma_addr;
340 : unsigned int i;
341 :
342 2 : if (!nents)
343 0 : return;
344 :
345 2 : if (WARN_ON(!iova))
346 0 : return;
347 :
348 : /*
349 : * Before IPU6 mmu unmap, return the pci dma address back to sg
350 : * assume the nents is less than orig_nents as the least granule
351 : * is 1 SZ_4K page
352 : */
353 2 : dev_dbg(dev, "trying to unmap concatenated %u ents\n", nents);
354 13 : for_each_sg(sglist, sg, nents, i) {
355 11 : dev_dbg(dev, "unmap sg[%d] %pad size %u\n", i,
356 : &sg_dma_address(sg), sg_dma_len(sg));
357 11 : pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
358 : sg_dma_address(sg));
359 11 : dev_dbg(dev, "return pci_dma_addr %pad back to sg[%d]\n",
360 : &pci_dma_addr, i);
361 11 : sg_dma_address(sg) = pci_dma_addr;
362 : }
363 :
364 2 : dev_dbg(dev, "ipu6_mmu_unmap low pfn %lu high pfn %lu\n",
365 : iova->pfn_lo, iova->pfn_hi);
366 2 : ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
367 : PFN_PHYS(iova_size(iova)));
368 :
369 2 : mmu->tlb_invalidate(mmu);
370 2 : __free_iova(&mmu->dmap->iovad, iova);
371 : }
372 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sg, INTEL_IPU6);
373 :
374 3 : int ipu6_dma_map_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
375 : int nents, enum dma_data_direction dir,
376 : unsigned long attrs)
377 : {
378 3 : struct device *dev = &sys->auxdev.dev;
379 3 : struct ipu6_mmu *mmu = sys->mmu;
380 : struct scatterlist *sg;
381 : struct iova *iova;
382 : size_t npages = 0;
383 : unsigned long iova_addr;
384 : int i;
385 :
386 15 : for_each_sg(sglist, sg, nents, i) {
387 12 : if (sg->offset) {
388 0 : dev_err(dev, "Unsupported non-zero sg[%d].offset %x\n",
389 : i, sg->offset);
390 0 : return -EFAULT;
391 : }
392 : }
393 :
394 15 : for_each_sg(sglist, sg, nents, i)
395 12 : npages += PFN_UP(sg_dma_len(sg));
396 :
397 3 : dev_dbg(dev, "dmamap trying to map %d ents %zu pages\n",
398 : nents, npages);
399 :
400 3 : iova = alloc_iova(&mmu->dmap->iovad, npages,
401 3 : PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
402 3 : if (!iova)
403 : return 0;
404 :
405 3 : dev_dbg(dev, "dmamap: iova low pfn %lu, high pfn %lu\n", iova->pfn_lo,
406 : iova->pfn_hi);
407 :
408 3 : iova_addr = iova->pfn_lo;
409 15 : for_each_sg(sglist, sg, nents, i) {
410 : phys_addr_t iova_pa;
411 : int ret;
412 :
413 12 : iova_pa = PFN_PHYS(iova_addr);
414 12 : dev_dbg(dev, "mapping entry %d: iova %pap phy %pap size %d\n",
415 : i, &iova_pa, &sg_dma_address(sg), sg_dma_len(sg));
416 :
417 12 : ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),
418 : sg_dma_address(sg),
419 12 : PAGE_ALIGN(sg_dma_len(sg)));
420 12 : if (ret)
421 0 : goto out_fail;
422 :
423 12 : sg_dma_address(sg) = PFN_PHYS(iova_addr);
424 :
425 12 : iova_addr += PFN_UP(sg_dma_len(sg));
426 : }
427 :
428 3 : dev_dbg(dev, "dmamap %d ents %zu pages mapped\n", nents, npages);
429 :
430 : return nents;
431 :
432 : out_fail:
433 0 : ipu6_dma_unmap_sg(sys, sglist, i, dir, attrs);
434 :
435 0 : return 0;
436 : }
437 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sg, INTEL_IPU6);
438 :
439 3 : int ipu6_dma_map_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
440 : enum dma_data_direction dir, unsigned long attrs)
441 : {
442 : int nents;
443 :
444 3 : nents = ipu6_dma_map_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
445 3 : if (nents < 0)
446 : return nents;
447 :
448 3 : sgt->nents = nents;
449 :
450 3 : return 0;
451 : }
452 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sgtable, INTEL_IPU6);
453 :
454 2 : void ipu6_dma_unmap_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
455 : enum dma_data_direction dir, unsigned long attrs)
456 : {
457 2 : ipu6_dma_unmap_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
458 2 : }
459 : EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sgtable, INTEL_IPU6);
|
