1. 概述

在linux啟動過程中會打印出如下信息，這些信息為我們呈現(xiàn)出系統(tǒng)下的保留內(nèi)存空間情況。

Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB
OF: reserved mem: initialized node vram@4c000000, compatible id shared-dma-pool

本文只介紹基本的保留內(nèi)存，不涉及CMA部分內(nèi)容

保留內(nèi)存的初始化流程如下圖所示：

本文所說的保留內(nèi)存的作用，概況來講包括如下四方面：

1. 驅(qū)動程序特定使用
2. 加載固件到指定內(nèi)存
3. DDR中某段內(nèi)存區(qū)域存放特定數(shù)據(jù)，如多核處理相關(guān)代碼
4. 調(diào)試驅(qū)動

2. 保留內(nèi)存初始化流程

從setup_arch()函數(shù)中我們可以發(fā)現(xiàn)，保留內(nèi)存初始化是在設(shè)備樹釋放之前，通過解析FDT，獲取保留內(nèi)存的參數(shù)來進行初始化。

void __init setup_arch(char **cmdline_p)
{
...
 arm_memblock_init(mdesc);
...
 unflatten_device_tree();
...

2.1 解析內(nèi)核中的保留內(nèi)存空間

在各平臺初始化過程中調(diào)用early_init_fdt_scan_reserved_mem()進行保留內(nèi)存的初始化。

setup_arch_memory in init.c (arch\\arc\\mm) :  early_init_fdt_scan_reserved_mem();
arm64_memblock_init in init.c (arch\\arm64\\mm) :  early_init_fdt_scan_reserved_mem();
arm_memblock_init in init.c (arch\\arm\\mm) :  early_init_fdt_scan_reserved_mem();
setup_bootmem in init.c (arch\\riscv\\mm) :  early_init_fdt_scan_reserved_mem();
bootmem_init in init.c (arch\\xtensa\\mm) :  early_init_fdt_scan_reserved_mem();
sh_of_mem_reserve in of-generic.c (arch\\sh\\boards) :  early_init_fdt_scan_reserved_mem();
of_fdt.h (include\\linux) line 66 : extern void early_init_fdt_scan_reserved_mem(void);
of_fdt.h (include\\linux) line 94 : static inline void early_init_fdt_scan_reserved_mem(void) {}
early_reserve_mem_dt in prom.c (arch\\powerpc\\kernel) :  early_init_fdt_scan_reserved_mem();
csky_memblock_init in setup.c (arch\\csky\\kernel) :  early_init_fdt_scan_reserved_mem();
bootmem_init in setup.c (arch\\h8300\\kernel) :  early_init_fdt_scan_reserved_mem();
arch_mem_init in setup.c (arch\\mips\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\nds32\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_arch in setup.c (arch\\nios2\\kernel) :  early_init_fdt_scan_reserved_mem();
setup_memory in setup.c (arch\\openrisc\\kernel) :  early_init_fdt_scan_reserved_mem();

它的定義位于drivers/of/fdt.c中，需要內(nèi)核配置打開CONFIG_OF_EARLY_FLATTREE宏。

主體函數(shù)如下：

void __init early_init_fdt_scan_reserved_mem(void)
{
 int n;
 u64 base, size;

 if (!initial_boot_params)
  return;

 for (n = 0; ; n++) {
  fdt_get_mem_rsv(initial_boot_params, n, &base, &size);
  if (!size)
   break;
  early_init_dt_reserve_memory_arch(base, size, false);
 }

 of_scan_flat_dt(__fdt_scan_reserved_mem, NULL);
 fdt_init_reserved_mem();
}

2.1.1 解析memreserve

從early_init_fdt_scan_reserved_mem()函數(shù)中的initial_boot_params可以再次確定這一點。initial_boot_params代表的是fdt的地址，如下：

## Flattened Device Tree blob at 41000000
   Booting using the fdt blob at 0x41000000
   Loading Kernel Image ... OK
   Loading Device Tree to 4ffef000, end 4ffffff2 ... OK

Starting kernel ...

Uncompressing Linux... done, booting the kernel.
[    0.000000] Booting Linux on physical CPU 0xa00
...
[    0.000000] --- initial_boot_params(fdt addr 0x4ffef000)

通過fdt_get_mem_rsv()解析設(shè)備樹中的/memreserve/fields，例如樹莓派處理器的設(shè)備樹中定義了該屬性，通常來講，這部分內(nèi)存區(qū)域是存放和rom或者多核啟動相關(guān)的程序，需要注意的是內(nèi)核無法使用這部分內(nèi)存。這是和reserver memory的區(qū)別。

/memreserve/ 0x00000000 0x00001000;
...
/ {
        compatible = "brcm,bcm2835";
...

若在設(shè)備樹中查找到了memreserve且未進行映射，則通過memblock_reserve()將這部分內(nèi)存區(qū)域加入到memblock.reserved，當進行memblock到buddy轉(zhuǎn)換時，釋放掉memblock.reserved所標記的內(nèi)存區(qū)域。

int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
{
 phys_addr_t end = base + size - 1;

 memblock_dbg("memblock_reserve: [%pa-%pa] %pS\\n",
       &base, &end, (void *)_RET_IP_);

 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
}

2.1.2 解析reserve memory

通過__fdt_scan_reserved_mem()解析設(shè)備樹中保留內(nèi)存相關(guān)的結(jié)點信息。

static int __init __fdt_scan_reserved_mem(unsigned long node, const char *uname,
       int depth, void *data)
{
 static int found;
 int err;

 if (!found && depth == 1 && strcmp(uname, "reserved-memory") == 0) {
  ...
 }

 if (!of_fdt_device_is_available(initial_boot_params, node))
  return 0;

 err = __reserved_mem_reserve_reg(node, uname);
 if (err == -ENOENT && of_get_flat_dt_prop(node, "size", NULL))
  fdt_reserved_mem_save_node(node, uname, 0, 0);
...
}

該函數(shù)首先解析設(shè)備樹中reserved-memory結(jié)點并確認是否有效。若有效，繼續(xù)檢查reg或size屬性定義的內(nèi)存區(qū)域，通過fdt_reserved_mem_save_node()將內(nèi)存信息更新到數(shù)據(jù)結(jié)構(gòu)struct reserved_mem。

void __init fdt_reserved_mem_save_node(unsigned long node, const char *uname,phys_addr_t base, phys_addr_t size)
{
 struct reserved_mem *rmem = &reserved_mem[reserved_mem_count];

 if (reserved_mem_count == ARRAY_SIZE(reserved_mem)) {
  pr_err("not enough space all defined regions.\\n");
  return;
 }

 rmem- >fdt_node = node;
 rmem- >name = uname;
 rmem- >base = base;
 rmem- >size = size;

 reserved_mem_count++;
 return;
}

2.2 保留內(nèi)存初始化

保留內(nèi)存初始化的主體函數(shù)是fdt_init_reserved_mem()，其首先解析設(shè)備樹結(jié)點no-map、phandle等信息，最后通過關(guān)鍵函數(shù)__reserved_mem_init_node()完成保留內(nèi)存子節(jié)點的初始化。

放開解析保留內(nèi)存解析相關(guān)的打印，再回頭再看kernel的啟動信息，啟動信息中保留內(nèi)存相關(guān)內(nèi)容正是此處打印出來的。

OF: fdt: Reserved memory: reserved region for node 'vram@4c000000': base 0x4c000000, size 8 MiB
Reserved memory: created DMA memory pool at 0x4c000000, size 8 MiB

created DMA memory pool...來自函數(shù)rmem_dma_setup()，這個函數(shù)從數(shù)據(jù)結(jié)構(gòu)struct reserved_mem獲取保留內(nèi)存的信息。

static int __init rmem_dma_setup(struct reserved_mem *rmem)
{
 unsigned long node = rmem- >fdt_node;

 if (of_get_flat_dt_prop(node, "reusable", NULL))
  return -EINVAL;

#ifdef CONFIG_ARM
 if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
  pr_err("Reserved memory: regions without no-map are not yet supported\\n");
  return -EINVAL;
 }

 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
  WARN(dma_reserved_default_memory,
       "Reserved memory: region for default DMA coherent area is redefined\\n");
  dma_reserved_default_memory = rmem;
 }
#endif

 rmem- >ops = &rmem_dma_ops;
 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\\n",
  &rmem- >base, (unsigned long)rmem- >size / SZ_1M);
 return 0;
}

在函數(shù)rmem_dma_setup()中還會例化reserved_mem.ops，如下：

static const struct reserved_mem_ops rmem_dma_ops = {
 .device_init = rmem_dma_device_init,
 .device_release = rmem_dma_device_release,
};

3. 設(shè)備樹中保留內(nèi)存的定義方式

以vexpress-v2p-ca9.dts中保留內(nèi)存的定義方式為例，說明dts文件中如何定義保留內(nèi)存。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* Chipselect 3 is physically at 0x4c000000 */
        vram: vram@4c000000 {
                /* 8 MB of designated video RAM */
                compatible = "shared-dma-pool";
                reg = < 0x4c000000 0x00800000 >;
                no-map;
        };
};

保留內(nèi)存由根節(jié)點和1個或多個子結(jié)點組成。

根節(jié)點包括如下信息：

• #address-cells、#size-cells

  必須項，需要同dts根節(jié)點中相關(guān)屬性保持一致。

/dts-v1/;
#include "vexpress-v2m.dtsi"

/ {
        model = "V2P-CA9";
        arm,hbi = < 0x191 >;
        arm,vexpress,site = < 0xf >;
        compatible = "arm,vexpress,v2p-ca9", "arm,vexpress";
        interrupt-parent = < &gic >;
        #address-cells = < 1 >;
        #size-cells = < 1 >;
...

• ranges

  必須項，且定義為空

子結(jié)點包括如下信息：

• 空間大小

  可以通過reg或size來指定保留內(nèi)存空間大小，若二者同時存在，以reg屬性為準。通過size的方式如下：

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        mfc_left: region_mfc_left {
                compatible = "shared-dma-pool";
                no-map;
                size = < 0x2400000 >;
...

• alignment

  可選項

• alloc-ranges

  可選項，通?？梢院蛃ize同時使用。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
...

• compatible

  可能包括shared-dma-pool或者shared-dma-pool。

  主要關(guān)注shared-dma-pool，當驅(qū)動程序需要申請DMA空間時，可以從這里進行申請內(nèi)存空間。

• no-map

  該屬性意為不會為這段內(nèi)存創(chuàng)建地址映射，在使用之前，需要調(diào)用者通過ioremap創(chuàng)建頁表映射關(guān)系才可以正常訪問。這個屬性與reusable是互斥的。

• no-map-fixup

  保持內(nèi)存映射。

• reusable

  當驅(qū)動程序不使用這些內(nèi)存的時候，OS可以使用這些內(nèi)存。

• linux,cma-default

  定義該段保留內(nèi)存空間是默認的CMA內(nèi)存池。

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;
        default-pool {
                compatible = "shared-dma-pool";
                size = < 0x6000000 >;
                alloc-ranges = < 0x40000000 0x10000000 >;
                reusable;
                linux,cma-default;
        };
};

4. 保留內(nèi)存的使用

4.1 設(shè)備樹編碼

定義保留內(nèi)存：

memory@60000000 {
        device_type = "memory";
        reg = < 0x60000000 0x40000000 >;
};

reserved-memory {
        #address-cells = < 1 >;
        #size-cells = < 1 >;
        ranges;

        /* test reserve memory */
        test_reserve: test_reserve@90000000 {
                /* 1 MB reserve memory */
                compatible = "shared-dma-pool";
                reg = < 0x90000000 0x00100000 >;
                no-map;
        };
};

定義memory-region，將保留內(nèi)存指定給特定設(shè)備，如下：

driver-test@8000 {
        /* compatible = "test_driver_0", "simple_bus"; */
        compatible = "test_driver_0";
        reg = < 0x80008000 0x1000 >;

        interrupt-parent = < &gic >;
        interrupts= < 0 89 4 >, < 0 90 4444 >;
        interrupt-names = "first_irq", "second_irq";

        clocks = < &oscclk2 >;
        clock-names = "apb_pclk";

        memory-region = < &test_reserve >;

        status = "okay";

        simple_bus_test{
            compatile = "simple_bus_test";
        };
};

加載kernel后，保留內(nèi)存相關(guān)的打印信息如下：

Reserved memory: created DMA memory pool at 0x90000000, size 1 MiB
OF: reserved mem: initialized node test_reserve@90000000, compatible id shared-dma-pool

4.2 驅(qū)動程序編碼

驅(qū)動程序中調(diào)用of_reserved_mem_device_init()申請保留內(nèi)存空間。

static inline int of_reserved_mem_device_init(struct device *dev)
{
 return of_reserved_mem_device_init_by_idx(dev, dev- >of_node, 0);
}

主體函數(shù)是of_reserved_mem_device_init_by_idx()

int of_reserved_mem_device_init_by_idx(struct device *dev,
           struct device_node *np, int idx)
{
 struct rmem_assigned_device *rd;
 struct device_node *target;
 struct reserved_mem *rmem;
 int ret;

 if (!np || !dev)
  return -EINVAL;

 target = of_parse_phandle(np, "memory-region", idx);
 if (!target)
  return -ENODEV;

 if (!of_device_is_available(target)) {
  of_node_put(target);
  return 0;
 }

 rmem = __find_rmem(target);
 of_node_put(target);

 if (!rmem || !rmem- >ops || !rmem- >ops- >device_init)
  return -EINVAL;

 rd = kmalloc(sizeof(struct rmem_assigned_device), GFP_KERNEL);
 if (!rd)
  return -ENOMEM;

 ret = rmem- >ops- >device_init(rmem, dev);
 if (ret == 0) {
  rd- >dev = dev;
  rd- >rmem = rmem;

  mutex_lock(&of_rmem_assigned_device_mutex);
  list_add(&rd- >list, &of_rmem_assigned_device_list);
  mutex_unlock(&of_rmem_assigned_device_mutex);

  dev_info(dev, "assigned reserved memory node %s\\n", rmem- >name);
 } else {
  kfree(rd);
 }

 return ret;
}

然后可以通過dma_alloc_coherent()在保留內(nèi)存空間申請DMA空間。

void *dma_vaddr;
dma_addr_t dma_handler;

/* Start: test reserve memory */
ret = of_reserved_mem_device_init(&pdev- >dev);
if (!ret) {
        dev_info(&pdev- >dev, "using device-specific reserved memory\\n");
}

dma_set_coherent_mask(&pdev- >dev, 0xFFFFFFFF);
dma_vaddr = dma_alloc_coherent(&pdev- >dev, 64*1024, &dma_handler, GFP_KERNEL);
if (!dma_vaddr) {
        pr_notice("DMA allocation failed\\n");
        return false;
}
dev_info(&pdev- >dev, "DMA alloc phy addr 0x%X\\n", (u32)dma_handler);
/* End: test reserve memory */

執(zhí)行結(jié)果：

test_driver ahb:driver-test@8000: assigned reserved memory node test_reserve@90000000
test_driver ahb:driver-test@8000: using device-specific reserved memory
test_driver ahb:driver-test@8000: DMA alloc phy addr 0x90000000

結(jié)果表明，DMA申請的內(nèi)存落于保留內(nèi)存空間0x90000000-0x90100000。