1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
        unsigned long flags)
{
    int err;
    s->name = name;
    s->size = s->object_size = size;
    s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
    err = __kmem_cache_create(s, flags);

    if (err)
        panic("Creation of kmalloc slab %s size=%zu failed. Reason %d\n",
                    name, size, err);

    s->refcount = -1;   /* Exempt from merging for now */
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

static int init_kmem_cache_nodes(struct kmem_cache *s)
{
    int node;
    /* 遍历每个具有normal内存的内存节点 */
    for_each_node_state(node, N_NORMAL_MEMORY) {
        struct kmem_cache_node *n;
        /*
         * 此时 kmem_cache_node 和 kmem_cache 两个 slab
         * cache 还没有建立，不能使用 
         */
        if (slab_state == DOWN) {
            early_kmem_cache_node_alloc(node);
            continue;
        }
        /*
         * kmem_cache_node 已经建立，直接从中分配一个
         * struct kmem_cache_node对象
         */
        n = kmem_cache_alloc_node(kmem_cache_node,
                        GFP_KERNEL, node);

        if (!n) {
            free_kmem_cache_nodes(s);
            return 0;
        }

        /* 设置 kmem_cache 的 per-cpu 变量指向正确的节点 */
        s->node[node] = n;

        /* 初始化 kmem_cache_node 的成员变量 */
        init_kmem_cache_node(n);
    }
    return 1;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

static void early_kmem_cache_node_alloc(int node)
{
    struct page *page;
    struct kmem_cache_node *n;

    BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));

    /*
     * 调用伙伴系统的 __alloc_pages_nodemask 函数分配
     * 新的 page 给 kmem_cache_node ，并设置 page 的成员变量 
     */
    page = new_slab(kmem_cache_node, GFP_NOWAIT, node);

    BUG_ON(!page);
    if (page_to_nid(page) != node) {
        pr_err("SLUB: Unable to allocate memory from node %d\n", node);
        pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
    }

    /* 对于新分配的 page ， page->freelist 为页面的起始地址 */
    n = page->freelist;
    BUG_ON(!n);

    /* page->freelist 指向 page 的起始地址，即第一个可用的 free object */
    page->freelist = get_freepointer(kmem_cache_node, n);
    page->inuse = 1;
    page->frozen = 0;

    /* 保存到 kmem_cache_node 中对应的节点中 */
    kmem_cache_node->node[node] = n;
#ifdef CONFIG_SLUB_DEBUG
    /* 根据 kmem_cache_node 的标志设置对象 */
    init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
    init_tracking(kmem_cache_node, n);
#endif

    /* 初始化 cache node 中 partial list，nr_slabs，total_objects，full list */
    init_kmem_cache_node(n);

    /* 增加 cache node 的统计信息，包括 nr_slabs，total_objects */
    inc_slabs_node(kmem_cache_node, node, page->objects);

    /*
     * No locks need to be taken here as it has just been
     * initialized and there is no concurrent access.
     */

     /* 将 page 添加到 node 的 partial list 中 */
    __add_partial(n, page, DEACTIVATE_TO_HEAD);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
    struct page *page;
    void *start;
    void *last;
    void *p;
    int order;

    BUG_ON(flags & GFP_SLAB_BUG_MASK);

    /* 
     * allocate_slab 最终调用 alloc_pages 函数分配新的 page ，
     * 并且设置 page->objects = s->oo.x 。
     * 函数根据传入的 kmem_cache->oo 的大小分配指定数量的 page 。
     */
    page = allocate_slab(s,
        flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
    if (!page)
        goto out;
    /* 如果是复合页面，返回复合页面包含的 page 数；否则返回 0 */
    order = compound_order(page);
    /* 
     * 将新的 page 包含的对象数添加到 slab cache 中对应的节点中，
     * 并增加 slab 的计数 ( 每个 page 计为一个 slab ) 
     */
    inc_slabs_node(s, page_to_nid(page), page->objects);

    /* 设置 page 所属的 slab cache */
    page->slab_cache = s;

    /* 设置 page 的 PG_slab 标志 */
    __SetPageSlab(page);
    /* pfmemalloc标志 */
    if (page->pfmemalloc)
        SetPageSlabPfmemalloc(page);

    /* start 设置为 page 的内核虚拟地址 */
    start = page_address(page);
    /* 设置 slab cache 为特定值 */
    if (unlikely(s->flags & SLAB_POISON))
        memset(start, POISON_INUSE, PAGE_SIZE << order);

    last = start;

    /*
     * 设置 slab cache 中的每个对象指向下一个对象，如果
     * slab cache 定义了构造函数，用构造函数初始化对象
     */
    for_each_object(p, s, start, page->objects) {
        setup_object(s, page, last);
        set_freepointer(s, last, p);
        last = p;
    }
    setup_object(s, page, last);

    /* 最后一个 object 指向NULL */
    set_freepointer(s, last, NULL);

    /* 设置 page->freelist 为内存页的起始虚拟地址 */
    page->freelist = start;

    /* page->inuse 等于页面中包含的对象的数量 */
    page->inuse = page->objects;
    page->frozen = 1;
out:
    return page;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
    void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);

    trace_kmem_cache_alloc_node(_RET_IP_, ret,
                    s->object_size, s->size, gfpflags, node);

    return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

1
2
3
4
5
6
7
8
9

struct kmem_cache_cpu {
    void **freelist;    /* Pointer to next available object */
    unsigned long tid;  /* Globally unique transaction id */
    struct page *page;  /* The slab from which we are allocating */
    struct page *partial;   /* Partially allocated frozen slabs */
#ifdef CONFIG_SLUB_STATS
    unsigned stat[NR_SLUB_STAT_ITEMS];
#endif
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
    BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
            KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));

    /*
     * Must align to double word boundary for the double cmpxchg
     * instructions to work; see __pcpu_double_call_return_bool().
     */
    s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
                     2 * sizeof(void *));

    if (!s->cpu_slab)
        return 0;
    /* 初始化cpu_slab的tid为CPU ID */
    init_kmem_cache_cpus(s);

    return 1;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

void __init create_kmalloc_caches(unsigned long flags)
{
    int i;
    ...
    for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
        if (!kmalloc_caches[i]) {

            /* create_alloc_cache函数完成实际工作 */
            kmalloc_caches[i] = create_kmalloc_cache(NULL,
                            1 << i, flags);
        }

        /*
         * Caches that are not of the two-to-the-power-of size.
         * These have to be created immediately after the
         * earlier power of two caches
         */
        if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
            kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);

        if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
            kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
    }

    /* 现在，kmalloc_caches[]的大小为 -，96，192，8，16 ... 2^13 */
    /* Kmalloc array is now usable */
    slab_state = UP;
    /* 设置slab的名称 */
    for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
        struct kmem_cache *s = kmalloc_caches[i];
        char *n;

        if (s) {
            n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));

            BUG_ON(!n);
            s->name = n;
        }
    }
    #ifdef CONFIG_ZONE_DMA
    for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
        struct kmem_cache *s = kmalloc_caches[i];

        if (s) {
            int size = kmalloc_size(i);
            char *n = kasprintf(GFP_NOWAIT,
                 "dma-kmalloc-%d", size);

            BUG_ON(!n);
            kmalloc_dma_caches[i] = create_kmalloc_cache(n,
                size, SLAB_CACHE_DMA | flags);
        }
    }
#endif
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
                unsigned long flags)
{
    struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);

    if (!s)
        panic("Out of memory when creating slab %s\n", name);

    create_boot_cache(s, name, size, flags);

    /* slab_cache 是保存系统中所有 slab cache 的链表 */
    list_add(&s->list, &slab_caches);
    s->refcount = 1;
    return s;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

/*
 * kmem_cache_create - Create a cache.
 * @name: A string which is used in /proc/slabinfo to identify this cache.
 * @size: The size of objects to be created in this cache.
 * @align: The required alignment for the objects.
 * @flags: SLAB flags
 * @ctor: A constructor for the objects.
 *
 * Returns a ptr to the cache on success, NULL on failure.
 * Cannot be called within a interrupt, but can be interrupted.
 * The @ctor is run when new pages are allocated by the cache.
 *
 * The flags are
 *
 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
 * to catch references to uninitialised memory.
 *
 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
 * for buffer overruns.
 *
 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
 * cacheline.  This can be beneficial if you're counting cycles as closely
 * as davem.
 */
struct kmem_cache *
kmem_cache_create(const char *name, size_t size, size_t align,
          unsigned long flags, void (*ctor)(void *))
{
    struct kmem_cache *s;
    char *cache_name;
    int err;

    get_online_cpus();
    get_online_mems();

    mutex_lock(&slab_mutex);

    err = kmem_cache_sanity_check(name, size);
    if (err)
        goto out_unlock;
     /*
     * Some allocators will constraint the set of valid flags to a subset
     * of all flags. We expect them to define CACHE_CREATE_MASK in this
     * case, and we'll just provide them with a sanitized version of the
     * passed flags.
     */
    flags &= CACHE_CREATE_MASK;

    /*
     * 寻找系统中是否有可以复用的 slab cache 。如果有，增加匹配的
     * slab 的引用计数，设置 slab 的参数；如果没有，则调用
     * do_kmem_cache_create 创建一个 slab cache 
     */
    s = __kmem_cache_alias(name, size, align, flags, ctor);
    if (s)
        goto out_unlock;

    cache_name = kstrdup(name, GFP_KERNEL);
    if (!cache_name) {
        err = -ENOMEM;
        goto out_unlock;
    }

    /* 
     * 和 create_kmalloc_cache 类似，先调用 kmem_cache_zalloc
     * 分配一个 kmem_cache 对象，然后调用 __kmem_cache_create
     * 执行 kmem_cache 的初始化 
     */
    s = do_kmem_cache_create(cache_name, size, size,
                 calculate_alignment(flags, align, size),
                 flags, ctor, NULL, NULL);
    if (IS_ERR(s)) {
        err = PTR_ERR(s);
        kfree(cache_name);
    }

out_unlock:
    mutex_unlock(&slab_mutex);

    put_online_mems();
    put_online_cpus();

    if (err) {
        if (flags & SLAB_PANIC)
            panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
                name, err);
        else {
            printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
                name, err);
            dump_stack();
        }
        return NULL;
    }
    return s;
}
EXPORT_SYMBOL(kmem_cache_create);