path: root/include/nvgpu/allocator.h

/*
 * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#ifndef NVGPU_ALLOCATOR_H
#define NVGPU_ALLOCATOR_H

#ifdef __KERNEL__
/*
 * The Linux kernel has this notion of seq_files for printing info to userspace.
 * One of the allocator function pointers takes advantage of this and allows the
 * debug output to be directed either to nvgpu_log() or a seq_file.
 */
#include <linux/seq_file.h>
#endif

#include <nvgpu/log.h>
#include <nvgpu/lock.h>
#include <nvgpu/list.h>
#include <nvgpu/types.h>

/* #define ALLOCATOR_DEBUG_FINE */

struct nvgpu_allocator;
struct nvgpu_alloc_carveout;
struct vm_gk20a;
struct gk20a;

/*
 * Operations for an allocator to implement.
 */
struct nvgpu_allocator_ops {
	u64  (*alloc)(struct nvgpu_allocator *allocator, u64 len);
	u64  (*alloc_pte)(struct nvgpu_allocator *allocator, u64 len,
			  u32 page_size);
	void (*free)(struct nvgpu_allocator *allocator, u64 addr);

	/*
	 * Special interface to allocate a memory region with a specific
	 * starting address. Yikes. Note: if free() works for freeing both
	 * regular and fixed allocations then free_fixed() does not need to
	 * be implemented. This behavior exists for legacy reasons and should
	 * not be propagated to new allocators.
	 *
	 * For allocators where the @page_size field is not applicable it can
	 * be left as 0. Otherwise a valid page size should be passed (4k or
	 * what the large page size is).
	 */
	u64  (*alloc_fixed)(struct nvgpu_allocator *allocator,
			    u64 base, u64 len, u32 page_size);
	void (*free_fixed)(struct nvgpu_allocator *allocator,
			   u64 base, u64 len);

	/*
	 * Allow allocators to reserve space for carveouts.
	 */
	int  (*reserve_carveout)(struct nvgpu_allocator *allocator,
				 struct nvgpu_alloc_carveout *co);
	void (*release_carveout)(struct nvgpu_allocator *allocator,
				 struct nvgpu_alloc_carveout *co);

	/*
	 * Returns info about the allocator.
	 */
	u64  (*base)(struct nvgpu_allocator *allocator);
	u64  (*length)(struct nvgpu_allocator *allocator);
	u64  (*end)(struct nvgpu_allocator *allocator);
	bool (*inited)(struct nvgpu_allocator *allocator);
	u64  (*space)(struct nvgpu_allocator *allocator);

	/* Destructor. */
	void (*fini)(struct nvgpu_allocator *allocator);

#ifdef __KERNEL__
	/* Debugging. */
	void (*print_stats)(struct nvgpu_allocator *allocator,
			    struct seq_file *s, int lock);
#endif
};

struct nvgpu_allocator {
	struct gk20a *g;

	char name[32];
	struct nvgpu_mutex lock;

	void *priv;
	const struct nvgpu_allocator_ops *ops;

	struct dentry *debugfs_entry;
	bool debug;				/* Control for debug msgs. */
};

struct nvgpu_alloc_carveout {
	const char *name;
	u64 base;
	u64 length;

	struct nvgpu_allocator *allocator;

	/*
	 * For usage by the allocator implementation.
	 */
	struct nvgpu_list_node co_entry;
};

static inline struct nvgpu_alloc_carveout *
nvgpu_alloc_carveout_from_co_entry(struct nvgpu_list_node *node)
{
	return (struct nvgpu_alloc_carveout *)
	((uintptr_t)node - offsetof(struct nvgpu_alloc_carveout, co_entry));
};

#define NVGPU_CARVEOUT(local_name, local_base, local_length)	\
	{							\
		.name = (local_name),				\
		.base = (local_base),				\
		.length = (local_length)			\
	}

/*
 * These are the available allocator flags.
 *
 *   GPU_ALLOC_GVA_SPACE
 *
 *     This flag makes sense for the buddy allocator only. It specifies that the
 *     allocator will be used for managing a GVA space. When managing GVA spaces
 *     special care has to be taken to ensure that allocations of similar PTE
 *     sizes are placed in the same PDE block. This allows the higher level
 *     code to skip defining both small and large PTE tables for every PDE. That
 *     can save considerable memory for address spaces that have a lot of
 *     allocations.
 *
 *   GPU_ALLOC_NO_ALLOC_PAGE
 *
 *     For any allocator that needs to manage a resource in a latency critical
 *     path this flag specifies that the allocator should not use any kmalloc()
 *     or similar functions during normal operation. Initialization routines
 *     may still use kmalloc(). This prevents the possibility of long waits for
 *     pages when using alloc_page(). Currently only the bitmap allocator
 *     implements this functionality.
 *
 *     Also note that if you accept this flag then you must also define the
 *     free_fixed() function. Since no meta-data is allocated to help free
 *     allocations you need to keep track of the meta-data yourself (in this
 *     case the base and length of the allocation as opposed to just the base
 *     of the allocation).
 *
 *   GPU_ALLOC_4K_VIDMEM_PAGES
 *
 *     We manage vidmem pages at a large page granularity for performance
 *     reasons; however, this can lead to wasting memory. For page allocators
 *     setting this flag will tell the allocator to manage pools of 4K pages
 *     inside internally allocated large pages.
 *
 *     Currently this flag is ignored since the only usage of the page allocator
 *     uses a 4K block size already. However, this flag has been reserved since
 *     it will be necessary in the future.
 *
 *   GPU_ALLOC_FORCE_CONTIG
 *
 *     Force allocations to be contiguous. Currently only relevant for page
 *     allocators since all other allocators are naturally contiguous.
 *
 *   GPU_ALLOC_NO_SCATTER_GATHER
 *
 *     The page allocator normally returns a scatter gather data structure for
 *     allocations (to handle discontiguous pages). However, at times that can
 *     be annoying so this flag forces the page allocator to return a u64
 *     pointing to the allocation base (requires GPU_ALLOC_FORCE_CONTIG to be
 *     set as well).
 */
#define GPU_ALLOC_GVA_SPACE		BIT64(0)
#define GPU_ALLOC_NO_ALLOC_PAGE		BIT64(1)
#define GPU_ALLOC_4K_VIDMEM_PAGES	BIT64(2)
#define GPU_ALLOC_FORCE_CONTIG		BIT64(3)
#define GPU_ALLOC_NO_SCATTER_GATHER	BIT64(4)

static inline void alloc_lock(struct nvgpu_allocator *a)
{
	nvgpu_mutex_acquire(&a->lock);
}

static inline void alloc_unlock(struct nvgpu_allocator *a)
{
	nvgpu_mutex_release(&a->lock);
}

/*
 * Buddy allocator specific initializers.
 */
int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
			       struct vm_gk20a *vm, const char *name,
			       u64 base, u64 size, u64 blk_size,
			       u64 max_order, u64 flags);

/*
 * Bitmap initializers.
 */
int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
				const char *name, u64 base, u64 length,
				u64 blk_size, u64 flags);

/*
 * Page allocator initializers.
 */
int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
			      const char *name, u64 base, u64 length,
			      u64 blk_size, u64 flags);

/*
 * Lockless allocatior initializers.
 * Note: This allocator can only allocate fixed-size structures of a
 * pre-defined size.
 */
int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
				  const char *name, u64 base, u64 length,
				  u64 struct_size, u64 flags);

#define GPU_BALLOC_MAX_ORDER		31U

/*
 * Allocator APIs.
 */
u64  nvgpu_alloc(struct nvgpu_allocator *allocator, u64 len);
u64  nvgpu_alloc_pte(struct nvgpu_allocator *a, u64 len, u32 page_size);
void nvgpu_free(struct nvgpu_allocator *allocator, u64 addr);

u64  nvgpu_alloc_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len,
		       u32 page_size);
void nvgpu_free_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len);

int  nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a,
				  struct nvgpu_alloc_carveout *co);
void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a,
				  struct nvgpu_alloc_carveout *co);

u64  nvgpu_alloc_base(struct nvgpu_allocator *a);
u64  nvgpu_alloc_length(struct nvgpu_allocator *a);
u64  nvgpu_alloc_end(struct nvgpu_allocator *a);
bool nvgpu_alloc_initialized(struct nvgpu_allocator *a);
u64  nvgpu_alloc_space(struct nvgpu_allocator *a);

void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator);

#ifdef __KERNEL__
void nvgpu_alloc_print_stats(struct nvgpu_allocator *a,
			     struct seq_file *s, int lock);
#endif

static inline struct gk20a *nvgpu_alloc_to_gpu(struct nvgpu_allocator *a)
{
	return a->g;
}

#ifdef CONFIG_DEBUG_FS
/*
 * Common functionality for the internals of the allocators.
 */
void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a);
void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a);
#endif

int  nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
			     const char *name, void *priv, bool dbg,
			     const struct nvgpu_allocator_ops *ops);

static inline void nvgpu_alloc_enable_dbg(struct nvgpu_allocator *a)
{
	a->debug = true;
}

static inline void nvgpu_alloc_disable_dbg(struct nvgpu_allocator *a)
{
	a->debug = false;
}

/*
 * Debug stuff.
 */
#ifdef __KERNEL__
#define __alloc_pstat(seq, allocator, fmt, arg...)		\
	do {							\
		if (seq)					\
			seq_printf(seq, fmt "\n", ##arg);	\
		else						\
			alloc_dbg(allocator, fmt, ##arg);	\
	} while (0)
#endif

#define do_alloc_dbg(a, fmt, arg...)				\
	nvgpu_log((a)->g, gpu_dbg_alloc, "%25s " fmt, (a)->name, ##arg)

/*
 * This gives finer control over debugging messages. By defining the
 * ALLOCATOR_DEBUG_FINE macro prints for an allocator will only get made if
 * that allocator's debug flag is set.
 *
 * Otherwise debugging is as normal: debug statements for all allocators
 * if the GPU debugging mask bit is set. Note: even when ALLOCATOR_DEBUG_FINE
 * is set gpu_dbg_alloc must still also be set to true.
 */
#if defined(ALLOCATOR_DEBUG_FINE)
#define alloc_dbg(a, fmt, arg...)				\
	do {							\
		if ((a)->debug)					\
			do_alloc_dbg((a), fmt, ##arg);		\
	} while (0)
#else
#define alloc_dbg(a, fmt, arg...) do_alloc_dbg(a, fmt, ##arg)
#endif

#endif /* NVGPU_ALLOCATOR_H */