From 1403b1a38e8b19a4cc17e2c158e278628943a436 Mon Sep 17 00:00:00 2001
From: Pauli Nieminen <suokkos@gmail.com>
Date: Thu, 1 Apr 2010 12:44:57 +0000
Subject: drm/ttm: add pool wc/uc page allocator V3

On AGP system we might allocate/free routinely uncached or wc memory,
changing page from cached (wb) to uc or wc is very expensive and involves
a lot of flushing. To improve performance this allocator use a pool
of uc,wc pages.

Pools are protected with spinlocks to allow multiple threads to allocate pages
simultanously. Expensive operations are done outside of spinlock to maximize
concurrency.

Pools are linked lists of pages that were recently freed. mm shrink callback
allows kernel to claim back pages when they are required for something else.

Fixes:
* set_pages_array_wb handles highmem pages so we don't have to remove them
  from pool.
* Add count parameter to ttm_put_pages to avoid looping in free code.
* Change looping from _safe to normal in pool fill error path.
* Initialize sum variable and make the loop prettier in get_num_unused_pages.

* Moved pages_freed reseting inside the loop in ttm_page_pool_free.
* Add warning comment about spinlock context in ttm_page_pool_free.

Based on Jerome Glisse's and Dave Airlie's pool allocator.

Signed-off-by: Jerome Glisse <jglisse@redhat.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
Signed-off-by: Pauli Nieminen <suokkos@gmail.com>
Reviewed-by: Jerome Glisse <jglisse@redhat.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 include/drm/ttm/ttm_page_alloc.h | 70 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 include/drm/ttm/ttm_page_alloc.h

(limited to 'include/drm')

diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
new file mode 100644
index 000000000000..043d817b8164
--- /dev/null
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) Red Hat Inc.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sub license,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Dave Airlie <airlied@redhat.com>
+ *          Jerome Glisse <jglisse@redhat.com>
+ */
+#ifndef TTM_PAGE_ALLOC
+#define TTM_PAGE_ALLOC
+
+#include "ttm_bo_driver.h"
+#include "ttm_memory.h"
+
+/**
+ * Get count number of pages from pool to pages list.
+ *
+ * @pages: heado of empty linked list where pages are filled.
+ * @flags: ttm flags for page allocation.
+ * @cstate: ttm caching state for the page.
+ * @count: number of pages to allocate.
+ */
+int ttm_get_pages(struct list_head *pages,
+		  int flags,
+		  enum ttm_caching_state cstate,
+		  unsigned count);
+/**
+ * Put linked list of pages to pool.
+ *
+ * @pages: list of pages to free.
+ * @page_count: number of pages in the list. Zero can be passed for unknown
+ * count.
+ * @flags: ttm flags for page allocation.
+ * @cstate: ttm caching state.
+ */
+void ttm_put_pages(struct list_head *pages,
+		   unsigned page_count,
+		   int flags,
+		   enum ttm_caching_state cstate);
+/**
+ * Initialize pool allocator.
+ *
+ * Pool allocator is internaly reference counted so it can be initialized
+ * multiple times but ttm_page_alloc_fini has to be called same number of
+ * times.
+ */
+int ttm_page_alloc_init(unsigned max_pages);
+/**
+ * Free pool allocator.
+ */
+void ttm_page_alloc_fini(void);
+
+#endif
-- 
cgit v1.2.3


From 0745866165598b067442c472911280527b08be3e Mon Sep 17 00:00:00 2001
From: Pauli Nieminen <suokkos@gmail.com>
Date: Thu, 1 Apr 2010 12:44:58 +0000
Subject: drm/ttm: Add debugfs output entry to pool allocator.

ttm_page_alloc_debugfs can be registered to output the state
of pools.

Debugfs file will output number of pages freed from the pool,
number of pages in pool now and the lowes number of pages in
pool since previous shrink.

Signed-off-by: Pauli Nieminen <suokkos@gmail.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/ttm/ttm_page_alloc.c | 45 ++++++++++++++++++++++++++++++------
 include/drm/ttm/ttm_page_alloc.h     |  4 ++++
 2 files changed, 42 insertions(+), 7 deletions(-)

(limited to 'include/drm')

diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index f46e40be0797..f82bf805903c 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -34,6 +34,7 @@
 #include <linux/spinlock.h>
 #include <linux/highmem.h>
 #include <linux/mm_types.h>
+#include <linux/module.h>
 #include <linux/mm.h>
 
 #include <asm/atomic.h>
@@ -66,6 +67,9 @@ struct ttm_page_pool {
 	struct list_head	list;
 	int			gfp_flags;
 	unsigned		npages;
+	char			*name;
+	unsigned long		nfrees;
+	unsigned long		nrefills;
 };
 
 struct ttm_pool_opts {
@@ -190,6 +194,7 @@ static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
 		unsigned freed_pages)
 {
 	pool->npages -= freed_pages;
+	pool->nfrees += freed_pages;
 }
 
 /**
@@ -263,7 +268,6 @@ restart:
 		}
 	}
 
-
 	/* remove range of pages from the pool */
 	if (freed_pages) {
 		__list_del(&p->lru, &pool->list);
@@ -490,6 +494,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 
 		if (!r) {
 			list_splice(&new_pages, &pool->list);
+			++pool->nrefills;
 			pool->npages += alloc_size;
 		} else {
 			printk(KERN_ERR "[ttm] Failed to fill pool (%p).", pool);
@@ -663,13 +668,15 @@ void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
 		ttm_page_pool_free(pool, page_count);
 }
 
-static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags)
+static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
+		char *name)
 {
 	spin_lock_init(&pool->lock);
 	pool->fill_lock = false;
 	INIT_LIST_HEAD(&pool->list);
-	pool->npages = 0;
+	pool->npages = pool->nfrees = 0;
 	pool->gfp_flags = flags;
+	pool->name = name;
 }
 
 int ttm_page_alloc_init(unsigned max_pages)
@@ -679,13 +686,15 @@ int ttm_page_alloc_init(unsigned max_pages)
 
 	printk(KERN_INFO "[ttm] Initializing pool allocator.\n");
 
-	ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER);
+	ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER, "wc");
 
-	ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER);
+	ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER, "uc");
 
-	ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32);
+	ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32,
+			"wc dma");
 
-	ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32);
+	ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32,
+			"uc dma");
 
 	_manager.options.max_size = max_pages;
 	_manager.options.small = SMALL_ALLOCATION;
@@ -709,3 +718,25 @@ void ttm_page_alloc_fini()
 	for (i = 0; i < NUM_POOLS; ++i)
 		ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES);
 }
+
+int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
+{
+	struct ttm_page_pool *p;
+	unsigned i;
+	char *h[] = {"pool", "refills", "pages freed", "size"};
+	if (atomic_read(&_manager.page_alloc_inited) == 0) {
+		seq_printf(m, "No pool allocator running.\n");
+		return 0;
+	}
+	seq_printf(m, "%6s %12s %13s %8s\n",
+			h[0], h[1], h[2], h[3]);
+	for (i = 0; i < NUM_POOLS; ++i) {
+		p = &_manager.pools[i];
+
+		seq_printf(m, "%6s %12ld %13ld %8d\n",
+				p->name, p->nrefills,
+				p->nfrees, p->npages);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(ttm_page_alloc_debugfs);
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 043d817b8164..8b091c309df4 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -67,4 +67,8 @@ int ttm_page_alloc_init(unsigned max_pages);
  */
 void ttm_page_alloc_fini(void);
 
+/**
+ * Output the state of pools to debugfs file
+ */
+extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data);
 #endif
-- 
cgit v1.2.3


From c96af79e3463d5d3f865625baa8bb8aa4c0944a0 Mon Sep 17 00:00:00 2001
From: Pauli Nieminen <suokkos@gmail.com>
Date: Thu, 1 Apr 2010 12:45:03 +0000
Subject: drm/ttm: Add sysfs interface to control pool allocator.

Sysfs interface allows user to configure pool allocator functionality and
change limits for the size of pool.

Signed-off-by: Pauli Nieminen <suokkos@gmail.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
---
 drivers/gpu/drm/ttm/ttm_memory.c     |   2 +-
 drivers/gpu/drm/ttm/ttm_page_alloc.c | 113 ++++++++++++++++++++++++++++++++++-
 include/drm/ttm/ttm_page_alloc.h     |   2 +-
 3 files changed, 114 insertions(+), 3 deletions(-)

(limited to 'include/drm')

diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index daff8a87977e..5e3f177323cb 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -393,7 +393,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob)
 		       "Zone %7s: Available graphics memory: %llu kiB.\n",
 		       zone->name, (unsigned long long) zone->max_mem >> 10);
 	}
-	ttm_page_alloc_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE));
+	ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));
 	return 0;
 out_no_zone:
 	ttm_mem_global_release(glob);
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 57799dba35e0..6ca9b27e33d5 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -72,6 +72,12 @@ struct ttm_page_pool {
 	unsigned long		nrefills;
 };
 
+/**
+ * Limits for the pool. They are handled without locks because only place where
+ * they may change is in sysfs store. They won't have immediate effect anyway
+ * so forcing serialiazation to access them is pointless.
+ */
+
 struct ttm_pool_opts {
 	unsigned	alloc_size;
 	unsigned	max_size;
@@ -94,6 +100,7 @@ struct ttm_pool_opts {
  * @pools: All pool objects in use.
  **/
 struct ttm_pool_manager {
+	struct kobject		kobj;
 	struct shrinker		mm_shrink;
 	atomic_t		page_alloc_inited;
 	struct ttm_pool_opts	options;
@@ -109,6 +116,100 @@ struct ttm_pool_manager {
 	};
 };
 
+static struct attribute ttm_page_pool_max = {
+	.name = "pool_max_size",
+	.mode = S_IRUGO | S_IWUSR
+};
+static struct attribute ttm_page_pool_small = {
+	.name = "pool_small_allocation",
+	.mode = S_IRUGO | S_IWUSR
+};
+static struct attribute ttm_page_pool_alloc_size = {
+	.name = "pool_allocation_size",
+	.mode = S_IRUGO | S_IWUSR
+};
+
+static struct attribute *ttm_pool_attrs[] = {
+	&ttm_page_pool_max,
+	&ttm_page_pool_small,
+	&ttm_page_pool_alloc_size,
+	NULL
+};
+
+static void ttm_pool_kobj_release(struct kobject *kobj)
+{
+	struct ttm_pool_manager *m =
+		container_of(kobj, struct ttm_pool_manager, kobj);
+	(void)m;
+}
+
+static ssize_t ttm_pool_store(struct kobject *kobj,
+		struct attribute *attr, const char *buffer, size_t size)
+{
+	struct ttm_pool_manager *m =
+		container_of(kobj, struct ttm_pool_manager, kobj);
+	int chars;
+	unsigned val;
+	chars = sscanf(buffer, "%u", &val);
+	if (chars == 0)
+		return size;
+
+	/* Convert kb to number of pages */
+	val = val / (PAGE_SIZE >> 10);
+
+	if (attr == &ttm_page_pool_max)
+		m->options.max_size = val;
+	else if (attr == &ttm_page_pool_small)
+		m->options.small = val;
+	else if (attr == &ttm_page_pool_alloc_size) {
+		if (val > NUM_PAGES_TO_ALLOC*8) {
+			printk(KERN_ERR "[ttm] Setting allocation size to %lu "
+					"is not allowed. Recomended size is "
+					"%lu\n",
+					NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
+					NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
+			return size;
+		} else if (val > NUM_PAGES_TO_ALLOC) {
+			printk(KERN_WARNING "[ttm] Setting allocation size to "
+					"larger than %lu is not recomended.\n",
+					NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
+		}
+		m->options.alloc_size = val;
+	}
+
+	return size;
+}
+
+static ssize_t ttm_pool_show(struct kobject *kobj,
+		struct attribute *attr, char *buffer)
+{
+	struct ttm_pool_manager *m =
+		container_of(kobj, struct ttm_pool_manager, kobj);
+	unsigned val = 0;
+
+	if (attr == &ttm_page_pool_max)
+		val = m->options.max_size;
+	else if (attr == &ttm_page_pool_small)
+		val = m->options.small;
+	else if (attr == &ttm_page_pool_alloc_size)
+		val = m->options.alloc_size;
+
+	val = val * (PAGE_SIZE >> 10);
+
+	return snprintf(buffer, PAGE_SIZE, "%u\n", val);
+}
+
+static const struct sysfs_ops ttm_pool_sysfs_ops = {
+	.show = &ttm_pool_show,
+	.store = &ttm_pool_store,
+};
+
+static struct kobj_type ttm_pool_kobj_type = {
+	.release = &ttm_pool_kobj_release,
+	.sysfs_ops = &ttm_pool_sysfs_ops,
+	.default_attrs = ttm_pool_attrs,
+};
+
 static struct ttm_pool_manager _manager = {
 	.page_alloc_inited	= ATOMIC_INIT(0)
 };
@@ -669,8 +770,9 @@ static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
 	pool->name = name;
 }
 
-int ttm_page_alloc_init(unsigned max_pages)
+int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 {
+	int ret;
 	if (atomic_add_return(1, &_manager.page_alloc_inited) > 1)
 		return 0;
 
@@ -690,6 +792,13 @@ int ttm_page_alloc_init(unsigned max_pages)
 	_manager.options.small = SMALL_ALLOCATION;
 	_manager.options.alloc_size = NUM_PAGES_TO_ALLOC;
 
+	kobject_init(&_manager.kobj, &ttm_pool_kobj_type);
+	ret = kobject_add(&_manager.kobj, &glob->kobj, "pool");
+	if (unlikely(ret != 0)) {
+		kobject_put(&_manager.kobj);
+		return ret;
+	}
+
 	ttm_pool_mm_shrink_init(&_manager);
 
 	return 0;
@@ -707,6 +816,8 @@ void ttm_page_alloc_fini()
 
 	for (i = 0; i < NUM_POOLS; ++i)
 		ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES);
+
+	kobject_put(&_manager.kobj);
 }
 
 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 8b091c309df4..8bb4de567b2c 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -61,7 +61,7 @@ void ttm_put_pages(struct list_head *pages,
  * multiple times but ttm_page_alloc_fini has to be called same number of
  * times.
  */
-int ttm_page_alloc_init(unsigned max_pages);
+int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages);
 /**
  * Free pool allocator.
  */
-- 
cgit v1.2.3