bcachefs: Make io timers less buggy

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

4 files changed, 46 insertions, 16 deletions

diff --git a/fs/bcachefs/clock.c b/fs/bcachefs/clock.c
index e227753563ab..51286520c5c7 100644
--- a/fs/bcachefs/clock.c
+++ b/fs/bcachefs/clock.c
@@ -18,6 +18,14 @@ void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
 	size_t i;
 
 	spin_lock(&clock->timer_lock);
+
+	if (time_after_eq((unsigned long) atomic_long_read(&clock->now),
+			  timer->expire)) {
+		spin_unlock(&clock->timer_lock);
+		timer->fn(timer);
+		return;
+	}
+
 	for (i = 0; i < clock->timers.used; i++)
 		if (clock->timers.data[i] == timer)
 			goto out;
@@ -135,26 +143,31 @@ static struct io_timer *get_expired_timer(struct io_clock *clock,
 	return ret;
 }
 
-void __bch2_increment_clock(struct io_clock *clock)
+void __bch2_increment_clock(struct io_clock *clock, unsigned sectors)
 {
 	struct io_timer *timer;
-	unsigned long now;
-	unsigned sectors;
+	unsigned long now = atomic_long_add_return(sectors, &clock->now);
 
-	/* Buffer up one megabyte worth of IO in the percpu counter */
-	preempt_disable();
+	while ((timer = get_expired_timer(clock, now)))
+		timer->fn(timer);
+}
 
-	if (this_cpu_read(*clock->pcpu_buf) < IO_CLOCK_PCPU_SECTORS) {
-		preempt_enable();
-		return;
-	}
+ssize_t bch2_io_timers_show(struct io_clock *clock, char *buf)
+{
+	struct printbuf out = _PBUF(buf, PAGE_SIZE);
+	unsigned long now;
+	unsigned i;
 
-	sectors = this_cpu_xchg(*clock->pcpu_buf, 0);
-	preempt_enable();
-	now = atomic_long_add_return(sectors, &clock->now);
+	spin_lock(&clock->timer_lock);
+	now = atomic_long_read(&clock->now);
 
-	while ((timer = get_expired_timer(clock, now)))
-		timer->fn(timer);
+	for (i = 0; i < clock->timers.used; i++)
+		pr_buf(&out, "%pf:\t%li\n",
+		       clock->timers.data[i]->fn,
+		       clock->timers.data[i]->expire - now);
+	spin_unlock(&clock->timer_lock);
+
+	return out.pos - buf;
 }
 
 void bch2_io_clock_exit(struct io_clock *clock)
@@ -168,6 +181,8 @@ int bch2_io_clock_init(struct io_clock *clock)
 	atomic_long_set(&clock->now, 0);
 	spin_lock_init(&clock->timer_lock);
 
+	clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
+
 	clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
 	if (!clock->pcpu_buf)
 		return -ENOMEM;
diff --git a/fs/bcachefs/clock.h b/fs/bcachefs/clock.h
index bfbbca8a207b..da50afe206cc 100644
--- a/fs/bcachefs/clock.h
+++ b/fs/bcachefs/clock.h
@@ -7,7 +7,7 @@ void bch2_io_timer_del(struct io_clock *, struct io_timer *);
 void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long,
 				unsigned long);
 
-void __bch2_increment_clock(struct io_clock *);
+void __bch2_increment_clock(struct io_clock *, unsigned);
 
 static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
 					int rw)
@@ -16,7 +16,7 @@ static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
 
 	if (unlikely(this_cpu_add_return(*clock->pcpu_buf, sectors) >=
 		   IO_CLOCK_PCPU_SECTORS))
-		__bch2_increment_clock(clock);
+		__bch2_increment_clock(clock, this_cpu_xchg(*clock->pcpu_buf, 0));
 }
 
 void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);
@@ -30,6 +30,8 @@ void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);
 	__ret;								\
 })
 
+ssize_t bch2_io_timers_show(struct io_clock *, char *);
+
 void bch2_io_clock_exit(struct io_clock *);
 int bch2_io_clock_init(struct io_clock *);
 
diff --git a/fs/bcachefs/clock_types.h b/fs/bcachefs/clock_types.h
index 2b5e499e12b4..92c740a47565 100644
--- a/fs/bcachefs/clock_types.h
+++ b/fs/bcachefs/clock_types.h
@@ -28,6 +28,7 @@ typedef HEAP(struct io_timer *)	io_timer_heap;
 struct io_clock {
 	atomic_long_t		now;
 	u16 __percpu		*pcpu_buf;
+	unsigned		max_slop;
 
 	spinlock_t		timer_lock;
 	io_timer_heap		timers;
diff --git a/fs/bcachefs/sysfs.c b/fs/bcachefs/sysfs.c
index 8d68331f8b63..767fd7bed2d0 100644
--- a/fs/bcachefs/sysfs.c
+++ b/fs/bcachefs/sysfs.c
@@ -18,6 +18,7 @@
 #include "btree_update_interior.h"
 #include "btree_gc.h"
 #include "buckets.h"
+#include "clock.h"
 #include "disk_groups.h"
 #include "ec.h"
 #include "inode.h"
@@ -198,6 +199,9 @@ rw_attribute(pd_controllers_update_seconds);
 read_attribute(meta_replicas_have);
 read_attribute(data_replicas_have);
 
+read_attribute(io_timers_read);
+read_attribute(io_timers_write);
+
 #ifdef CONFIG_BCACHEFS_TESTS
 write_attribute(perf_test);
 #endif /* CONFIG_BCACHEFS_TESTS */
@@ -404,6 +408,11 @@ SHOW(bch2_fs)
 	if (attr == &sysfs_new_stripes)
 		return bch2_new_stripes(c, buf);
 
+	if (attr == &sysfs_io_timers_read)
+		return bch2_io_timers_show(&c->io_clock[READ], buf);
+	if (attr == &sysfs_io_timers_write)
+		return bch2_io_timers_show(&c->io_clock[WRITE], buf);
+
 #define BCH_DEBUG_PARAM(name, description) sysfs_print(name, c->name);
 	BCH_DEBUG_PARAMS()
 #undef BCH_DEBUG_PARAM
@@ -581,6 +590,9 @@ struct attribute *bch2_fs_internal_files[] = {
 
 	&sysfs_new_stripes,
 
+	&sysfs_io_timers_read,
+	&sysfs_io_timers_write,
+
 	&sysfs_internal_uuid,
 
 #define BCH_DEBUG_PARAM(name, description) &sysfs_##name,