diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 6fcbb2184d1bdc0c68f0289d6da321236b9ab780..a31069001add18d7e772acc5d0aa45aba07298e9 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1179,6 +1179,10 @@ struct ext4_inode_info { __u32 i_csum_seed; kprojid_t i_projid; + + /* where last allocation was done - for stream allocation */ + ext4_group_t i_mb_last_group; + ext4_grpblk_t i_mb_last_start; }; /* @@ -1611,9 +1615,6 @@ struct ext4_sb_info { unsigned int s_mb_order2_reqs; unsigned int s_mb_group_prealloc; unsigned int s_max_dir_size_kb; - /* where last allocation was done - for stream allocation */ - unsigned long s_mb_last_group; - unsigned long s_mb_last_start; unsigned int s_mb_prefetch; unsigned int s_mb_prefetch_limit; unsigned int s_mb_best_avail_max_trim_order; @@ -3487,23 +3488,28 @@ static inline int ext4_fs_is_busy(struct ext4_sb_info *sbi) return (atomic_read(&sbi->s_lock_busy) > EXT4_CONTENTION_THRESHOLD); } +static inline bool ext4_try_lock_group(struct super_block *sb, ext4_group_t group) +{ + if (!spin_trylock(ext4_group_lock_ptr(sb, group))) + return false; + /* + * We're able to grab the lock right away, so drop the lock + * contention counter. + */ + atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, -1, 0); + return true; +} + static inline void ext4_lock_group(struct super_block *sb, ext4_group_t group) { - spinlock_t *lock = ext4_group_lock_ptr(sb, group); - if (spin_trylock(lock)) - /* - * We're able to grab the lock right away, so drop the - * lock contention counter. - */ - atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, -1, 0); - else { + if (!ext4_try_lock_group(sb, group)) { /* * The lock is busy, so bump the contention counter, * and then wait on the spin lock. */ atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, 1, EXT4_MAX_CONTENTION); - spin_lock(lock); + spin_lock(ext4_group_lock_ptr(sb, group)); } } diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 7c8cb88b426ecc736d6065735a9c63489c19288f..ec2808247f4f53378f91d3a2bd262728d26cc944 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -899,7 +899,8 @@ static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context bb_largest_free_order_node) { if (sbi->s_mb_stats) atomic64_inc(&sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED]); - if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) { + if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED)) && + !spin_is_locked(ext4_group_lock_ptr(ac->ac_sb, iter->bb_group))) { *group = iter->bb_group; ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED; read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); @@ -935,7 +936,8 @@ ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int o list_for_each_entry(iter, frag_list, bb_avg_fragment_size_node) { if (sbi->s_mb_stats) atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]); - if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) { + if (likely(ext4_mb_good_group(ac, iter->bb_group, cr)) && + !spin_is_locked(ext4_group_lock_ptr(ac->ac_sb, iter->bb_group))) { grp = iter; break; } @@ -2137,7 +2139,6 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, struct ext4_buddy *e4b) { - struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); int ret; BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); @@ -2168,10 +2169,8 @@ static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, get_page(ac->ac_buddy_page); /* store last allocated for subsequent stream allocation */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { - spin_lock(&sbi->s_md_lock); - sbi->s_mb_last_group = ac->ac_f_ex.fe_group; - sbi->s_mb_last_start = ac->ac_f_ex.fe_start; - spin_unlock(&sbi->s_md_lock); + EXT4_I(ac->ac_inode)->i_mb_last_group = ac->ac_f_ex.fe_group; + EXT4_I(ac->ac_inode)->i_mb_last_start = ac->ac_f_ex.fe_start; } /* * As we've just preallocated more space than @@ -2843,13 +2842,14 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) MB_NUM_ORDERS(sb)); } - /* if stream allocation is enabled, use global goal */ + /* if stream allocation is enabled, use last goal */ if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { - /* TBD: may be hot point */ - spin_lock(&sbi->s_md_lock); - ac->ac_g_ex.fe_group = sbi->s_mb_last_group; - ac->ac_g_ex.fe_start = sbi->s_mb_last_start; - spin_unlock(&sbi->s_md_lock); + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + + if (ei->i_mb_last_group || ei->i_mb_last_start) { + ac->ac_g_ex.fe_group = ei->i_mb_last_group; + ac->ac_g_ex.fe_start = ei->i_mb_last_start; + } } /* @@ -2911,7 +2911,13 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) if (err) goto out; - ext4_lock_group(sb, group); + /* skip busy group */ + if (cr >= CR_ANY_FREE) { + ext4_lock_group(sb, group); + } else if (!ext4_try_lock_group(sb, group)) { + ext4_mb_unload_buddy(&e4b); + continue; + } /* * We need to check again after locking the diff --git a/fs/ext4/super.c b/fs/ext4/super.c index c45dfcf9ac6243ff8b47336e00a50601ba482443..f7a301116d597930ff06973b8371b7ae562343e9 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1517,6 +1517,8 @@ static struct inode *ext4_alloc_inode(struct super_block *sb) INIT_WORK(&ei->i_iomap_ioend_work, ext4_iomap_end_io); ext4_fc_init_inode(&ei->vfs_inode); mutex_init(&ei->i_fc_lock); + ei->i_mb_last_group = 0; + ei->i_mb_last_start = 0; return &ei->vfs_inode; }