fs/ext2/balloc.c | 131 +++++++++++++++++++++++----------------- fs/ext2/ialloc.c | 8 +- fs/ext2/super.c | 8 ++ include/asm-alpha/bitops.h | 2 include/asm-arm/bitops.h | 4 + include/asm-cris/bitops.h | 2 include/asm-i386/bitops.h | 4 + include/asm-ia64/bitops.h | 2 include/asm-m68k/bitops.h | 18 +++++ include/asm-m68knommu/bitops.h | 18 +++++ include/asm-mips/bitops.h | 20 ++++++ include/asm-mips64/bitops.h | 20 ++++++ include/asm-parisc/bitops.h | 4 + include/asm-ppc/bitops.h | 2 include/asm-ppc64/bitops.h | 19 +++++ include/asm-s390/bitops.h | 4 + include/asm-s390x/bitops.h | 4 + include/asm-sh/bitops.h | 18 +++++ include/asm-sparc/bitops.h | 19 +++++ include/asm-sparc64/bitops.h | 2 include/asm-v850/bitops.h | 2 include/asm-x86_64/bitops.h | 4 + include/linux/blockgroup_lock.h | 58 +++++++++++++++++ include/linux/ext2_fs_sb.h | 5 + include/linux/percpu_counter.h | 81 ++++++++++++++++++++++++ kernel/ksyms.c | 2 lib/Makefile | 1 lib/percpu_counter.c | 18 +++++ 28 files changed, 423 insertions(+), 57 deletions(-) diff -puN fs/ext2/balloc.c~ext2-no-lock_super-ng fs/ext2/balloc.c --- 25/fs/ext2/balloc.c~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/fs/ext2/balloc.c 2003-03-25 19:45:22.000000000 -0800 @@ -94,12 +94,19 @@ error_out: return bh; } -static inline int reserve_blocks(struct super_block *sb, int count) +/* + * Set sb->s_dirt here because the superblock was "logically" altered. We + * need to recalculate its free blocks count and flush it out. + */ +static int reserve_blocks(struct super_block *sb, int count) { - struct ext2_sb_info * sbi = EXT2_SB(sb); - struct ext2_super_block * es = sbi->s_es; - unsigned free_blocks = le32_to_cpu(es->s_free_blocks_count); - unsigned root_blocks = le32_to_cpu(es->s_r_blocks_count); + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = sbi->s_es; + unsigned free_blocks; + unsigned root_blocks; + + free_blocks = percpu_counter_read(&sbi->freeblocks_counter); + root_blocks = le32_to_cpu(es->s_r_blocks_count); if (free_blocks < count) count = free_blocks; @@ -117,46 +124,51 @@ static inline int reserve_blocks(struct return 0; } - es->s_free_blocks_count = cpu_to_le32(free_blocks - count); - mark_buffer_dirty(sbi->s_sbh); + percpu_counter_mod(&sbi->freeblocks_counter, -count); sb->s_dirt = 1; return count; } -static inline void release_blocks(struct super_block *sb, int count) +static void release_blocks(struct super_block *sb, int count) { if (count) { - struct ext2_sb_info * sbi = EXT2_SB(sb); - struct ext2_super_block * es = sbi->s_es; - unsigned free_blocks = le32_to_cpu(es->s_free_blocks_count); - es->s_free_blocks_count = cpu_to_le32(free_blocks + count); - mark_buffer_dirty(sbi->s_sbh); + struct ext2_sb_info *sbi = EXT2_SB(sb); + + percpu_counter_mod(&sbi->freeblocks_counter, count); sb->s_dirt = 1; } } -static inline int group_reserve_blocks(struct ext2_group_desc *desc, - struct buffer_head *bh, int count) +static int group_reserve_blocks(struct ext2_sb_info *sbi, int group_no, + struct ext2_group_desc *desc, struct buffer_head *bh, int count) { unsigned free_blocks; if (!desc->bg_free_blocks_count) return 0; + spin_lock(sb_bgl_lock(sbi, group_no)); free_blocks = le16_to_cpu(desc->bg_free_blocks_count); if (free_blocks < count) count = free_blocks; desc->bg_free_blocks_count = cpu_to_le16(free_blocks - count); + spin_unlock(sb_bgl_lock(sbi, group_no)); mark_buffer_dirty(bh); return count; } -static inline void group_release_blocks(struct ext2_group_desc *desc, - struct buffer_head *bh, int count) +static void group_release_blocks(struct super_block *sb, int group_no, + struct ext2_group_desc *desc, struct buffer_head *bh, int count) { if (count) { - unsigned free_blocks = le16_to_cpu(desc->bg_free_blocks_count); + struct ext2_sb_info *sbi = EXT2_SB(sb); + unsigned free_blocks; + + spin_lock(sb_bgl_lock(sbi, group_no)); + free_blocks = le16_to_cpu(desc->bg_free_blocks_count); desc->bg_free_blocks_count = cpu_to_le16(free_blocks + count); + spin_unlock(sb_bgl_lock(sbi, group_no)); + sb->s_dirt = 1; mark_buffer_dirty(bh); } } @@ -172,12 +184,11 @@ void ext2_free_blocks (struct inode * in unsigned long i; unsigned long overflow; struct super_block * sb = inode->i_sb; + struct ext2_sb_info * sbi = EXT2_SB(sb); struct ext2_group_desc * desc; - struct ext2_super_block * es; + struct ext2_super_block * es = sbi->s_es; unsigned freed = 0, group_freed; - lock_super (sb); - es = EXT2_SB(sb)->s_es; if (block < le32_to_cpu(es->s_first_data_block) || block + count < block || block + count > le32_to_cpu(es->s_blocks_count)) { @@ -215,16 +226,17 @@ do_more: if (in_range (le32_to_cpu(desc->bg_block_bitmap), block, count) || in_range (le32_to_cpu(desc->bg_inode_bitmap), block, count) || in_range (block, le32_to_cpu(desc->bg_inode_table), - EXT2_SB(sb)->s_itb_per_group) || + sbi->s_itb_per_group) || in_range (block + count - 1, le32_to_cpu(desc->bg_inode_table), - EXT2_SB(sb)->s_itb_per_group)) + sbi->s_itb_per_group)) ext2_error (sb, "ext2_free_blocks", "Freeing blocks in system zones - " "Block = %lu, count = %lu", block, count); for (i = 0, group_freed = 0; i < count; i++) { - if (!ext2_clear_bit(bit + i, bitmap_bh->b_data)) + if (!ext2_clear_bit_atomic(sb_bgl_lock(sbi, block_group), + bit + i, (void *) bitmap_bh->b_data)) ext2_error (sb, "ext2_free_blocks", "bit already cleared for block %lu", block + i); @@ -236,7 +248,7 @@ do_more: if (sb->s_flags & MS_SYNCHRONOUS) sync_dirty_buffer(bitmap_bh); - group_release_blocks(desc, bh2, group_freed); + group_release_blocks(sb, block_group, desc, bh2, group_freed); freed += group_freed; if (overflow) { @@ -247,17 +259,18 @@ do_more: error_return: brelse(bitmap_bh); release_blocks(sb, freed); - unlock_super (sb); DQUOT_FREE_BLOCK(inode, freed); } -static int grab_block(char *map, unsigned size, int goal) +static int grab_block(spinlock_t *lock, char *map, unsigned size, int goal) { int k; char *p, *r; if (!ext2_test_bit(goal, map)) goto got_it; + +repeat: if (goal) { /* * The goal was occupied; search forward for a free @@ -297,7 +310,8 @@ static int grab_block(char *map, unsigne } return -1; got_it: - ext2_set_bit(goal, map); + if (ext2_set_bit_atomic(lock, goal, (void *) map)) + goto repeat; return goal; } @@ -309,15 +323,15 @@ got_it: * bitmap, and then for any free bit if that fails. * This function also updates quota and i_blocks field. */ -int ext2_new_block (struct inode * inode, unsigned long goal, - u32 * prealloc_count, u32 * prealloc_block, int * err) +int ext2_new_block(struct inode *inode, unsigned long goal, + u32 *prealloc_count, u32 *prealloc_block, int *err) { struct buffer_head *bitmap_bh = NULL; struct buffer_head *gdp_bh; /* bh2 */ struct ext2_group_desc *desc; int group_no; /* i */ int ret_block; /* j */ - int bit; /* k */ + int bit; /* k */ int target_block; /* tmp */ int block = 0; struct super_block *sb = inode->i_sb; @@ -341,13 +355,10 @@ int ext2_new_block (struct inode * inode prealloc_goal--; dq_alloc = prealloc_goal + 1; - - lock_super (sb); - es_alloc = reserve_blocks(sb, dq_alloc); if (!es_alloc) { *err = -ENOSPC; - goto out_unlock; + goto out_dquot; } ext2_debug ("goal=%lu.\n", goal); @@ -357,10 +368,16 @@ int ext2_new_block (struct inode * inode goal = le32_to_cpu(es->s_first_data_block); group_no = (goal - le32_to_cpu(es->s_first_data_block)) / group_size; desc = ext2_get_group_desc (sb, group_no, &gdp_bh); - if (!desc) + if (!desc) { + /* + * gdp_bh may still be uninitialised. But group_release_blocks + * will not touch it because group_alloc is zero. + */ goto io_error; + } - group_alloc = group_reserve_blocks(desc, gdp_bh, es_alloc); + group_alloc = group_reserve_blocks(sbi, group_no, desc, + gdp_bh, es_alloc); if (group_alloc) { ret_block = ((goal - le32_to_cpu(es->s_first_data_block)) % group_size); @@ -371,11 +388,11 @@ int ext2_new_block (struct inode * inode ext2_debug("goal is at %d:%d.\n", group_no, ret_block); - ret_block = grab_block(bitmap_bh->b_data, - group_size, ret_block); + ret_block = grab_block(sb_bgl_lock(sbi, group_no), + bitmap_bh->b_data, group_size, ret_block); if (ret_block >= 0) goto got_block; - group_release_blocks(desc, gdp_bh, group_alloc); + group_release_blocks(sb, group_no, desc, gdp_bh, group_alloc); group_alloc = 0; } @@ -393,7 +410,8 @@ int ext2_new_block (struct inode * inode desc = ext2_get_group_desc(sb, group_no, &gdp_bh); if (!desc) goto io_error; - group_alloc = group_reserve_blocks(desc, gdp_bh, es_alloc); + group_alloc = group_reserve_blocks(sbi, group_no, desc, + gdp_bh, es_alloc); } if (!group_alloc) { *err = -ENOSPC; @@ -404,7 +422,8 @@ int ext2_new_block (struct inode * inode if (!bitmap_bh) goto io_error; - ret_block = grab_block(bitmap_bh->b_data, group_size, 0); + ret_block = grab_block(sb_bgl_lock(sbi, group_no), bitmap_bh->b_data, + group_size, 0); if (ret_block < 0) { ext2_error (sb, "ext2_new_block", "Free blocks count corrupted for block group %d", @@ -452,7 +471,9 @@ got_block: unsigned n; for (n = 0; n < group_alloc && ++ret_block < group_size; n++) { - if (ext2_set_bit(ret_block, bitmap_bh->b_data)) + if (ext2_set_bit_atomic(sb_bgl_lock(sbi, group_no), + ret_block, + (void*) bitmap_bh->b_data)) break; } *prealloc_block = block + 1; @@ -471,10 +492,9 @@ got_block: *err = 0; out_release: - group_release_blocks(desc, gdp_bh, group_alloc); + group_release_blocks(sb, group_no, desc, gdp_bh, group_alloc); release_blocks(sb, es_alloc); -out_unlock: - unlock_super (sb); +out_dquot: DQUOT_FREE_BLOCK(inode, dq_alloc); out: brelse(bitmap_bh); @@ -487,11 +507,11 @@ io_error: unsigned long ext2_count_free_blocks (struct super_block * sb) { -#ifdef EXT2FS_DEBUG - struct ext2_super_block * es; - unsigned long desc_count, bitmap_count, x; struct ext2_group_desc * desc; + unsigned long desc_count = 0; int i; +#ifdef EXT2FS_DEBUG + unsigned long bitmap_count, x; lock_super (sb); es = EXT2_SB(sb)->s_es; @@ -519,13 +539,18 @@ unsigned long ext2_count_free_blocks (st unlock_super (sb); return bitmap_count; #else - return le32_to_cpu(EXT2_SB(sb)->s_es->s_free_blocks_count); + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + desc = ext2_get_group_desc (sb, i, NULL); + if (!desc) + continue; + desc_count += le16_to_cpu(desc->bg_free_blocks_count); + } + return desc_count; #endif } -static inline int block_in_use (unsigned long block, - struct super_block * sb, - unsigned char * map) +static inline int +block_in_use(unsigned long block, struct super_block *sb, unsigned char *map) { return ext2_test_bit ((block - le32_to_cpu(EXT2_SB(sb)->s_es->s_first_data_block)) % EXT2_BLOCKS_PER_GROUP(sb), map); diff -puN fs/ext2/ialloc.c~ext2-no-lock_super-ng fs/ext2/ialloc.c --- 25/fs/ext2/ialloc.c~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/fs/ext2/ialloc.c 2003-03-25 19:20:42.000000000 -0800 @@ -278,7 +278,8 @@ static int find_group_orlov(struct super int ngroups = sbi->s_groups_count; int inodes_per_group = EXT2_INODES_PER_GROUP(sb); int avefreei = le32_to_cpu(es->s_free_inodes_count) / ngroups; - int avefreeb = le32_to_cpu(es->s_free_blocks_count) / ngroups; + int free_blocks = percpu_counter_read(&sbi->freeblocks_counter); + int avefreeb = free_blocks / ngroups; int blocks_per_dir; int ndirs = sbi->s_dir_count; int max_debt, max_dirs, min_blocks, min_inodes; @@ -320,8 +321,7 @@ static int find_group_orlov(struct super goto fallback; } - blocks_per_dir = (le32_to_cpu(es->s_blocks_count) - - le32_to_cpu(es->s_free_blocks_count)) / ndirs; + blocks_per_dir = (le32_to_cpu(es->s_blocks_count) - free_blocks) / ndirs; max_dirs = ndirs / ngroups + inodes_per_group / 16; min_inodes = avefreei - inodes_per_group / 4; @@ -500,6 +500,7 @@ repeat: es->s_free_inodes_count = cpu_to_le32(le32_to_cpu(es->s_free_inodes_count) - 1); + spin_lock(sb_bgl_lock(EXT2_SB(sb), group)); if (S_ISDIR(mode)) { if (EXT2_SB(sb)->s_debts[group] < 255) EXT2_SB(sb)->s_debts[group]++; @@ -507,6 +508,7 @@ repeat: if (EXT2_SB(sb)->s_debts[group]) EXT2_SB(sb)->s_debts[group]--; } + spin_unlock(sb_bgl_lock(EXT2_SB(sb), group)); mark_buffer_dirty(EXT2_SB(sb)->s_sbh); sb->s_dirt = 1; diff -puN fs/ext2/super.c~ext2-no-lock_super-ng fs/ext2/super.c --- 25/fs/ext2/super.c~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/fs/ext2/super.c 2003-03-25 19:20:42.000000000 -0800 @@ -769,6 +769,8 @@ static int ext2_fill_super(struct super_ printk ("EXT2-fs: not enough memory\n"); goto failed_mount; } + percpu_counter_init(&sbi->freeblocks_counter); + bgl_lock_init(&sbi->blockgroup_lock); sbi->s_debts = kmalloc(sbi->s_groups_count * sizeof(*sbi->s_debts), GFP_KERNEL); if (!sbi->s_debts) { @@ -814,6 +816,8 @@ static int ext2_fill_super(struct super_ ext2_warning(sb, __FUNCTION__, "mounting ext3 filesystem as ext2\n"); ext2_setup_super (sb, es, sb->s_flags & MS_RDONLY); + percpu_counter_mod(&sbi->freeblocks_counter, + ext2_count_free_blocks(sb)); return 0; failed_mount2: for (i = 0; i < db_count; i++) @@ -840,6 +844,7 @@ static void ext2_commit_super (struct su static void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es) { + es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb)); es->s_wtime = cpu_to_le32(get_seconds()); mark_buffer_dirty(EXT2_SB(sb)->s_sbh); sync_dirty_buffer(EXT2_SB(sb)->s_sbh); @@ -868,6 +873,7 @@ void ext2_write_super (struct super_bloc ext2_debug ("setting valid to 0\n"); es->s_state = cpu_to_le16(le16_to_cpu(es->s_state) & ~EXT2_VALID_FS); + es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb)); es->s_mtime = cpu_to_le32(get_seconds()); ext2_sync_super(sb, es); } else @@ -965,7 +971,7 @@ static int ext2_statfs (struct super_blo buf->f_type = EXT2_SUPER_MAGIC; buf->f_bsize = sb->s_blocksize; buf->f_blocks = le32_to_cpu(sbi->s_es->s_blocks_count) - overhead; - buf->f_bfree = ext2_count_free_blocks (sb); + buf->f_bfree = ext2_count_free_blocks(sb); buf->f_bavail = buf->f_bfree - le32_to_cpu(sbi->s_es->s_r_blocks_count); if (buf->f_bfree < le32_to_cpu(sbi->s_es->s_r_blocks_count)) buf->f_bavail = 0; diff -puN include/asm-alpha/bitops.h~ext2-no-lock_super-ng include/asm-alpha/bitops.h --- 25/include/asm-alpha/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-alpha/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -487,7 +487,9 @@ sched_find_first_bit(unsigned long b[3]) #define ext2_set_bit __test_and_set_bit +#define ext2_set_bit_atomic(l,n,a) test_and_set_bit(n,a) #define ext2_clear_bit __test_and_clear_bit +#define ext2_clear_bit_atomic(l,n,a) test_and_clear_bit(n,a) #define ext2_test_bit test_bit #define ext2_find_first_zero_bit find_first_zero_bit #define ext2_find_next_zero_bit find_next_zero_bit diff -puN include/asm-arm/bitops.h~ext2-no-lock_super-ng include/asm-arm/bitops.h --- 25/include/asm-arm/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-arm/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -357,8 +357,12 @@ static inline int sched_find_first_bit(u */ #define ext2_set_bit(nr,p) \ __test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) +#define ext2_set_bit_atomic(lock,nr,p) \ + test_and_set_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) #define ext2_clear_bit(nr,p) \ __test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) +#define ext2_clear_bit_atomic(lock,nr,p) \ + test_and_clear_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) #define ext2_test_bit(nr,p) \ __test_bit(WORD_BITOFF_TO_LE(nr), (unsigned long *)(p)) #define ext2_find_first_zero_bit(p,sz) \ diff -puN include/asm-cris/bitops.h~ext2-no-lock_super-ng include/asm-cris/bitops.h --- 25/include/asm-cris/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-cris/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -360,7 +360,9 @@ static inline int find_next_zero_bit (vo #define hweight8(x) generic_hweight8(x) #define ext2_set_bit test_and_set_bit +#define ext2_set_bit_atomic(l,n,a) test_and_set_bit(n,a) #define ext2_clear_bit test_and_clear_bit +#define ext2_clear_bit_atomic(l,n,a) test_and_clear_bit(n,a) #define ext2_test_bit test_bit #define ext2_find_first_zero_bit find_first_zero_bit #define ext2_find_next_zero_bit find_next_zero_bit diff -puN include/asm-i386/bitops.h~ext2-no-lock_super-ng include/asm-i386/bitops.h --- 25/include/asm-i386/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-i386/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -479,8 +479,12 @@ static __inline__ int ffs(int x) #define ext2_set_bit(nr,addr) \ __test_and_set_bit((nr),(unsigned long*)addr) +#define ext2_set_bit_atomic(lock,nr,addr) \ + test_and_set_bit((nr),(unsigned long*)addr) #define ext2_clear_bit(nr, addr) \ __test_and_clear_bit((nr),(unsigned long*)addr) +#define ext2_clear_bit_atomic(lock,nr, addr) \ + test_and_clear_bit((nr),(unsigned long*)addr) #define ext2_test_bit(nr, addr) test_bit((nr),(unsigned long*)addr) #define ext2_find_first_zero_bit(addr, size) \ find_first_zero_bit((unsigned long*)addr, size) diff -puN include/asm-ia64/bitops.h~ext2-no-lock_super-ng include/asm-ia64/bitops.h --- 25/include/asm-ia64/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-ia64/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -453,7 +453,9 @@ find_next_bit (void *addr, unsigned long #define __clear_bit(nr, addr) clear_bit(nr, addr) #define ext2_set_bit test_and_set_bit +#define ext2_set_atomic(l,n,a) test_and_set_bit(n,a) #define ext2_clear_bit test_and_clear_bit +#define ext2_clear_atomic(l,n,a) test_and_clear_bit(n,a) #define ext2_test_bit test_bit #define ext2_find_first_zero_bit find_first_zero_bit #define ext2_find_next_zero_bit find_next_zero_bit diff -puN include/asm-m68k/bitops.h~ext2-no-lock_super-ng include/asm-m68k/bitops.h --- 25/include/asm-m68k/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-m68k/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -365,6 +365,24 @@ ext2_clear_bit (int nr, volatile void *v return retval; } +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + extern __inline__ int ext2_test_bit (int nr, const volatile void *vaddr) { diff -puN include/asm-m68knommu/bitops.h~ext2-no-lock_super-ng include/asm-m68knommu/bitops.h --- 25/include/asm-m68knommu/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-m68knommu/bitops.h 2003-03-25 18:41:54.000000000 -0800 @@ -402,6 +402,24 @@ extern __inline__ int ext2_clear_bit(int return retval; } +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + extern __inline__ int ext2_test_bit(int nr, const volatile void * addr) { int mask; diff -puN include/asm-mips64/bitops.h~ext2-no-lock_super-ng include/asm-mips64/bitops.h --- 25/include/asm-mips64/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-mips64/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -531,6 +531,24 @@ ext2_clear_bit(int nr, void * addr) return retval; } +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + extern inline int ext2_test_bit(int nr, const void * addr) { @@ -599,7 +617,9 @@ found_middle: /* Native ext2 byte ordering, just collapse using defines. */ #define ext2_set_bit(nr, addr) test_and_set_bit((nr), (addr)) +#define ext2_set_bit_atomic(lock, nr, addr) test_and_set_bit((nr), (addr)) #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr), (addr)) +#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr), (addr)) #define ext2_test_bit(nr, addr) test_bit((nr), (addr)) #define ext2_find_first_zero_bit(addr, size) find_first_zero_bit((addr), (size)) #define ext2_find_next_zero_bit(addr, size, offset) \ diff -puN include/asm-mips/bitops.h~ext2-no-lock_super-ng include/asm-mips/bitops.h --- 25/include/asm-mips/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-mips/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -824,6 +824,24 @@ extern __inline__ int ext2_clear_bit(int return retval; } +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + extern __inline__ int ext2_test_bit(int nr, const void * addr) { int mask; @@ -890,7 +908,9 @@ found_middle: /* Native ext2 byte ordering, just collapse using defines. */ #define ext2_set_bit(nr, addr) test_and_set_bit((nr), (addr)) +#define ext2_set_bit_atomic(lock, nr, addr) test_and_set_bit((nr), (addr)) #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr), (addr)) +#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr), (addr)) #define ext2_test_bit(nr, addr) test_bit((nr), (addr)) #define ext2_find_first_zero_bit(addr, size) find_first_zero_bit((addr), (size)) #define ext2_find_next_zero_bit(addr, size, offset) \ diff -puN include/asm-parisc/bitops.h~ext2-no-lock_super-ng include/asm-parisc/bitops.h --- 25/include/asm-parisc/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-parisc/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -389,10 +389,14 @@ found_middle: */ #ifdef __LP64__ #define ext2_set_bit(nr, addr) test_and_set_bit((nr) ^ 0x38, addr) +#define ext2_set_bit_atomic(l,nr,addr) test_and_set_bit((nr) ^ 0x38, addr) #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr) ^ 0x38, addr) +#define ext2_clear_bit_atomic(l,nr,addr) test_and_clear_bit((nr) ^ 0x38, addr) #else #define ext2_set_bit(nr, addr) test_and_set_bit((nr) ^ 0x18, addr) +#define ext2_set_bit_atomic(l,nr,addr) test_and_set_bit((nr) ^ 0x18, addr) #define ext2_clear_bit(nr, addr) test_and_clear_bit((nr) ^ 0x18, addr) +#define ext2_clear_bit_atomic(l,nr,addr) test_and_clear_bit((nr) ^ 0x18, addr) #endif #endif /* __KERNEL__ */ diff -puN include/asm-ppc64/bitops.h~ext2-no-lock_super-ng include/asm-ppc64/bitops.h --- 25/include/asm-ppc64/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-ppc64/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -338,6 +338,25 @@ static __inline__ int __test_and_clear_l __test_and_set_le_bit((nr),(unsigned long*)addr) #define ext2_clear_bit(nr, addr) \ __test_and_clear_le_bit((nr),(unsigned long*)addr) + +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + #define ext2_test_bit(nr, addr) test_le_bit((nr),(unsigned long*)addr) #define ext2_find_first_zero_bit(addr, size) \ find_first_zero_le_bit((unsigned long*)addr, size) diff -puN include/asm-ppc/bitops.h~ext2-no-lock_super-ng include/asm-ppc/bitops.h --- 25/include/asm-ppc/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-ppc/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -392,7 +392,9 @@ found_middle: #define ext2_set_bit(nr, addr) __test_and_set_bit((nr) ^ 0x18, (unsigned long *)(addr)) +#define ext2_set_bit_atomic(lock, nr, addr) test_and_set_bit((nr) ^ 0x18, (unsigned long *)(addr)) #define ext2_clear_bit(nr, addr) __test_and_clear_bit((nr) ^ 0x18, (unsigned long *)(addr)) +#define ext2_clear_bit_atomic(lock, nr, addr) test_and_clear_bit((nr) ^ 0x18, (unsigned long *)(addr)) static __inline__ int ext2_test_bit(int nr, __const__ void * addr) { diff -puN include/asm-s390/bitops.h~ext2-no-lock_super-ng include/asm-s390/bitops.h --- 25/include/asm-s390/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-s390/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -805,8 +805,12 @@ extern __inline__ int fls(int x) #define ext2_set_bit(nr, addr) \ test_and_set_bit((nr)^24, (unsigned long *)addr) +#define ext2_set_bit_atomic(lock, nr, addr) \ + test_and_set_bit((nr)^24, (unsigned long *)addr) #define ext2_clear_bit(nr, addr) \ test_and_clear_bit((nr)^24, (unsigned long *)addr) +#define ext2_clear_bit_atomic(lock, nr, addr) \ + test_and_clear_bit((nr)^24, (unsigned long *)addr) #define ext2_test_bit(nr, addr) \ test_bit((nr)^24, (unsigned long *)addr) diff -puN include/asm-s390x/bitops.h~ext2-no-lock_super-ng include/asm-s390x/bitops.h --- 25/include/asm-s390x/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-s390x/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -838,8 +838,12 @@ extern __inline__ int fls(int x) #define ext2_set_bit(nr, addr) \ test_and_set_bit((nr)^56, (unsigned long *)addr) +#define ext2_set_bit_atomic(lock, nr, addr) \ + test_and_set_bit((nr)^56, (unsigned long *)addr) #define ext2_clear_bit(nr, addr) \ test_and_clear_bit((nr)^56, (unsigned long *)addr) +#define ext2_clear_bit_atomic(lock, nr, addr) \ + test_and_clear_bit((nr)^56, (unsigned long *)addr) #define ext2_test_bit(nr, addr) \ test_bit((nr)^56, (unsigned long *)addr) diff -puN include/asm-sh/bitops.h~ext2-no-lock_super-ng include/asm-sh/bitops.h --- 25/include/asm-sh/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-sh/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -344,6 +344,24 @@ found_middle: } #endif +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + /* Bitmap functions for the minix filesystem. */ #define minix_test_and_set_bit(nr,addr) test_and_set_bit(nr,addr) #define minix_set_bit(nr,addr) set_bit(nr,addr) diff -puN include/asm-sparc64/bitops.h~ext2-no-lock_super-ng include/asm-sparc64/bitops.h --- 25/include/asm-sparc64/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-sparc64/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -351,7 +351,9 @@ found_middle: #ifdef __KERNEL__ #define ext2_set_bit(nr,addr) test_and_set_le_bit((nr),(unsigned long *)(addr)) +#define ext2_set_bit_atomic(lock,nr,addr) test_and_set_le_bit((nr),(unsigned long *)(addr)) #define ext2_clear_bit(nr,addr) test_and_clear_le_bit((nr),(unsigned long *)(addr)) +#define ext2_clear_bit_atomic(lock,nr,addr) test_and_clear_le_bit((nr),(unsigned long *)(addr)) #define ext2_test_bit(nr,addr) test_le_bit((nr),(unsigned long *)(addr)) #define ext2_find_first_zero_bit(addr, size) \ find_first_zero_le_bit((unsigned long *)(addr), (size)) diff -puN include/asm-sparc/bitops.h~ext2-no-lock_super-ng include/asm-sparc/bitops.h --- 25/include/asm-sparc/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-sparc/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -455,6 +455,25 @@ found_middle: #define ext2_set_bit __test_and_set_le_bit #define ext2_clear_bit __test_and_clear_le_bit + +#define ext2_set_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_set_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + +#define ext2_clear_bit_atomic(lock, nr, addr) \ + ({ \ + int ret; \ + spin_lock(lock); \ + ret = ext2_clear_bit((nr), (addr)); \ + spin_unlock(lock); \ + ret; \ + }) + #define ext2_test_bit test_le_bit #define ext2_find_first_zero_bit find_first_zero_le_bit #define ext2_find_next_zero_bit find_next_zero_le_bit diff -puN include/asm-v850/bitops.h~ext2-no-lock_super-ng include/asm-v850/bitops.h --- 25/include/asm-v850/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-v850/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -252,7 +252,9 @@ static inline int sched_find_first_bit(u #define hweight8(x) generic_hweight8 (x) #define ext2_set_bit test_and_set_bit +#define ext2_set_bit_atomic(l,n,a) test_and_set_bit(n,a) #define ext2_clear_bit test_and_clear_bit +#define ext2_clear_bit_atomic(l,n,a) test_and_clear_bit(n,a) #define ext2_test_bit test_bit #define ext2_find_first_zero_bit find_first_zero_bit #define ext2_find_next_zero_bit find_next_zero_bit diff -puN include/asm-x86_64/bitops.h~ext2-no-lock_super-ng include/asm-x86_64/bitops.h --- 25/include/asm-x86_64/bitops.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/asm-x86_64/bitops.h 2003-03-25 18:41:55.000000000 -0800 @@ -487,8 +487,12 @@ static __inline__ int ffs(int x) #define ext2_set_bit(nr,addr) \ __test_and_set_bit((nr),(unsigned long*)addr) +#define ext2_set_bit_atomic(lock,nr,addr) \ + test_and_set_bit((nr),(unsigned long*)addr) #define ext2_clear_bit(nr, addr) \ __test_and_clear_bit((nr),(unsigned long*)addr) +#define ext2_clear_bit_atomic(lock,nr,addr) \ + test_and_clear_bit((nr),(unsigned long*)addr) #define ext2_test_bit(nr, addr) test_bit((nr),(unsigned long*)addr) #define ext2_find_first_zero_bit(addr, size) \ find_first_zero_bit((unsigned long*)addr, size) diff -puN /dev/null include/linux/blockgroup_lock.h --- /dev/null 2002-08-30 16:31:37.000000000 -0700 +++ 25-akpm/include/linux/blockgroup_lock.h 2003-03-25 18:41:55.000000000 -0800 @@ -0,0 +1,58 @@ +/* + * Per-blockgroup locking for ext2 and ext3. + * + * Simple hashed spinlocking. + */ + +#include +#include +#include + +#ifdef CONFIG_SMP + +/* + * We want a power-of-two. Is there a better way than this? + */ + +#if NR_CPUS >= 32 +#define NR_BG_LOCKS 128 +#elif NR_CPUS >= 16 +#define NR_BG_LOCKS 64 +#elif NR_CPUS >= 8 +#define NR_BG_LOCKS 32 +#elif NR_CPUS >= 4 +#define NR_BG_LOCKS 16 +#elif NR_CPUS >= 2 +#define NR_BG_LOCKS 8 +#else +#define NR_BG_LOCKS 4 +#endif + +#else /* CONFIG_SMP */ +#define NR_BG_LOCKS 1 +#endif + +struct bgl_lock { + spinlock_t lock; +} ____cacheline_aligned_in_smp; + +struct blockgroup_lock { + struct bgl_lock locks[NR_BG_LOCKS]; +}; + +static inline void bgl_lock_init(struct blockgroup_lock *bgl) +{ + int i; + + for (i = 0; i < NR_BG_LOCKS; i++) + spin_lock_init(&bgl->locks[i].lock); +} + +/* + * The accessor is a macro so we can embed a blockgroup_lock into different + * superblock types + */ +#define sb_bgl_lock(sb, block_group) \ + (&(sb)->blockgroup_lock.locks[(block_group) & (NR_BG_LOCKS-1)].lock) + + diff -puN include/linux/ext2_fs_sb.h~ext2-no-lock_super-ng include/linux/ext2_fs_sb.h --- 25/include/linux/ext2_fs_sb.h~ext2-no-lock_super-ng 2003-03-25 18:41:54.000000000 -0800 +++ 25-akpm/include/linux/ext2_fs_sb.h 2003-03-25 19:20:42.000000000 -0800 @@ -16,6 +16,9 @@ #ifndef _LINUX_EXT2_FS_SB #define _LINUX_EXT2_FS_SB +#include +#include + /* * second extended-fs super-block data in memory */ @@ -45,6 +48,8 @@ struct ext2_sb_info { u32 s_next_generation; unsigned long s_dir_count; u8 *s_debts; + struct percpu_counter freeblocks_counter; + struct blockgroup_lock blockgroup_lock; }; #endif /* _LINUX_EXT2_FS_SB */ diff -puN /dev/null include/linux/percpu_counter.h --- /dev/null 2002-08-30 16:31:37.000000000 -0700 +++ 25-akpm/include/linux/percpu_counter.h 2003-03-25 19:32:53.000000000 -0800 @@ -0,0 +1,81 @@ +/* + * A simple "approximate counter" for use in ext2 and ext3 superblocks. + * + * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4. + */ + +#include +#include +#include +#include + +#ifdef CONFIG_SMP + +struct __percpu_counter { + long count; +} ____cacheline_aligned; + +struct percpu_counter { + spinlock_t lock; + long count; + struct __percpu_counter counters[NR_CPUS]; +}; + +#if NR_CPUS >= 16 +#define FBC_BATCH (NR_CPUS*2) +#else +#define FBC_BATCH (NR_CPUS*4) +#endif + +static inline void percpu_counter_init(struct percpu_counter *fbc) +{ + int i; + + spin_lock_init(&fbc->lock); + fbc->count = 0; + for (i = 0; i < NR_CPUS; i++) + fbc->counters[i].count = 0; +} + +void percpu_counter_mod(struct percpu_counter *fbc, long amount); + +static inline long percpu_counter_read(struct percpu_counter *fbc) +{ + return fbc->count; +} + +#else + +struct percpu_counter { + long count; +}; + +static inline void percpu_counter_init(struct percpu_counter *fbc) +{ + fbc->count = 0; +} + +static inline void +percpu_counter_mod(struct percpu_counter *fbc, long amount) +{ + preempt_disable(); + fbc->count += amount; + preempt_enable(); +} + +static inline long percpu_counter_read(struct percpu_counter *fbc) +{ + return fbc->count; +} + +#endif /* CONFIG_SMP */ + +static inline void percpu_counter_inc(struct percpu_counter *fbc) +{ + percpu_counter_mod(fbc, 1); +} + +static inline void percpu_counter_dec(struct percpu_counter *fbc) +{ + percpu_counter_mod(fbc, -1); +} diff -puN /dev/null lib/percpu_counter.c --- /dev/null 2002-08-30 16:31:37.000000000 -0700 +++ 25-akpm/lib/percpu_counter.c 2003-03-25 19:34:02.000000000 -0800 @@ -0,0 +1,18 @@ + +#include + +void percpu_counter_mod(struct percpu_counter *fbc, long amount) +{ + int cpu = get_cpu(); + long count = fbc->counters[cpu].count; + + count += amount; + if (count >= FBC_BATCH || count <= -FBC_BATCH) { + spin_lock(&fbc->lock); + fbc->count += count; + spin_unlock(&fbc->lock); + count = 0; + } + fbc->counters[cpu].count = count; + put_cpu(); +} diff -puN lib/Makefile~ext2-no-lock_super-ng lib/Makefile --- 25/lib/Makefile~ext2-no-lock_super-ng 2003-03-25 19:31:51.000000000 -0800 +++ 25-akpm/lib/Makefile 2003-03-25 19:32:14.000000000 -0800 @@ -14,6 +14,7 @@ obj-y := errno.o ctype.o string.o vsprin obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o +obj-$(CONFIG_SMP) += percpu_counter.o ifneq ($(CONFIG_HAVE_DEC_LOCK),y) obj-y += dec_and_lock.o diff -puN kernel/ksyms.c~ext2-no-lock_super-ng kernel/ksyms.c --- 25/kernel/ksyms.c~ext2-no-lock_super-ng 2003-03-25 19:34:10.000000000 -0800 +++ 25-akpm/kernel/ksyms.c 2003-03-25 19:34:59.000000000 -0800 @@ -58,6 +58,7 @@ #include #include #include +#include #include #if defined(CONFIG_PROC_FS) @@ -100,6 +101,7 @@ EXPORT_SYMBOL(kfree); #ifdef CONFIG_SMP EXPORT_SYMBOL(kmalloc_percpu); EXPORT_SYMBOL(kfree_percpu); +EXPORT_SYMBOL(percpu_counter_mod); #endif EXPORT_SYMBOL(vfree); EXPORT_SYMBOL(__vmalloc); _