jbd2日志do_checkpoint和do_get_write_access并发在掉电场景导致文件系统损坏

【标题描述】jbd2日志do_checkpoint和do_get_write_access并发在掉电场景导致文件系统损坏
【环境信息】
硬件信息：
NA
软件信息：
OLK-5.10
【问题复现步骤】
1. Apply diff and compile kernel
2. ./test.sh
3. reboot
4. fsck.ext4 -fn /dev/sda
出现概率(是否必现，概率性错误)必现
【预期结果】
再次启动后文件系统损坏
【实际结果】
```
1. Apply diff and compile kernel
2. ./test.sh
[   21.226521] assign g_bh
[   21.244972] jbd2_journal_commit_transaction ---
[   21.246089] insert ffff888110d72750 into trans ffff8881004a0100
[   21.247396] mark buffer dirty
[   21.248069] jbd2_journal_commit_transaction ====
[   21.259153] commit ffff8881004a0100 chmod(2574)
[   21.260303] Do access to bh 
[   22.248480] do_get_write_access: clear bh dirty
[   22.249420] do_get_write_access: wait checkpoint remove
[   23.275690] checkpoint: bh dirty bit should be cleared
[   23.277361] jbd2_log_do_checkpoint: remove from ck list
[   23.278968] commit ffff8881004a0100 done
[   23.280295] commit ffff8881004a0100 done
[   23.301318] Kernel panic - not syncing: DONE
[   23.302195] CPU: 3 PID: 2574 Comm: chmod Not tainted 5.10.0-02793-g5b6c5d56de8f-dirty #520
[   23.304076] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20190727_073836-buildvm-ppc64le-16.ppc.fedoraproject.org-3.fc4
[   23.307291] Call Trace:
[   23.307921]  dump_stack+0xa6/0xd5
[   23.308748]  panic+0x164/0x491
[   23.309510]  jbd2_log_do_checkpoint.cold+0x87/0x5aa
[   23.310708]  __jbd2_log_wait_for_space+0x18a/0x360
[   23.311886]  ? create_object+0x27e/0x450
[   23.312856]  add_transaction_credits+0x419/0x4f0
[   23.313987]  ? add_timer+0x1f3/0x350
[   23.314870]  start_this_handle+0x150/0x960
[   23.315403]  ? kmemleak_alloc+0x66/0xd0
[   23.315917]  ? jbd2__journal_start+0xa5/0x2d0
[   23.316498]  jbd2__journal_start+0x124/0x2d0
[   23.317056]  __ext4_journal_start_sb+0x174/0x1d0
[   23.317669]  ext4_dirty_inode+0x49/0xb0
[   23.318172]  __mark_inode_dirty+0x71/0x590
[   23.318719]  ext4_setattr+0x7bd/0xf40
[   23.319199]  ? putname+0x6c/0xa0
[   23.319629]  ? filename_lookup+0x11c/0x230
[   23.320166]  notify_change+0x3c0/0x730
[   23.320663]  ? chmod_common+0xfc/0x1e0
[   23.321153]  chmod_common+0xfc/0x1e0
[   23.321631]  do_fchmodat+0x6e/0xf0
[   23.322079]  __x64_sys_fchmodat+0x1e/0x30
[   23.322604]  do_syscall_64+0x45/0x70
[   23.323073]  entry_SYSCALL_64_after_hwframe+0x61/0xc6
[   23.323741] RIP: 0033:0x7fa6336addb9
[   23.324209] Code: ff ff ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 66 90 f7 c1 ff fe ff ff 75 38 80 e5 01 75 4b 48 63 ff 89 d2 b8 0c 01 00 0
[   23.326609] RSP: 002b:00007ffdd371f018 EFLAGS: 00000246 ORIG_RAX: 000000000000010c
[   23.327605] RAX: ffffffffffffffda RBX: 0000000000000001 RCX: 00007fa6336addb9
[   23.328547] RDX: 00000000000001ed RSI: 00005628e98a30f0 RDI: ffffffffffffff9c
[   23.329484] RBP: 00005628e98a3060 R08: 0000000000000000 R09: 0000000000000000
[   23.330423] R10: 0000000000000000 R11: 0000000000000246 R12: 00005628e900a734
[   23.331361] R13: 00005628e98a30f0 R14: 00005628e98a4320 R15: 0000000000008000
[   23.332378] Kernel Offset: disabled
[   23.332847] ---[ end Kernel panic - not syncing: DONE ]---
3. reboot
4. [root@localhost ~]# fsck.ext4 -fn /dev/sda
e2fsck 1.43.4 (31-Jan-2017)
Warning: skipping journal recovery because doing a read-only filesystem check.
Pass 1: Checking inodes, blocks, and sizes
Pass 2: Checking directory structure
Entry 'a' in / (2) has deleted/unused inode 12.  Clear? no

Entry 'a' in / (2) has an incorrect filetype (was 1, should be 0).
Fix? no

Pass 3: Checking directory connectivity
Pass 4: Checking reference counts
Pass 5: Checking group summary information
Inode bitmap differences:  -12
Fix? no

Free inodes count wrong (1310709, counted=1310708).
Fix? no

/dev/sda: ********** WARNING: Filesystem still has errors **********

/dev/sda: 11/1310720 files (0.0% non-contiguous), 126322/5242880 blocks
```
【附件信息】
diff
```
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 3fcfc156d0d7..f598284e1e4a 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -5127,8 +5127,10 @@ static void ext4_update_other_inodes_time(struct super_block *sb,
  * buffer-cache.  This gobbles the caller's reference to the
  * buffer_head in the inode location struct.
  *
- * The caller must have write access to iloc->bh.
+ * The cailler must have write access to iloc->bh.
  */
+struct buffer_head *g_bh;
+int g_bh_dirty;
 static int ext4_do_update_inode(handle_t *handle,
 				struct inode *inode,
 				struct ext4_iloc *iloc)
@@ -5172,6 +5174,12 @@ static int ext4_do_update_inode(handle_t *handle,
 	err = ext4_handle_dirty_metadata(handle, NULL, bh);
 	if (err)
 		goto out_error;
+	smp_rmb();
+	if (!g_bh && !strcmp(current->comm, "aa")) {
+		pr_err("assign g_bh\n");
+		g_bh = bh;
+		smp_wmb();
+	}
 	ext4_clear_inode_state(inode, EXT4_STATE_NEW);
 	if (set_large_file) {
 		BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c
index 646fb512bd1e..7eb558703136 100644
--- a/fs/jbd2/checkpoint.c
+++ b/fs/jbd2/checkpoint.c
@@ -97,15 +97,17 @@ static inline bool __cp_buffer_busy(struct journal_head *jh)
  * Called under j-state_lock *only*.  It will be unlocked if we have to wait
  * for a checkpoint to free up some space in the log.
  */
+extern int g_bh_dirty;
 void __jbd2_log_wait_for_space(journal_t *journal)
 __acquires(&journal->j_state_lock)
 __releases(&journal->j_state_lock)
 {
 	int nblocks, space_left;
+	static int entered = 0;
 	/* assert_spin_locked(&journal->j_state_lock); */
 
 	nblocks = journal->j_max_transaction_buffers;
-	while (jbd2_log_space_left(journal) < nblocks) {
+	while (jbd2_log_space_left(journal) < nblocks || (!strcmp(current->comm, "chmod") && g_bh_dirty && !entered)) {
 		write_unlock(&journal->j_state_lock);
 		mutex_lock_io(&journal->j_checkpoint_mutex);
 
@@ -127,7 +129,7 @@ __releases(&journal->j_state_lock)
 		}
 		spin_lock(&journal->j_list_lock);
 		space_left = jbd2_log_space_left(journal);
-		if (space_left < nblocks) {
+		if (space_left < nblocks || (!strcmp(current->comm, "chmod") && g_bh_dirty)) {
 			int chkpt = journal->j_checkpoint_transactions != NULL;
 			tid_t tid = 0;
 
@@ -136,6 +138,7 @@ __releases(&journal->j_state_lock)
 			spin_unlock(&journal->j_list_lock);
 			write_unlock(&journal->j_state_lock);
 			if (chkpt) {
+				entered = 1;
 				jbd2_log_do_checkpoint(journal);
 			} else if (jbd2_cleanup_journal_tail(journal) == 0) {
 				/* We were able to recover space; yay! */
@@ -195,6 +198,8 @@ __flush_batch(journal_t *journal, int *batch_count)
  * The journal should be locked before calling this function.
  * Called with j_checkpoint_mutex held.
  */
+#include <linux/delay.h>
+extern struct buffer_head *g_bh;
 int jbd2_log_do_checkpoint(journal_t *journal)
 {
 	struct journal_head	*jh;
@@ -202,6 +207,7 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 	transaction_t		*transaction;
 	tid_t			this_tid;
 	int			result, batch_count = 0;
+	int need_shutdown = 0;
 
 	jbd_debug(1, "Start checkpoint\n");
 
@@ -227,6 +233,7 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 	if (transaction->t_chp_stats.cs_chp_time == 0)
 		transaction->t_chp_stats.cs_chp_time = jiffies;
 	this_tid = transaction->t_tid;
+	pr_err("commit %px %s(%d)\n", transaction, current->comm, current->pid);
 restart:
 	/*
 	 * If someone cleaned up this transaction while we slept, we're
@@ -279,14 +286,21 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 			 * checkpoint_mutex.  So we need to temporarily
 			 * drop it.
 			 */
+
 			mutex_unlock(&journal->j_checkpoint_mutex);
 			jbd2_log_wait_commit(journal, tid);
 			mutex_lock_io(&journal->j_checkpoint_mutex);
 			spin_lock(&journal->j_list_lock);
 			goto restart;
 		}
+		if (g_bh == bh)
+			pr_err("checkpoint: bh dirty bit should be cleared\n");
 		if (!buffer_dirty(bh)) {
 			BUFFER_TRACE(bh, "remove from checkpoint");
+			if (g_bh == bh) {
+				pr_err("%s: remove from ck list\n", __func__);
+				need_shutdown = 1;
+			}
 			if (__jbd2_journal_remove_checkpoint(jh))
 				/* The transaction was released; we're done */
 				goto out;
@@ -306,11 +320,13 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 		journal->j_chkpt_bhs[batch_count++] = bh;
 		__buffer_relink_io(jh);
 		transaction->t_chp_stats.cs_written++;
+		smp_rmb();
 		if ((batch_count == JBD2_NR_BATCH) ||
-		    need_resched() ||
+		    need_resched() || (transaction->t_checkpoint_list && g_bh == jh2bh(transaction->t_checkpoint_list)) ||
 		    spin_needbreak(&journal->j_list_lock))
 			goto unlock_and_flush;
 	}
+	pr_err("commit %px done\n", transaction);
 
 	if (batch_count) {
 		unlock_and_flush:
@@ -318,6 +334,10 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 		retry:
 			if (batch_count)
 				__flush_batch(journal, &batch_count);
+			if (transaction->t_checkpoint_list && g_bh == jh2bh(transaction->t_checkpoint_list)) {
+				pr_err("Do access to bh \n");
+				msleep(2000);
+			}
 			spin_lock(&journal->j_list_lock);
 			goto restart;
 	}
@@ -357,6 +377,8 @@ int jbd2_log_do_checkpoint(journal_t *journal)
 out:
 	spin_unlock(&journal->j_list_lock);
 	result = jbd2_cleanup_journal_tail(journal);
+	if (need_shutdown)
+		panic("DONE");
 
 	return (result < 0) ? result : 0;
 }
diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
index fa24b407a9dc..38c752deae68 100644
--- a/fs/jbd2/commit.c
+++ b/fs/jbd2/commit.c
@@ -378,6 +378,7 @@ static void jbd2_block_tag_csum_set(journal_t *j, journal_block_tag_t *tag,
  * The primary function for committing a transaction to the log.  This
  * function is called by the journal thread to begin a complete commit.
  */
+extern struct buffer_head *g_bh;
 void jbd2_journal_commit_transaction(journal_t *journal)
 {
 	struct transaction_stats_s stats;
@@ -968,6 +969,8 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 	 * As there are other places (journal_unmap_buffer()) adding buffers
 	 * to this list we have to be careful and hold the j_list_lock.
 	 */
+	if (!strcmp(journal->j_sb_buffer->b_bdev->bd_disk->disk_name, "sda"))
+		pr_err("%s ---\n", __func__);
 	spin_lock(&journal->j_list_lock);
 	while (commit_transaction->t_forget) {
 		transaction_t *cp_transaction;
@@ -1066,6 +1069,9 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 		if (buffer_jbddirty(bh)) {
 			JBUFFER_TRACE(jh, "add to new checkpointing trans");
 			__jbd2_journal_insert_checkpoint(jh, commit_transaction);
+			smp_rmb();
+			if (g_bh == bh)
+				pr_err("insert %px into trans %px\n", g_bh, commit_transaction);
 			if (is_journal_aborted(journal))
 				clear_buffer_jbddirty(bh);
 		} else {
@@ -1193,6 +1199,8 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 	write_unlock(&journal->j_state_lock);
 	wake_up(&journal->j_wait_done_commit);
 	wake_up(&journal->j_fc_wait);
+	if (!strcmp(journal->j_sb_buffer->b_bdev->bd_disk->disk_name, "sda"))
+		pr_err("%s ====\n", __func__);
 
 	/*
 	 * Calculate overall stats
diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c
index e6f244d3d744..4c6e9ed2b9c9 100644
--- a/fs/jbd2/transaction.c
+++ b/fs/jbd2/transaction.c
@@ -224,12 +224,14 @@ static void sub_reserved_credits(journal_t *journal, int blocks)
  * transaction. Returns 1 if we had to wait, j_state_lock is dropped, and
  * caller must retry.
  */
+extern int g_bh_dirty;
 static int add_transaction_credits(journal_t *journal, int blocks,
 				   int rsv_blocks)
 {
 	transaction_t *t = journal->j_running_transaction;
 	int needed;
 	int total = blocks + rsv_blocks;
+	static int entered = 0;
 
 	/*
 	 * If the current transaction is locked down for commit, wait
@@ -284,13 +286,14 @@ static int add_transaction_credits(journal_t *journal, int blocks,
 	 * *before* starting to dirty potentially checkpointed buffers
 	 * in the new transaction.
 	 */
-	if (jbd2_log_space_left(journal) < journal->j_max_transaction_buffers) {
+	if (jbd2_log_space_left(journal) < journal->j_max_transaction_buffers || (!strcmp(current->comm, "chmod") && g_bh_dirty && !entered)) {
+		entered = 1;
 		atomic_sub(total, &t->t_outstanding_credits);
 		read_unlock(&journal->j_state_lock);
 		jbd2_might_wait_for_commit(journal);
 		write_lock(&journal->j_state_lock);
 		if (jbd2_log_space_left(journal) <
-					journal->j_max_transaction_buffers)
+					journal->j_max_transaction_buffers || (!strcmp(current->comm, "chmod") && g_bh_dirty))
 			__jbd2_log_wait_for_space(journal);
 		write_unlock(&journal->j_state_lock);
 		return 1;
@@ -940,6 +943,9 @@ static void jbd2_freeze_jh_data(struct journal_head *jh)
  * part of the transaction, that is).
  *
  */
+#include <linux/delay.h>
+extern struct buffer_head *g_bh;
+
 static int
 do_get_write_access(handle_t *handle, struct journal_head *jh,
 			int force_copy)
@@ -1008,9 +1014,18 @@ do_get_write_access(handle_t *handle, struct journal_head *jh,
 		JBUFFER_TRACE(jh, "Journalling dirty buffer");
 		clear_buffer_dirty(bh);
 		set_buffer_jbddirty(bh);
+		smp_rmb();
+		if (g_bh_dirty && g_bh == bh)
+			pr_err("%s: clear bh dirty\n", __func__);
 	}
 
 	unlock_buffer(bh);
+	smp_rmb();
+	if (g_bh_dirty && g_bh == bh) {
+		pr_err("%s: wait checkpoint remove\n", __func__);
+		mdelay(5000);
+		pr_err("%s: wait checkpoint remove done\n", __func__);
+	}
 
 	error = -EROFS;
 	if (is_handle_aborted(handle)) {
@@ -1045,6 +1060,7 @@ do_get_write_access(handle_t *handle, struct journal_head *jh,
 		JBUFFER_TRACE(jh, "no transaction");
 		J_ASSERT_JH(jh, !jh->b_next_transaction);
 		JBUFFER_TRACE(jh, "file as BJ_Reserved");
+
 		/*
 		 * Make sure all stores to jh (b_modified, b_frozen_data) are
 		 * visible before attaching it to the running transaction.
@@ -1285,7 +1301,6 @@ int jbd2_journal_get_create_access(handle_t *handle, struct buffer_head *bh)
 		clear_buffer_dirty(jh2bh(jh));
 		/* first access by this transaction */
 		jh->b_modified = 0;
-
 		JBUFFER_TRACE(jh, "file as BJ_Reserved");
 		spin_lock(&journal->j_list_lock);
 		__jbd2_journal_file_buffer(jh, transaction, BJ_Reserved);
@@ -2036,8 +2051,14 @@ static void __jbd2_journal_temp_unlink_buffer(struct journal_head *jh)
 	jh->b_jlist = BJ_None;
 	if (transaction && is_journal_aborted(transaction->t_journal))
 		clear_buffer_jbddirty(bh);
-	else if (test_clear_buffer_jbddirty(bh))
+	else if (test_clear_buffer_jbddirty(bh)) {
 		mark_buffer_dirty(bh);	/* Expose it to the VM */
+		smp_rmb();
+		if (g_bh == bh) {
+			pr_err("mark buffer dirty\n");
+			g_bh_dirty = 1;
+		}
+	}
 }
 
 /*
```
a.c
```
#define _GNU_SOURCE             /* See feature_test_macros(7) */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mount.h>
#include <getopt.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/xattr.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <errno.h>
#include <string.h>
#include <linux/reboot.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <grp.h>
#include <sys/prctl.h>
#include <linux/fs.h>
#include <signal.h>
#include <sched.h>
#include <sys/wait.h>
#include <sys/mman.h>
#include <linux/perf_event.h>
#include <linux/bpf.h>
#include <sys/sysinfo.h>
#include <sys/epoll.h>
#include <asm/unistd.h>
#include <sys/time.h>
#include <sys/sendfile.h>
#include <stdarg.h>
#include <assert.h>
#include <linux/aio_abi.h>

int main(void)
{
	__u64 sz = 1048576;
	int ret, fd = open("/root/temp", O_RDONLY);
	if (fd < 0) {
		perror("open");
		return -1;
	}
	int new_fd = open("/root/temp/a", O_CREAT | O_RDWR, 0666);
	fsync(new_fd);   // trans commit and mark bh dirty

ret = fork();
	if (ret < 0) {
		perror("fork");
		return -1;
	} else if (ret == 0) {
		system("chmod +x /root/temp/a");
		return 0;
	} else {
		/*printf("wait \'Do access to bh\'\n");
		getchar();
		printf("do_get_write_access\n");*/
		sleep(1);
		creat("/root/temp/b", 0666);  // clear bh dirty
	}
	return 0;
}
```
test.sh
```
#!/bin/bash

gcc -o aa a.c

umount /root/temp
mkfs.ext4 -F -E lazy_itable_init=0 -b 4K /dev/sda
mount /dev/sda /root/temp
./aa
```

src-openEuler/kernel

内容风险标识

评论 (1)

src-openEuler/kernel .gitee-modal { width: 500px !important; }

内容风险标识