kgdb

--0313--

ken 不好意思 请教一下 之前你有交过我 在没有coredump的情况下 可以用gdb直接载入ko.debug, 然后dis -l register_bcache, 我记得使用方式是不是这样, 不过我现在dis -l出现错误^^"
[root@k22 ~]# gdb  /usr/lib/debug/lib/modules/4.18.0-147.5.1.el8_1.5es.1.x86_64/kernel/drivers/md/bcache/escache.ko.debug
...
(gdb) dis -l register_bcache
negative value
ken.kuo profile image
郭冠禹
4:15 PM
disass register_bcache
kc.chuang profile image
庄凯杰
4:17 PM
那如果我要看 bch_cached_dev_store+0x23/0xb0 [escache] 这个function的哪一行呢?
[root@node-1 ~]# cat /proc/8217/stack
[<0>] bch_cached_dev_store+0x23/0xb0 [escache]
[<0>] kernfs_fop_write+0x116/0x190
[<0>] vfs_write+0xa5/0x1a0
[<0>] ksys_write+0x4f/0xb0
[<0>] do_syscall_64+0x5b/0x1b0
[<0>] entry_SYSCALL_64_after_hwframe+0x65/0xca
[<0>] 0xffffffffffffffff
New Messages
ken.kuo profile image
郭冠禹
4:18 PM
l *(bch_cached_dev_store+0x23)


--
[  257.059789] BUG: unable to handle kernel NULL pointer dereference at 00000000000009b0
[  257.059939] IP: [<ffffffffc0490484>] register_bcache+0x1164/0x1360 [escache]
[  257.060098] PGD 22d23b067 PUD 229358067 PMD 0
[  257.060174] Oops: 0000 [#1] SMP
[  257.060241] Modules linked in: veth nf_conntrack_netlink xt_statistic xt_nat xt_recent xt_conntrack ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables ip_set nfnetlink xt_addrtype ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_comment iptable_filter xt_mark iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ext4 mbcache jbd2 escache snd_hda_codec_generic snd_hda_intel ppdev snd_hda_codec snd_hda_core iosf_mbi crc32_pclmul pcspkr snd_hwdep ghash_clmulni_intel snd_seq snd_seq_device aesni_intel lrw gf128mul snd_pcm glue_helper ablk_helper cryptd sg snd_timer joydev virtio_rng snd soundcore parport_pc virtio_balloon i2c_piix4 parport br_netfilter bridge stp llc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic
[  257.061081]  ata_generic pata_acpi virtio_console virtio_net virtio_blk qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel floppy serio_raw i2c_core virtio_pci virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod
[  257.061438] CPU: 0 PID: 22740 Comm: bcache-register Not tainted 3.10.0-693.11.1.el7.es.12.x86_64 #1
[  257.061528] Hardware name: Red Hat KVM, BIOS 1.10.2-3.el7.centos.es.1 04/01/2014
[  257.061604] task: ffff8801b829ef20 ti: ffff88022d250000 task.ti: ffff88022d250000
[  257.061682] RIP: 0010:[<ffffffffc0490484>]  [<ffffffffc0490484>] register_bcache+0x1164/0x1360 [escache]
[  257.061789] RSP: 0018:ffff88022d253de8  EFLAGS: 00010202
[  257.061873] RAX: ffff88022ee20000 RBX: ffff880084254000 RCX: ffff8800843a0000
[  257.061960] RDX: ffff8800843a0000 RSI: ffff88022ee20000 RDI: 0000000000000001
[  257.062032] RBP: ffff88022d253e98 R08: 0000000000000000 R09: ffffffff81211a1d
[  257.062105] R10: ffff88023fc19cc0 R11: ffffea0002109400 R12: ffffffffc049f180
[  257.062177] R13: ffff88017f828d00 R14: ffffffff81718380 R15: 000000000000000a
[  257.062259] FS:  00007f89ff367740(0000) GS:ffff88023fc00000(0000) knlGS:0000000000000000
[  257.062345] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  257.062412] CR2: 00000000000009b0 CR3: 0000000229b2b000 CR4: 00000000003406f0
[  257.062510] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  257.062600] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  257.062676] Stack:
[  257.062722]  0000000200000000 ffff88023ffd8000 ffff88023ffd7000 0000000000000000
[  257.062813]  0000000000000000 00000000975430b0 ffffffff85e579c0 00000000000080d0
[  257.062907]  000000000000000a 0000000000000000 ffff88007dec51d0 fffffffffffffff0
[  257.062991] Call Trace:
[  257.063075]  [<ffffffff811d1098>] ? alloc_pages_current+0x98/0x110
[  257.063162]  [<ffffffff81324bff>] kobj_attr_store+0xf/0x20
[  257.063258]  [<ffffffff81280046>] sysfs_write_file+0xc6/0x140
[  257.063333]  [<ffffffff81200a9d>] vfs_write+0xbd/0x1e0
[  257.063404]  [<ffffffff812018af>] SyS_write+0x7f/0xe0
[  257.063484]  [<ffffffff816b50c9>] system_call_fastpath+0x16/0x1b
[  257.063570] Code: 4a d0 48 83 e8 30 48 81 fa b0 f1 49 c0 0f 84 59 01 00 00 48 89 c6 48 89 ca 0f b7 ba 34 04 00 00 4c 8b 82 40 0c 00 00 85 ff 74 3a <4d> 3b a8 b0 09 00 00 0f 84 0c 01 00 00 44 8d 47 ff 31 ff 49 c1
[  257.063927] RIP  [<ffffffffc0490484>] register_bcache+0x1164/0x1360 [escache]
[  257.064006]  RSP <ffff88022d253de8>
[  257.064057] CR2: 00000000000009b0


起动crash
crash /usr/lib/debug/lib/modules/3.10.0-693.11.1.el7.es.12.x86_64/vmlinux /var/crash/127.0.0.1-2019-03-05-18\:07\:19/vmcore
bt 印出出问题时的backtrace
如果要在disassemble的是module，印出对应的c code是那一行，要先load module进来
mod -s escache
disassemble某个address，最后面是要disassemble多少行
dis -l  ffffffffc0490484 1

disassemble某个function
dis -l register_bcache
crash> dis -l ffffffffc0490484
/usr/src/debug/kernel-3.10.0-693.11.1.el7/linux-3.10.0-693.11.1.el7.es.12.x86_64/drivers/md/bcache/super.c: 1989
0xffffffffc0490484 <register_bcache+4452>:      cmp    0x9b0(%r8),%r13
4452=0x1164，所以可以透過dis -l register_bcache, 然後搜尋4452就可以找到上面這行
或者透過dump"IP: [<ffffffffc0490484>] register_bcache+0x1164/0x1360 [escache]"這行也直接告訴你是死在ffffffffc0490484，所以可以直接dis -l ffffffffc0490484

要看structure的define可以用struct，加上-o可以看到member的offset，struct后面加上address，可以直接把address的内容套到structure上
struct -o cache

kernel RIP location map to c code

1. rpm2cpio kernel-debuginfo-3.10.0-693.11.1.el7.es.9.x86_64.rpm | cpio -idm
[kevin@se129 kernel-debuginfo]$ rpm2cpio kernel-debuginfo-3.10.0-693.11.1.el7.es.9.x86_64.rpm | cpio -idm

[kevin@se129 kernel-debuginfo]$ cd usr/lib/debug/lib/modules/3.10.0-693.11.1.el7.es.9.x86_64/kernel/drivers/md/bcache/

[kevin@se129 bcache]$ gdb escache.ko.debug

(gdb) l *(read_dirty+0x3e5)

0x23ef5 is in read_dirty (drivers/md/bcache/writeback.c:235).

230     in drivers/md/bcache/writeback.c

drivers/md/bcache/writeback.c

235         BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));

比較怪的是，從gdb看到的read_dirty addr跟objdump看到的不一樣：

(gdb) p read_dirty

$1 = {void (struct cached_dev *)} 0x23b10 <read_dirty>

objdump -d escache.ko  |  grep  "\<read_dirty\>"

0000000000023ae0 <read_dirty>:

   23ae0:       e8 00 00 00 00          callq  23ae5 <read_dirty+0x5>

   23ec5:       0f 0b                   ud2

一開始看0x23ae0+0x3e5=0x23ec5的位置"ud2"看不出來發生什麼事，後來才直接用gdb l*(read_dirty+0x3e5)才找到writeback:235.

strace

strace -v -s 1000 -f -e ioctl -P /dev/mapper/control -p <docker>
strace -v -f -e ioctl -s -p <docker>

rpm query dependencies

repoquery --tree-requires bash-4.2.46-30.el7.x86_64

git send-email

git send-email --annotate --compose --to=kcchuang@cs.nctu.edu.tw 0001-bcache-clean-plugout-ed-backing-dev.patch

這樣會把commit額外加在0001-bcache-clean-plugout-ed-backing-dev.patch，而且還不能attach file.

C use O_DIRECT vs O_SYNC => blktrace

1. O_DIRECT
fd = open("/dev/escache0", O_RDWR | O_DIRECT);

blktrace -d /dev/escache0 -o - | blkparse -i -

252,0    4        1     0.000000000 30655  Q  WS 0 + 1 [openescache]

2. O_SYNC

252,0    6        1   129.714324326 30666  Q  WS 0 + 8 [openescache]

252,0    6        2   129.725375608 30666  Q FWS [openescache]

F means 'flush'

openescache.c

#define _GNU_SOURCE

#include <string.h>

#include <stdlib.h>

#include <fcntl.h>

#include <stdio.h>

#define BLOCKSIZE 512

int main() {

    int fd, i;

    void *buffer;

    char image[BLOCKSIZE];

    for(i=0; i<BLOCKSIZE; i++)

        image[i] = 0xa2;

    posix_memalign(&buffer, BLOCKSIZE, BLOCKSIZE);

    memcpy(buffer, image, sizeof(image));

    fd = open("/dev/escache0", O_RDWR | O_SYNC);

    if (fd == -1) {

        printf("open failed");

        return -1;

    }

    write(fd, buffer, BLOCKSIZE);

    close(fd);

    free(buffer);

    return 0;

}

shell script learning

--20180411--
$1:/dev/sdb

for i in $(ls /dev/escache*)
do
        find /sys/block/`echo $i|sed 's/\/dev\///g'`/slaves -name `echo $1|sed 's/\/dev\///g'`
        if [ $? -eq 0 ] ;then
                cache=$(ls /sys/block/`echo $i|sed 's/\/dev\///g'`/slaves |grep -v `echo $1|sed 's/\/dev\///g'`)
                /usr/sbin/bcache-super-show /dev/$cache | grep "cache device"
                if [ $? -eq 0 ] ;then
                        echo 1 > /sys/block/$cache/escache/set/stop
                fi
        fi
done

-- Test --

[' is actually a program, just like ls and other programs, so it must be surrounded by spaces: