<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 TRANSITIONAL//EN">
<HTML>
<HEAD>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; CHARSET=UTF-8">
<META NAME="GENERATOR" CONTENT="GtkHTML/3.16.3">
</HEAD>
<BODY>
On Mon, 2015-08-10 at 22:22 +0530, Atin Mukherjee wrote:<BR>
[snip]<BR>
<BLOCKQUOTE TYPE=CITE>
<FONT COLOR="#000000">strace output claims the command exited successfully. Are you sure ls got hung?</FONT><BR>
</BLOCKQUOTE>
<BR>
Not sure, but this one definitely hung. 'mkdir("test", 0777' was the last output, and it's been stuck here for about 7 minutes now:<BR>
<BR>
[root@voicemail1b-1 14391.broken]# strace mkdir test<BR>
execve("/usr/bin/mkdir", ["mkdir", "test"], [/* 27 vars */]) = 0<BR>
brk(0) = 0x8db000<BR>
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3468a89000<BR>
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)<BR>
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3<BR>
fstat(3, {st_mode=S_IFREG|0644, st_size=31874, ...}) = 0<BR>
mmap(NULL, 31874, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3468a81000<BR>
close(3) = 0<BR>
open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240d\0\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=147120, ...}) = 0<BR>
mmap(NULL, 2246784, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3468644000<BR>
mprotect(0x7f3468665000, 2097152, PROT_NONE) = 0<BR>
mmap(0x7f3468865000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x7f3468865000<BR>
mmap(0x7f3468867000, 6272, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3468867000<BR>
close(3) = 0<BR>
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\34\2\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=2107760, ...}) = 0<BR>
mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3468283000<BR>
mprotect(0x7f3468439000, 2097152, PROT_NONE) = 0<BR>
mmap(0x7f3468639000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7f3468639000<BR>
mmap(0x7f346863f000, 16960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f346863f000<BR>
close(3) = 0<BR>
open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=398272, ...}) = 0<BR>
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3468a80000<BR>
mmap(NULL, 2490888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3468022000<BR>
mprotect(0x7f3468081000, 2097152, PROT_NONE) = 0<BR>
mmap(0x7f3468281000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5f000) = 0x7f3468281000<BR>
close(3) = 0<BR>
open("/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000/\0\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=153184, ...}) = 0<BR>
mmap(NULL, 2245240, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3467dfd000<BR>
mprotect(0x7f3467e21000, 2093056, PROT_NONE) = 0<BR>
mmap(0x7f3468020000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f3468020000<BR>
close(3) = 0<BR>
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=19512, ...}) = 0<BR>
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3467bf9000<BR>
mprotect(0x7f3467bfc000, 2093056, PROT_NONE) = 0<BR>
mmap(0x7f3467dfb000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3467dfb000<BR>
close(3) = 0<BR>
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3<BR>
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"..., 832) = 832<BR>
fstat(3, {st_mode=S_IFREG|0755, st_size=141616, ...}) = 0<BR>
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3468a7f000<BR>
mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f34679dd000<BR>
mprotect(0x7f34679f3000, 2097152, PROT_NONE) = 0<BR>
mmap(0x7f3467bf3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3467bf3000<BR>
mmap(0x7f3467bf5000, 13408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3467bf5000<BR>
close(3) = 0<BR>
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3468a7e000<BR>
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3468a7c000<BR>
arch_prctl(ARCH_SET_FS, 0x7f3468a7c800) = 0<BR>
mprotect(0x7f3468639000, 16384, PROT_READ) = 0<BR>
mprotect(0x7f3467bf3000, 4096, PROT_READ) = 0<BR>
mprotect(0x7f3467dfb000, 4096, PROT_READ) = 0<BR>
mprotect(0x7f3468020000, 4096, PROT_READ) = 0<BR>
mprotect(0x7f3468281000, 4096, PROT_READ) = 0<BR>
mprotect(0x7f3468865000, 4096, PROT_READ) = 0<BR>
mprotect(0x611000, 4096, PROT_READ) = 0<BR>
mprotect(0x7f3468a8a000, 4096, PROT_READ) = 0<BR>
munmap(0x7f3468a81000, 31874) = 0<BR>
set_tid_address(0x7f3468a7cad0) = 24942<BR>
set_robust_list(0x7f3468a7cae0, 24) = 0<BR>
rt_sigaction(SIGRTMIN, {0x7f34679e3780, [], SA_RESTORER|SA_SIGINFO, 0x7f34679ec130}, NULL, 8) = 0<BR>
rt_sigaction(SIGRT_1, {0x7f34679e3810, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f34679ec130}, NULL, 8) = 0<BR>
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0<BR>
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0<BR>
statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0<BR>
statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0<BR>
stat("/sys/fs/selinux", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0<BR>
brk(0) = 0x8db000<BR>
brk(0x8fc000) = 0x8fc000<BR>
mkdir("test", 0777 <BR>
<BR>
<BLOCKQUOTE TYPE=CITE>
<FONT COLOR="#000000">></FONT><BR>
<FONT COLOR="#000000">></FONT><BR>
<FONT COLOR="#000000">></FONT><BR>
<FONT COLOR="#000000">></FONT><BR>
<FONT COLOR="#000000">>> ></FONT><BR>
<FONT COLOR="#000000">>> > Then ... do I need to run something on one of the bricks while strace is</FONT><BR>
<FONT COLOR="#000000">>> > running?</FONT><BR>
<FONT COLOR="#000000">>> ></FONT><BR>
<FONT COLOR="#000000">>> > Cheers,</FONT><BR>
<FONT COLOR="#000000">>> > Kingsley.</FONT><BR>
<FONT COLOR="#000000">>> ></FONT><BR>
<FONT COLOR="#000000">>> ></FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > [root@gluster1b-1 ~]# gluster volume heal callrec info</FONT><BR>
<FONT COLOR="#000000">>> > > > Brick gluster1a-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:164f888f-2049-49e6-ad26-c758ee091863></FONT><BR>
<FONT COLOR="#000000">>> > > > /recordings/834723/14391 - Possibly undergoing heal</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:e280b40c-d8b7-43c5-9da7-4737054d7a7f></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:b1fbda4a-732f-4f5d-b5a1-8355d786073e></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:edb74524-b4b7-4190-85e7-4aad002f6e7c></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:9b8b8446-1e27-4113-93c2-6727b1f457eb></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:650efeca-b45c-413b-acc3-f0a5853ccebd></FONT><BR>
<FONT COLOR="#000000">>> > > > Number of entries: 7</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > Brick gluster1b-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > Number of entries: 0</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > Brick gluster2a-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:e280b40c-d8b7-43c5-9da7-4737054d7a7f></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:164f888f-2049-49e6-ad26-c758ee091863></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:650efeca-b45c-413b-acc3-f0a5853ccebd></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:b1fbda4a-732f-4f5d-b5a1-8355d786073e></FONT><BR>
<FONT COLOR="#000000">>> > > > /recordings/834723/14391 - Possibly undergoing heal</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:edb74524-b4b7-4190-85e7-4aad002f6e7c></FONT><BR>
<FONT COLOR="#000000">>> > > > <gfid:9b8b8446-1e27-4113-93c2-6727b1f457eb></FONT><BR>
<FONT COLOR="#000000">>> > > > Number of entries: 7</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > Brick gluster2b-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > Number of entries: 0</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > If I query each brick directly for the number of files/directories</FONT><BR>
<FONT COLOR="#000000">>> > > > within that, I get 1731 on gluster1a-1 and gluster2a-1, but 1737 on</FONT><BR>
<FONT COLOR="#000000">>> > > the</FONT><BR>
<FONT COLOR="#000000">>> > > > other two, using this command:</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > # find /data/brick/callrec/recordings/834723/14391 -print | wc -l</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > Cheers,</FONT><BR>
<FONT COLOR="#000000">>> > > > Kingsley.</FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > On Mon, 2015-08-10 at 11:05 +0100, Kingsley wrote:</FONT><BR>
<FONT COLOR="#000000">>> > > > > Sorry for the blind panic - restarting the volume seems to have</FONT><BR>
<FONT COLOR="#000000">>> > > fixed</FONT><BR>
<FONT COLOR="#000000">>> > > > > it.</FONT><BR>
<FONT COLOR="#000000">>> > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > But then my next question - why is this necessary? Surely it</FONT><BR>
<FONT COLOR="#000000">>> > > undermines</FONT><BR>
<FONT COLOR="#000000">>> > > > > the whole point of a high availability system?</FONT><BR>
<FONT COLOR="#000000">>> > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > Cheers,</FONT><BR>
<FONT COLOR="#000000">>> > > > > Kingsley.</FONT><BR>
<FONT COLOR="#000000">>> > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > On Mon, 2015-08-10 at 10:53 +0100, Kingsley wrote:</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Hi,</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > We have a 4 way replicated volume using gluster 3.6.3 on CentOS</FONT><BR>
<FONT COLOR="#000000">>> > > 7.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Over the weekend I did a yum update on each of the bricks in</FONT><BR>
<FONT COLOR="#000000">>> > > turn, but</FONT><BR>
<FONT COLOR="#000000">>> > > > > > now when clients (using fuse mounts) try to access the volume,</FONT><BR>
<FONT COLOR="#000000">>> > > it hangs.</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Gluster itself wasn't updated (we've disabled that repo so that</FONT><BR>
<FONT COLOR="#000000">>> > > we keep</FONT><BR>
<FONT COLOR="#000000">>> > > > > > to 3.6.3 for now).</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > This was what I did:</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > * on first brick, "yum update"</FONT><BR>
<FONT COLOR="#000000">>> > > > > > * reboot brick</FONT><BR>
<FONT COLOR="#000000">>> > > > > > * watch "gluster volume status" on another brick and wait</FONT><BR>
<FONT COLOR="#000000">>> > > for it</FONT><BR>
<FONT COLOR="#000000">>> > > > > > to say all 4 bricks are online before proceeding to</FONT><BR>
<FONT COLOR="#000000">>> > > update the</FONT><BR>
<FONT COLOR="#000000">>> > > > > > next brick</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > I was expecting the clients might pause 30 seconds while they</FONT><BR>
<FONT COLOR="#000000">>> > > notice a</FONT><BR>
<FONT COLOR="#000000">>> > > > > > brick is offline, but then recover.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > I've tried re-mounting clients, but that hasn't helped.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > I can't see much data in any of the log files.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > I've tried "gluster volume heal callrec" but it doesn't seem to</FONT><BR>
<FONT COLOR="#000000">>> > > have</FONT><BR>
<FONT COLOR="#000000">>> > > > > > helped.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > What shall I do next?</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > I've pasted some stuff below in case any of it helps.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Cheers,</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Kingsley.</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]# gluster volume info callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Volume Name: callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Type: Replicate</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Volume ID: a39830b7-eddb-4061-b381-39411274131a</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Status: Started</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Number of Bricks: 1 x 4 = 4</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Transport-type: tcp</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Bricks:</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick1: gluster1a-1:/data/brick/callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick2: gluster1b-1:/data/brick/callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick3: gluster2a-1:/data/brick/callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick4: gluster2b-1:/data/brick/callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Options Reconfigured:</FONT><BR>
<FONT COLOR="#000000">>> > > > > > performance.flush-behind: off</FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]#</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]# gluster volume status callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Status of volume: callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Gluster process Port</FONT><BR>
<FONT COLOR="#000000">>> > > Online Pid</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > ------------------------------------------------------------------------------</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster1a-1:/data/brick/callrec 49153</FONT><BR>
<FONT COLOR="#000000">>> > > Y 6803</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster1b-1:/data/brick/callrec 49153</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2614</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster2a-1:/data/brick/callrec 49153</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2645</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster2b-1:/data/brick/callrec 49153</FONT><BR>
<FONT COLOR="#000000">>> > > Y 4325</FONT><BR>
<FONT COLOR="#000000">>> > > > > > NFS Server on localhost 2049</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2769</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Self-heal Daemon on localhost N/A</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2789</FONT><BR>
<FONT COLOR="#000000">>> > > > > > NFS Server on gluster2a-1 2049</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2857</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Self-heal Daemon on gluster2a-1 N/A</FONT><BR>
<FONT COLOR="#000000">>> > > Y 2814</FONT><BR>
<FONT COLOR="#000000">>> > > > > > NFS Server on 88.151.41.100 2049</FONT><BR>
<FONT COLOR="#000000">>> > > Y 6833</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Self-heal Daemon on 88.151.41.100 N/A</FONT><BR>
<FONT COLOR="#000000">>> > > Y 6824</FONT><BR>
<FONT COLOR="#000000">>> > > > > > NFS Server on gluster2b-1 2049</FONT><BR>
<FONT COLOR="#000000">>> > > Y 4428</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Self-heal Daemon on gluster2b-1 N/A</FONT><BR>
<FONT COLOR="#000000">>> > > Y 4387</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Task Status of Volume callrec</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > ------------------------------------------------------------------------------</FONT><BR>
<FONT COLOR="#000000">>> > > > > > There are no active volume tasks</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]#</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]# gluster volume heal callrec info</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster1a-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > > > /to_process - Possibly undergoing heal</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Number of entries: 1</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster1b-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Number of entries: 0</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster2a-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > > > /to_process - Possibly undergoing heal</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Number of entries: 1</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > Brick gluster2b-1.dns99.co.uk:/data/brick/callrec/</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Number of entries: 0</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > [root@gluster1b-1 ~]#</FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > > _______________________________________________</FONT><BR>
<FONT COLOR="#000000">>> > > > > > Gluster-users mailing list</FONT><BR>
<FONT COLOR="#000000">>> > > > > > <A HREF="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</A></FONT><BR>
<FONT COLOR="#000000">>> > > > > > <A HREF="http://www.gluster.org/mailman/listinfo/gluster-users">http://www.gluster.org/mailman/listinfo/gluster-users</A></FONT><BR>
<FONT COLOR="#000000">>> > > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > > _______________________________________________</FONT><BR>
<FONT COLOR="#000000">>> > > > > Gluster-users mailing list</FONT><BR>
<FONT COLOR="#000000">>> > > > > <A HREF="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</A></FONT><BR>
<FONT COLOR="#000000">>> > > > > <A HREF="http://www.gluster.org/mailman/listinfo/gluster-users">http://www.gluster.org/mailman/listinfo/gluster-users</A></FONT><BR>
<FONT COLOR="#000000">>> > > > ></FONT><BR>
<FONT COLOR="#000000">>> > > ></FONT><BR>
<FONT COLOR="#000000">>> > > > _______________________________________________</FONT><BR>
<FONT COLOR="#000000">>> > > > Gluster-users mailing list</FONT><BR>
<FONT COLOR="#000000">>> > > > <A HREF="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</A></FONT><BR>
<FONT COLOR="#000000">>> > > > <A HREF="http://www.gluster.org/mailman/listinfo/gluster-users">http://www.gluster.org/mailman/listinfo/gluster-users</A></FONT><BR>
<FONT COLOR="#000000">>> > ></FONT><BR>
<FONT COLOR="#000000">>> > ></FONT><BR>
<FONT COLOR="#000000">>> ></FONT><BR>
<FONT COLOR="#000000">>></FONT><BR>
<FONT COLOR="#000000">>></FONT><BR>
<BR>
<BR>
<PRE>
<FONT COLOR="#000000">_______________________________________________</FONT>
<FONT COLOR="#000000">Gluster-users mailing list</FONT>
<FONT COLOR="#000000"><A HREF="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</A></FONT>
<FONT COLOR="#000000"><A HREF="http://www.gluster.org/mailman/listinfo/gluster-users">http://www.gluster.org/mailman/listinfo/gluster-users</A></FONT>
</PRE>
</BLOCKQUOTE>
</BODY>
</HTML>