<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
        {font-family:宋体;
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:"Cambria Math";
        panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
        {font-family:等线;
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:"Arial Unicode MS";
        panose-1:2 11 6 4 2 2 2 2 2 4;}
@font-face
        {font-family:"Calibri Light";
        panose-1:2 15 3 2 2 2 4 3 2 4;}
@font-face
        {font-family:Calibri;
        panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
        {font-family:微软雅黑;
        panose-1:2 11 5 3 2 2 4 2 2 4;}
@font-face
        {font-family:"\@微软雅黑";
        panose-1:2 11 5 3 2 2 4 2 2 4;}
@font-face
        {font-family:"\@宋体";
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:Tahoma;
        panose-1:2 11 6 4 3 5 4 4 2 4;}
@font-face
        {font-family:"\@Arial Unicode MS";
        panose-1:2 11 6 4 2 2 2 2 2 4;}
@font-face
        {font-family:"\@等线";
        panose-1:2 1 6 0 3 1 1 1 1 1;}
@font-face
        {font-family:Consolas;
        panose-1:2 11 6 9 2 2 4 3 2 4;}
@font-face
        {font-family:����;}
@font-face
        {font-family:΢���ź�;}
@font-face
        {font-family:"Times New Roman \,serif";
        panose-1:0 0 0 0 0 0 0 0 0 0;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
        {margin:0in;
        margin-bottom:.0001pt;
        font-size:11.0pt;
        font-family:"Calibri",sans-serif;
        color:black;}
h1
        {mso-style-priority:9;
        mso-style-link:"标题 1 字符";
        mso-margin-top-alt:auto;
        margin-right:0in;
        mso-margin-bottom-alt:auto;
        margin-left:0in;
        font-size:24.0pt;
        font-family:"Times New Roman",serif;
        color:black;}
a:link, span.MsoHyperlink
        {mso-style-priority:99;
        color:#0563C1;
        text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
        {mso-style-priority:99;
        color:#954F72;
        text-decoration:underline;}
pre
        {mso-style-priority:99;
        mso-style-link:"HTML 预设格式 字符";
        margin:0in;
        margin-bottom:.0001pt;
        font-size:10.0pt;
        font-family:"Courier New";
        color:black;}
p.msonormal0, li.msonormal0, div.msonormal0
        {mso-style-name:msonormal;
        mso-margin-top-alt:auto;
        margin-right:0in;
        mso-margin-bottom-alt:auto;
        margin-left:0in;
        font-size:12.0pt;
        font-family:����;
        color:black;}
span.1
        {mso-style-name:"标题 1 字符";
        mso-style-priority:9;
        mso-style-link:"标题 1";
        font-family:"Calibri Light",sans-serif;
        color:#2E74B5;}
span.HTML
        {mso-style-name:"HTML 预设格式 字符";
        mso-style-priority:99;
        mso-style-link:"HTML 预设格式";
        font-family:Consolas;
        color:black;}
span.10
        {mso-style-name:"���� 1 �ַ�";
        mso-style-priority:9;
        mso-style-link:"���� 1";
        font-family:"Times New Roman",serif;
        font-weight:bold;}
p.11, li.11, div.11
        {mso-style-name:"���� 1";
        mso-style-link:"���� 1 �ַ�";
        margin:0in;
        margin-bottom:.0001pt;
        font-size:11.0pt;
        font-family:"Calibri",sans-serif;
        color:black;}
span.EmailStyle23
        {mso-style-type:personal;
        font-family:"Calibri",sans-serif;
        color:windowtext;}
span.EmailStyle24
        {mso-style-type:personal;
        font-family:"Calibri",sans-serif;
        color:#1F497D;}
span.EmailStyle25
        {mso-style-type:personal;
        font-family:"Calibri",sans-serif;
        color:#1F497D;}
span.EmailStyle26
        {mso-style-type:personal-reply;
        font-family:"Calibri",sans-serif;
        color:#1F497D;}
.MsoChpDefault
        {mso-style-type:export-only;
        font-size:10.0pt;}
@page WordSection1
        {size:8.5in 11.0in;
        margin:1.0in 1.25in 1.0in 1.25in;}
div.WordSection1
        {page:WordSection1;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body bgcolor="white" lang="EN-US" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span style="color:#1F497D">I have checked all the nodes both on masters and slaves, the software is the same.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">I am puzzled why there were half masters work, halt faulty.&nbsp; &nbsp;<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">[admin@SVR6996HW2285 ~]$ rpm -qa |grep gluster<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-api-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-fuse-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-geo-replication-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-cli-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-server-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D">glusterfs-libs-3.6.3-1.el6.x86_64<o:p></o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<div>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.0pt;font-family:&quot;微软雅黑&quot;,sans-serif;color:#6E6E6F">Best Regards
</span><b><span style="font-size:10.5pt;font-family:&quot;Arial Unicode MS&quot;,sans-serif;color:#1F497D"><o:p></o:p></span></b></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="527" style="width:394.9pt">
<tbody>
<tr style="height:19.4pt">
<td width="527" valign="top" style="width:394.9pt;padding:0in 0in 0in 0in;height:19.4pt">
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><b><span lang="ZH-CN" style="font-size:10.0pt;font-family:等线;color:#6E6E6F">杨雨阳</span></b><b><span style="font-size:10.0pt;font-family:宋体;color:#6E6E6F"> Yuyang Yang</span></b><b><span style="font-size:16.0pt;font-family:&quot;Tahoma&quot;,sans-serif;color:#6E6E6F"><o:p></o:p></span></b></p>
</td>
</tr>
<tr style="height:80.45pt">
<td width="527" valign="top" style="width:394.9pt;padding:0in 0in 0in 0in;height:80.45pt">
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph;line-height:12.0pt">
<span style="font-size:10.0pt;font-family:&quot;Tahoma&quot;,sans-serif;color:#6E6E6F">OPS <o:p>
</o:p></span></p>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph;line-height:12.0pt">
<span style="font-size:10.0pt;font-family:&quot;Tahoma&quot;,sans-serif;color:#6E6E6F">Ctrip Infrastructure Service (CIS)<o:p></o:p></span></p>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.0pt;font-family:&quot;Tahoma&quot;,sans-serif;color:#6E6E6F">Ctrip Computer Technology (Shanghai) Co., Ltd&nbsp;
<br>
Phone:&nbsp;&#43; 86 21 34064880-15554&nbsp;| Fax: &#43; 86 21 52514588-13389 <br>
Web:&nbsp;</span><span style="font-size:10.0pt;color:blue"><a href="http://www.ctrip.com/"><span style="font-family:&quot;Tahoma&quot;,sans-serif;color:blue">www.Ctrip.com</span></a></span><span style="font-size:9.0pt;font-family:&quot;Tahoma&quot;,sans-serif"><o:p></o:p></span></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.5pt;color:#1F497D"><o:p>&nbsp;</o:p></span></p>
</div>
<p class="MsoNormal"><span style="color:#1F497D"><o:p>&nbsp;</o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b><span lang="ZH-CN" style="font-family:等线;color:windowtext">发件人</span></b><b><span style="font-family:等线;color:windowtext">:</span></b><span style="font-family:等线;color:windowtext"> Saravanakumar Arumugam [mailto:sarumuga@redhat.com]
<br>
<b><span lang="ZH-CN">发送时间</span>:</b> Thursday, May 19, 2016 4:33 PM<br>
<b><span lang="ZH-CN">收件人</span>:</b> vyyy<span lang="ZH-CN">杨雨阳</span> &lt;yuyangyang@Ctrip.com&gt;; Gluster-users@gluster.org; Aravinda Vishwanathapura Krishna Murthy &lt;avishwan@redhat.com&gt;; Kotresh Hiremath Ravishankar &lt;khiremat@redhat.com&gt;<br>
<b><span lang="ZH-CN">主题</span>:</b> Re: <span lang="ZH-CN">答复</span>: [Gluster-users]
<span lang="ZH-CN">答复</span>: geo-replication status partial faulty<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p>&nbsp;</o:p></p>
<p class="MsoNormal" style="margin-bottom:12.0pt">Hi,<br>
&#43;geo-rep team.<br>
<br>
Can you get the gluster version you are using?<br>
<br>
# For example: <br>
rpm -qa | grep gluster <br>
<br>
I hope you have same gluster version installed everywhere.<br>
Please double check and share the same.<br>
<br>
Thanks,<br>
Saravana<span style="font-size:12.0pt"><o:p></o:p></span></p>
<div>
<p class="MsoNormal">On 05/19/2016 01:37 PM, vyyy<span lang="ZH-CN" style="font-family:&quot;微软雅黑&quot;,sans-serif">杨雨阳</span> wrote:<o:p></o:p></p>
</div>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<p class="MsoNormal"><span style="color:#1F497D">Hi,</span> Saravana<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">I have changed log level to DEBUG. Then start geo-replication with log-file option, attached the file.</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">&nbsp;</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">gluster volume geo-replication filews glusterfs01.sh3.ctripcorp.com::filews_slave start --log-file=geo.log</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">&nbsp;</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">I have checked </span>&nbsp;/root/.ssh/authorized_keys in glusterfs01.sh3.ctripcorp.com<span style="font-family:等线"> ,</span> It &nbsp;have entries in /var/lib/glusterd/geo-replication/common_secret.pem.pub<span style="font-family:等线">.&nbsp;
 and I have removed the lines not started with <span lang="ZH-CN">“</span>command=<span lang="ZH-CN">”</span>&nbsp;&nbsp;
</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-family:等线">&nbsp;</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:10.0pt;font-family:&quot;Courier New&quot;">ssh -i /var/lib/glusterd/geo-replication/secret.pem&nbsp; root@</span> glusterfs01.sh3.ctripcorp.com<o:p></o:p></p>
<p class="MsoNormal">I can see gsyncd messages and no ssh error.<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">Attached etc-glusterfs-glusterd.vol.log from faulty node, it shows :<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:23.405974] I [glusterd-geo-rep.c:3516:glusterd_read_status_file] 0-: Using passed config template(/var/lib/glusterd/geo-replication/filews_glusterfs01.sh3.ctripcorp.com_filews_slave/gsyncd.conf).<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:23.541169] E [glusterd-geo-rep.c:3200:glusterd_gsync_read_frm_status] 0-: Unable to read gsyncd status file<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:23.541210] E [glusterd-geo-rep.c:3603:glusterd_read_status_file] 0-: Unable to read the statusfile for /export/sdb/filews brick for&nbsp; filews(master), glusterfs01.sh3.ctripcorp.com::filews_slave(slave) session<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:29.472047] I [glusterd-geo-rep.c:1835:glusterd_get_statefile_name] 0-: Using passed config template(/var/lib/glusterd/geo-replication/filews_glusterfs01.sh3.ctripcorp.com_filews_slave/gsyncd.conf).<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:34.939709] I [glusterd-geo-rep.c:3516:glusterd_read_status_file] 0-: Using passed config template(/var/lib/glusterd/geo-replication/filews_glusterfs01.sh3.ctripcorp.com_filews_slave/gsyncd.conf).<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 06:39:35.058520] E [glusterd-geo-rep.c:3200:glusterd_gsync_read_frm_status] 0-: Unable to read gsyncd status file<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">/var/log/glusterfs/geo-replication/filews/ ssh%3A%2F%2Froot%4010.15.65.66%3Agluster%3A%2F%2F127.0.0.1%3Afilews_slave.log &nbsp;shows as following:<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.307755] I [monitor(monitor):215:monitor] Monitor: ------------------------------------------------------------<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.308059] I [monitor(monitor):216:monitor] Monitor: starting gsyncd worker<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.423320] D [gsyncd(agent):627:main_i] &lt;top&gt;: rpc_fd: '7,11,10,9'<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.423882] I [changelogagent(agent):72:__init__] ChangelogAgent: Agent listining...<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.423906] I [monitor(monitor):267:monitor] Monitor: worker(/export/sdb/filews) died before establishing connection<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.424151] I [repce(agent):92:service_loop] RepceServer: terminating on reaching EOF.<o:p></o:p></p>
<p class="MsoNormal">[2016-05-19 15:11:37.424335] I [syncdutils(agent):214:finalize] &lt;top&gt;: exiting.<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal"><span style="color:#1F497D">&nbsp;</span><o:p></o:p></p>
<p class="MsoNormal"><span style="color:#1F497D">&nbsp;</span><o:p></o:p></p>
<div>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.0pt;font-family:&quot;微软雅黑&quot;,sans-serif;color:#6E6E6F">Best Regards
</span><o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="527" style="width:394.9pt">
<tbody>
<tr style="height:19.4pt">
<td width="527" valign="top" style="width:394.9pt;padding:0in 0in 0in 0in;height:19.4pt">
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><b><span style="font-size:10.0pt;font-family:宋体;color:#6E6E6F">Yuyang Yang</span></b><o:p></o:p></p>
</td>
</tr>
<tr style="height:80.45pt">
<td width="527" valign="top" style="width:394.9pt;padding:0in 0in 0in 0in;height:80.45pt">
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:9.0pt;font-family:&quot;Tahoma&quot;,sans-serif">&nbsp;</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.5pt;color:#1F497D">&nbsp;</span><o:p></o:p></p>
</div>
<p class="MsoNormal"><span style="color:#1F497D">&nbsp;</span><o:p></o:p></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b><span lang="ZH-CN" style="font-family:等线;color:windowtext">发 件人</span></b><b><span style="font-family:等线;color:windowtext">:</span></b><span style="font-family:等线;color:windowtext"> Saravanakumar Arumugam [<a href="mailto:sarumuga@redhat.com">mailto:sarumuga@redhat.com</a>]
<br>
<b><span lang="ZH-CN">发送时间</span>:</b> Thursday, May 19, 2016 1:59 PM<br>
<b><span lang="ZH-CN">收件人</span>:</b> vyyy<span lang="ZH-CN">杨雨阳 </span><a href="mailto:yuyangyang@Ctrip.com">&lt;yuyangyang@Ctrip.com&gt;</a>;
<a href="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</a><br>
<b><span lang="ZH-CN">主题</span>:</b> Re: [Gluster-users] <span lang="ZH-CN">答复</span>: geo-replication status partial faulty</span><o:p></o:p></p>
</div>
</div>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal" style="margin-bottom:12.0pt">Hi,<br>
<br>
There seems to be some issue in glusterfs01.sh3.ctripcorp.com slave node.<br>
Can you share the complete logs ? <br>
<br>
You can increase verbosity of debug messages like this:<br>
gluster volume geo-replication &lt;master volume&gt; &lt;slave host&gt;::&lt;slave volume&gt; config log-level DEBUG<br>
<br>
<br>
Also, check&nbsp; /root/.ssh/authorized_keys in glusterfs01.sh3.ctripcorp.com<br>
It should have entries in /var/lib/glusterd/geo-replication/common_secret.pem.pub (present in master node).<br>
<br>
Have a look at this one for example:<br>
<a href="https://www.gluster.org/pipermail/gluster-users/2015-August/023174.html">https://www.gluster.org/pipermail/gluster-users/2015-August/023174.html</a><br>
<br>
Thanks,<br>
Saravana<o:p></o:p></p>
<div>
<p class="MsoNormal">On 05/19/2016 07:53 AM, vyyy<span lang="ZH-CN" style="font-family:&quot;微软雅黑&quot;,sans-serif">杨雨阳</span> wrote:<o:p></o:p></p>
</div>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<p class="MsoNormal">Hello,<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">I have tried to config a geo-replication volume , all the master nodes configuration are the same, When I start this volume, the status shows partial faulty as following:<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">gluster volume geo-replication filews glusterfs01.sh3.ctripcorp.com::filews_slave status<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">MASTER NODE&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; MASTER VOL&nbsp;&nbsp;&nbsp; MASTER BRICK&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; SLAVE&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; STATUS&nbsp;&nbsp;&nbsp;&nbsp; CHECKPOINT STATUS&nbsp;&nbsp;&nbsp; CRAWL STATUS&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">-------------------------------------------------------------------------------------------------------------------------------------------------<o:p></o:p></p>
<p class="MsoNormal">SVR8048HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR8050HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs03.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Passive&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR8047HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Active&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Hybrid Crawl&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR8049HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs05.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Active&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Hybrid Crawl&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SH02SVR5951&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/brick1&nbsp;&nbsp;&nbsp; glusterfs06.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Passive&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SH02SVR5953&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/brick1&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR6995HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SH02SVR5954&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/brick1&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp; &nbsp;&nbsp;N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR6994HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs02.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Passive&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR6993HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SH02SVR5952&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/brick1&nbsp;&nbsp;&nbsp; glusterfs01.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; faulty&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">SVR6996HW2285&nbsp;&nbsp;&nbsp; filews&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; /export/sdb/filews&nbsp;&nbsp;&nbsp; glusterfs04.sh3.ctripcorp.com::filews_slave&nbsp;&nbsp;&nbsp; Passive&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; N/A&nbsp;&nbsp;&nbsp;
<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">On the faulty node, log file /var/log/glusterfs/geo-replication/filews shows worker(/export/sdb/filews) died before establishing connection<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.402622] I [monitor(monitor):215:monitor] Monitor: ------------------------------------------------------------<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.402930] I [monitor(monitor):216:monitor] Monitor: starting gsyncd worker<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.517460] I [changelogagent(agent):72:__init__] ChangelogAgent: Agent listining...<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.518066] I [repce(agent):92:service_loop] RepceServer: terminating on reaching EOF.<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.518279] I [syncdutils(agent):214:finalize] &lt;top&gt;: exiting.<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:46.518194] I [monitor(monitor):267:monitor] Monitor: worker(/export/sdb/filews) died before establishing connection<o:p></o:p></p>
<p class="MsoNormal">[2016-05-18 16:55:56.697036] I [monitor(monitor):215:monitor] Monitor: ------------------------------------------------------------<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<div style="border:none;border-bottom:solid #EEEEEE 1.0pt;padding:0in 0in 14.0pt 0in">
<p class="MsoNormal" style="margin-bottom:15.0pt;line-height:30.0pt;background:white;vertical-align:baseline">
Any advice and suggestions will be greatly appreciated.<o:p></o:p></p>
</div>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><span style="font-size:10.0pt;font-family:΢���ź�;color:#6E6E6F">Best Regards
</span><o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="527" style="width:394.9pt">
<tbody>
<tr style="height:19.4pt">
<td width="527" valign="top" style="width:394.9pt;padding:0in 0in 0in 0in;height:19.4pt">
<p class="MsoNormal" style="text-align:justify;text-justify:inter-ideograph"><b><span style="font-size:10.0pt;font-family:&quot;Tahoma&quot;,sans-serif;color:#6E6E6F">������</span></b><b><span style="font-size:10.0pt;font-family:����;color:#6E6E6F"> Yuyang Yang</span></b><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal">&nbsp;<o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:&quot;Times New Roman ,serif&quot;,serif"><br>
<br>
<br>
<br>
</span><o:p></o:p></p>
<pre>_______________________________________________<o:p></o:p></pre>
<pre>Gluster-users mailing list<o:p></o:p></pre>
<pre><a href="mailto:Gluster-users@gluster.org">Gluster-users@gluster.org</a><o:p></o:p></pre>
<pre><a href="http://www.gluster.org/mailman/listinfo/gluster-users">http://www.gluster.org/mailman/listinfo/gluster-users</a><o:p></o:p></pre>
</blockquote>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:&quot;Times New Roman ,serif&quot;,serif">&nbsp;</span><o:p></o:p></p>
</blockquote>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:&quot;Times New Roman&quot;,serif"><o:p>&nbsp;</o:p></span></p>
</div>
</body>
</html>