[root@SC-182 log]# glusterfsd --version glusterfs 3.7.16 built on Nov 1 2016 10:41:53 Repository revision: git://git.gluster.com/glusterfs.git Copyright (c) 2006-2013 Red Hat, Inc. GlusterFS comes with ABSOLUTELY NO WARRANTY. It is licensed to you under your choice of the GNU Lesser General Public License, version 3 or any later version (LGPLv3 or later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. [root@SC-183 log]# glusterfsd --version glusterfs 3.7.16 built on Nov 1 2016 10:41:53 Repository revision: git://git.gluster.com/glusterfs.git Copyright (c) 2006-2013 Red Hat, Inc. GlusterFS comes with ABSOLUTELY NO WARRANTY. It is licensed to you under your choice of the GNU Lesser General Public License, version 3 or any later version (LGPLv3 or later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. [root@SC-182 log]# gluster volume info master-for-183-0003 Volume Name: master-for-183-0003 Type: Distribute Volume ID: 84501a83-b07c-4768-bfaa-418b038e1a9e Status: Started Number of Bricks: 1 Transport-type: tcp Bricks: Brick1: 10.10.60.182:/exports/nas-segment-0012/master-for-183-0003 Options Reconfigured: changelog.changelog: on geo-replication.ignore-pid-check: on geo-replication.indexing: on performance.readdir-ahead: on nfs.addr-namelookup: off nfs.disable: on performance.stat-prefetch: off performance.quick-read: off server.allow-insecure: on cluster.enable-shared-storage: disable snap-activate-on-create: enable [root@SC-182 log]# gluster volume status master-for-183-0003 Status of volume: master-for-183-0003 Gluster process TCP Port RDMA Port Online Pid ------------------------------------------------------------------------------ Brick 10.10.60.182:/exports/nas-segment-001 2/master-for-183-0003 62015 0 Y 14330 Task Status of Volume master-for-183-0003 ------------------------------------------------------------------------------ There are no active volume tasks [root@SC-183 log]# gluster volume info rem-volume-0001 Volume Name: rem-volume-0001 Type: Distribute Volume ID: a1e64801-473a-4ccf-a5e7-530f83c1a1a5 Status: Started Number of Bricks: 1 Transport-type: tcp Bricks: Brick1: 10.10.60.183:/exports/nas183-segment-0001/rem-volume-0001 Options Reconfigured: performance.readdir-ahead: on nfs.addr-namelookup: off nfs.disable: on performance.stat-prefetch: off performance.quick-read: off server.allow-insecure: on cluster.enable-shared-storage: disable snap-activate-on-create: enable [root@SC-183 log]# gluster volume status rem-volume-0001 Status of volume: rem-volume-0001 Gluster process TCP Port RDMA Port Online Pid ------------------------------------------------------------------------------ Brick 10.10.60.183:/exports/nas183-segment- 0001/rem-volume-0001 62006 0 Y 3723 Task Status of Volume rem-volume-0001 ------------------------------------------------------------------------------ There are no active volume tasks [root@SC-182 log]# date Fri Dec 9 11:46:51 PST 2016 [root@SC-182 log]# /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 status detail MASTER NODE MASTER VOL MASTER BRICK SLAVE USER SLAVE SLAVE NODE STATUS CRAWL STATUS LAST_SYNCED ENTRY DATA META FAILURES CHECKPOINT TIME CHECKPOINT COMPLETED CHECKPOINT COMPLETION TIME -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10.10.60.182 master-for-183-0003 /exports/nas-segment-0012/master-for-183-0003 nasgorep nasgorep@10.10.60.183::rem-volume-0001 10.10.60.183 Active Changelog Crawl 2016-12-09 11:43:49 0 0 0 0 2016-12-09 11:43:35 Yes 2016-12-09 11:43:53 [root@SC-182 log]# mount 127.0.0.1:/master-for-183-0003 on /samba/master-for-183-0003 type fuse.glusterfs (rw,allow_other,max_read=131072) [root@SC-182 log]# df /samba/master-for-183-0003/ Filesystem 1K-blocks Used Available Use% Mounted on 127.0.0.1:/master-for-183-0003 1038336 941696 96640 91% /samba/master-for-183-0003 [root@SC-183 log]# mount /dev/sdg on /exports/nas183-segment-0001 type xfs (rw,noatime,nodiratime,barrier,nouuid,inode64,logbufs=8,logbsize=256k) [root@SC-183 log]# df /exports/nas183-segment-0001 Filesystem 1K-blocks Used Available Use% Mounted on /dev/sdg 2086912 1041016 1045896 50% /exports/nas183-segment-0001 [root@SC-182 log]# date Fri Dec 9 11:52:29 PST 2016 [root@SC-182 log]# cp big.file /samba/master-for-183-0003/cifs_share/dir3/ [root@SC-182 log]# df /samba/master-for-183-0003/cifs_share/dir3/ Filesystem 1K-blocks Used Available Use% Mounted on 127.0.0.1:/master-for-183-0003 1038336 1018752 19584 99% /samba/master-for-183-0003 [root@SC-183 log]# df /exports/nas183-segment-0001 Filesystem 1K-blocks Used Available Use% Mounted on /dev/sdg 2086912 1118100 968812 54% /exports/nas183-segment-0001 [root@SC-182 log]# date Fri Dec 9 11:54:48 PST 2016 [root@SC-182 log]# cp big.file /samba/master-for-183-0003/cifs_share/dir3/big.file.1 cp: writing `/samba/master-for-183-0003/cifs_share/dir3/big.file.1': No space left on device cp: closing `/samba/master-for-183-0003/cifs_share/dir3/big.file.1': No space left on device [root@SC-182 log]# ls -l /samba/master-for-183-0003/cifs_share/dir3/* -rwx------ 1 root root 78930370 Dec 9 11:52 /samba/master-for-183-0003/cifs_share/dir3/big.file -rwx------ 1 root root 20582400 Dec 9 11:55 /samba/master-for-183-0003/cifs_share/dir3/big.file.1 [root@SC-182 log]# df /samba/master-for-183-0003/cifs_share/dir3/ Filesystem 1K-blocks Used Available Use% Mounted on 127.0.0.1:/master-for-183-0003 1038336 1038336 0 100% /samba/master-for-183-0003 [root@SC-183 log]# ls -l /exports/nas183-segment-0001/rem-volume-0001/cifs_share/dir3/ total 97184 -rwx------ 2 root root 78930370 Dec 9 11:52 big.file -rwx------ 2 root root 20582400 Dec 9 11:55 big.file.1 [root@SC-183 log]# df /exports/nas183-segment-0001 Filesystem 1K-blocks Used Available Use% Mounted on /dev/sdg 2086912 1138200 948712 55% /exports/nas183-segment-0001 [root@SC-182 log]# /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 status detail MASTER NODE MASTER VOL MASTER BRICK SLAVE USER SLAVE SLAVE NODE STATUS CRAWL STATUS LAST_SYNCED ENTRY DATA META FAILURES CHECKPOINT TIME CHECKPOINT COMPLETED CHECKPOINT COMPLETION TIME -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10.10.60.182 master-for-183-0003 /exports/nas-segment-0012/master-for-183-0003 nasgorep nasgorep@10.10.60.183::rem-volume-0001 10.10.60.183 Active Changelog Crawl 2016-12-09 11:55:19 0 0 0 0 2016-12-09 11:43:35 Yes 2016-12-09 11:43:53 [root@SC-182 log]# cp test.file /samba/master-for-183-0003/cifs_share/dir3/ cp: closing `/samba/master-for-183-0003/cifs_share/dir3/test.file': No space left on device [root@SC-182 log]# date Fri Dec 9 12:00:19 PST 2016 [root@SC-182 log]# rm /samba/master-for-183-0003/cifs_share/dir3/big.file.1 rm: remove regular file `/samba/master-for-183-0003/cifs_share/dir3/big.file.1'? y 30756 12/09/2016 12:01:23.063062008 1481313683 command: /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 30756 12/09/2016 12:01:24.121560922 1481313684 status=0 /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 30835 12/09/2016 12:01:24.136998832 1481313684 command: /bin/touch -a /samba/master-for-183-0003/ 30835 12/09/2016 12:01:24.141095827 1481313684 status=0 /bin/touch -a /samba/master-for-183-0003/ [root@SC-182 log]# /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 status detail MASTER NODE MASTER VOL MASTER BRICK SLAVE USER SLAVE SLAVE NODE STATUS CRAWL STATUS LAST_SYNCED ENTRY DATA META FAILURES CHECKPOINT TIME CHECKPOINT COMPLETED CHECKPOINT COMPLETION TIME -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10.10.60.182 master-for-183-0003 /exports/nas-segment-0012/master-for-183-0003 nasgorep nasgorep@10.10.60.183::rem-volume-0001 10.10.60.183 Active Changelog Crawl 2016-12-09 12:01:34 0 0 0 0 2016-12-09 12:01:23 Yes 2016-12-09 12:01:35 [root@SC-183 log]# date Fri Dec 9 12:04:21 PST 2016 [root@SC-183 log]# ls -l /exports/nas183-segment-0001/rem-volume-0001/cifs_share/dir3/ total 97184 -rwx------ 2 root root 78930370 Dec 9 11:52 big.file -rwx------ 2 root root 20582400 Dec 9 11:55 big.file.1 [root@SC-182 log]# ls -l /samba/master-for-183-0003/cifs_share/dir3/ total 77081 -rwx------ 1 root root 78930370 Dec 9 11:52 big.file -rw-r--r-- 1 root root 0 Dec 9 11:59 test.file [root@SC-182 log]# cp test.file /samba/master-for-183-0003/cifs_share/dir3/ cp: overwrite `/samba/master-for-183-0003/cifs_share/dir3/test.file'? y [root@SC-182 log]# ls -l /samba/master-for-183-0003/cifs_share/dir3/ total 77081 -rwx------ 1 root root 78930370 Dec 9 11:52 big.file -rw-r--r-- 1 root root 69 Dec 9 12:06 test.file 1067 12/09/2016 12:07:06.103427109 1481314026 command: /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 1067 12/09/2016 12:07:07.168979032 1481314027 status=0 /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 1155 12/09/2016 12:07:07.184514219 1481314027 command: /bin/touch -a /samba/master-for-183-0003/ 1155 12/09/2016 12:07:07.188478269 1481314027 status=0 /bin/touch -a /samba/master-for-183-0003/ [root@SC-182 log]# /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 status detail MASTER NODE MASTER VOL MASTER BRICK SLAVE USER SLAVE SLAVE NODE STATUS CRAWL STATUS LAST_SYNCED ENTRY DATA META FAILURES CHECKPOINT TIME CHECKPOINT COMPLETED CHECKPOINT COMPLETION TIME -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10.10.60.182 master-for-183-0003 /exports/nas-segment-0012/master-for-183-0003 nasgorep nasgorep@10.10.60.183::rem-volume-0001 10.10.60.183 Active Changelog Crawl 2016-12-09 12:07:19 0 0 0 0 2016-12-09 12:07:06 Yes 2016-12-09 12:07:20 [root@SC-183 log]# date Fri Dec 9 12:09:14 PST 2016 [root@SC-183 log]# ls -l /exports/nas183-segment-0001/rem-volume-0001/cifs_share/dir3/ total 97184 -rwx------ 2 root root 78930370 Dec 9 11:52 big.file -rwx------ 2 root root 20582400 Dec 9 11:55 big.file.1 [root@SC-182 log]# cp test.file /samba/master-for-183-0003/cifs_share/dir3/test.file.1 [root@SC-182 log]# date Fri Dec 9 12:11:00 PST 2016 3680 12/09/2016 12:11:29.690548541 1481314289 command: /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 3680 12/09/2016 12:11:30.748268857 1481314290 status=0 /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 config checkpoint now 2>&1 3760 12/09/2016 12:11:30.763822097 1481314290 command: /bin/touch -a /samba/master-for-183-0003/ 3760 12/09/2016 12:11:30.767884976 1481314290 status=0 /bin/touch -a /samba/master-for-183-0003/ [root@SC-182 log]# /usr/sbin/gluster volume geo-replication master-for-183-0003 nasgorep@10.10.60.183::rem-volume-0001 status detail MASTER NODE MASTER VOL MASTER BRICK SLAVE USER SLAVE SLAVE NODE STATUS CRAWL STATUS LAST_SYNCED ENTRY DATA META FAILURES CHECKPOINT TIME CHECKPOINT COMPLETED CHECKPOINT COMPLETION TIME -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 10.10.60.182 master-for-183-0003 /exports/nas-segment-0012/master-for-183-0003 nasgorep nasgorep@10.10.60.183::rem-volume-0001 10.10.60.183 Active Changelog Crawl 2016-12-09 12:11:34 0 0 0 0 2016-12-09 12:11:29 Yes 2016-12-09 12:11:38 [root@SC-183 log]# date Fri Dec 9 12:12:57 PST 2016 [root@SC-183 log]# ls -l /exports/nas183-segment-0001/rem-volume-0001/cifs_share/dir3/ total 97188 -rwx------ 2 root root 78930370 Dec 9 11:52 big.file -rwx------ 2 root root 20582400 Dec 9 11:55 big.file.1 -rw-r--r-- 2 root root 69 Dec 9 12:10 test.file.1