[Gluster-devel] problems with option "export-statfs-size no"

[José M. Martín]

Hello,

I'm having problems when I set option export-statfs-size to "no" in
brick-posix.  Clients can't connect to the servers.
If I comment that option, all works. Is there any bug or am I doing
something wrong?

I'm using glusterfs-2.0.0.

Thank you.


LOG SERVER

May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
ns-posix: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1023
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
ns-posix: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1022
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
brick: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1019
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
brick: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1018
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
stripe: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1009
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [addr.c:174:gf_auth]
stripe: allowed = "192.168.*", received addr = "192.168.3.56"
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1008
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[server-protocol.c:5235:server_statfs] brick: 3: STATFS '/ (0)'
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[server-protocol.c:5235:server_statfs] brick: 3: STATFS '/ (0)'
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:90:__socket_rwv] server: EOF from peer 192.168.3.56:1023
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:562:__socket_proto_state_machine] server: read (Transport
endpoint is not connected) in state 1 (192.168.3.56:1023)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1023 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc000910 destroyed
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:90:__socket_rwv] server: EOF from peer 192.168.3.56:1022
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:562:__socket_proto_state_machine] server: read (Transport
endpoint is not connected) in state 1 (192.168.3.56:1022)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1022 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-helpers.c:723:server_connection_destroy] server: destroyed
connection of calypso56-14634-2009/05/13-10:42:50:315134-ns-remote
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc000c30 destroyed
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 E
[socket.c:102:__socket_rwv] server: readv failed (Connection reset by peer)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 E
[socket.c:562:__socket_proto_state_machine] server: read (Connection
reset by peer) in state 1 (192.168.3.56:1019)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1019 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc000f50 destroyed
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 E
[socket.c:102:__socket_rwv] server: readv failed (Connection reset by peer)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 E
[socket.c:562:__socket_proto_state_machine] server: read (Connection
reset by peer) in state 1 (192.168.3.56:1018)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1018 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-helpers.c:723:server_connection_destroy] server: destroyed
connection of calypso56-14634-2009/05/13-10:42:50:315134-brick2
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc001270 destroyed
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:90:__socket_rwv] server: EOF from peer 192.168.3.56:1009
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:562:__socket_proto_state_machine] server: read (Transport
endpoint is not connected) in state 1 (192.168.3.56:1009)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1009 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc001590 destroyed
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:90:__socket_rwv] server: EOF from peer 192.168.3.56:1008
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D
[socket.c:562:__socket_proto_state_machine] server: read (Transport
endpoint is not connected) in state 1 (192.168.3.56:1008)
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1008 disconnected
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 N
[server-helpers.c:723:server_connection_destroy] server: destroyed
connection of calypso56-14634-2009/05/13-10:42:50:315134-stripe2
May 13 10:45:00 nix02 logger: 2009-05-13 10:44:59 D [socket.c:1332:fini]
server: transport 0x7f50bc0018b0 destroyed
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1013
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1012
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1003
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:7502:mop_setvolume] server: accepted client from
192.168.3.56:1002
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 E
[socket.c:102:__socket_rwv] server: readv failed (Connection reset by peer)
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 E
[socket.c:562:__socket_proto_state_machine] server: read (Connection
reset by peer) in state 1 (192.168.3.56:1013)
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1013 disconnected
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 E
[socket.c:102:__socket_rwv] server: readv failed (Connection reset by peer)
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 E
[socket.c:562:__socket_proto_state_machine] server: read (Connection
reset by peer) in state 1 (192.168.3.56:1012)
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1012 disconnected
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-helpers.c:723:server_connection_destroy] server: destroyed
connection of calypso56-14634-2009/05/13-10:42:50:315134-brick5
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1003 disconnected
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-protocol.c:8272:notify] server: 192.168.3.56:1002 disconnected
May 13 10:45:12 nix05 logger: 2009-05-13 10:45:11 N
[server-helpers.c:723:server_connection_destroy] server: destroyed
connection of calypso56-14634-2009/05/13-10:42:50:315134-stripe5



SERVER CONFIG

### Export volume "brick" with the contents of
"/mnt/raid0/glusterfs/export" directory.
volume brick-posix
  type storage/posix                   # POSIX FS translator
  option directory /mnt/raid0/glusterfs/exports        # Export this
directory
#When more than one directory is exported from a single mountpoint in
backend
#(ie, there is another volume exporting directory from this mount point), it
#is possible that 'df' on the client shows more diskspace than that of
it actually has.
#In that case, one can say 'no' for exporting statfs-size from redundant
exports.
  option export-statfs-size no
end-volume

volume stripe-posix
  type storage/posix
  option directory /mnt/raid0/glusterfs/stripe
#  option export-statfs-size no
# it was observed that when a very big file (file size > 100GB) is
getting unlinked,
# and the backend storage is of ext3 or some other journaling fs,
glusterfs mountpoint
# becomes un-usable till unlink() call is returned. This can be solved
by enabling this
# option in posix layer. Advised to use it when the file sizes are
serveral GBs.
  option background-unlink yes
end-volume

volume ns-posix
  type storage/posix
  option directory /mnt/raid0/glusterfs/ns
#  option export-statfs-size no
end-volume

# lock
volume brick-locks
  type features/locks
  subvolumes brick-posix
end-volume
volume stripe
  type features/locks
  subvolumes stripe-posix
end-volume

# io-cache
volume brick-cache
  type performance/io-cache
  option cache-size 1GB
  subvolumes brick-locks
end-volume

# iothreads
volume brick
  type performance/io-threads
  option thread-count 16
  subvolumes brick-cache
end-volume

### Add network serving capability to above brick.
volume server
  type protocol/server
  option transport-type tcp
  option transport.socket.listen-port 6993              # Default is 6996
  subvolumes brick stripe ns-posix
# NOTE: Access to any volume through protocol/server is denied by
# default. You need to explicitly grant access through # "auth"
# option.
  option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume
  option auth.addr.stripe.allow 192.168.*
  option auth.addr.ns-posix.allow 192.168.*
end-volume



CLIENT CONFIG


volume brick1
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume brick          # name of the remote volume
  option remote-host nix01
  option remote-port 6993
end-volume

volume brick2
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume brick          # name of the remote volume
  option remote-host nix02
  option remote-port 6993
end-volume

volume brick3
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume brick          # name of the remote volume
  option remote-host nix03
  option remote-port 6993
end-volume

volume brick4
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume brick          # name of the remote volume
  option remote-host nix04
  option remote-port 6993
end-volume

volume brick5
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume brick          # name of the remote volume
  option remote-host nix05
  option remote-port 6993
end-volume

############################################################################
#  DHT
#
volume dht
  type cluster/distribute
#  option lookup-unhashed yes
#  option min-free-disk 10%
  subvolumes brick1 brick2 brick3 brick4 brick5
end-volume

############################################################################
#  CLIENTE STRIPE
#
volume stripe1
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume stripe          # name of the remote volume
  option remote-host nix01
  option remote-port 6993
end-volume

volume stripe2
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume stripe          # name of the remote volume
  option remote-host nix02
  option remote-port 6993
end-volume

volume stripe3
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume stripe          # name of the remote volume
  option remote-host nix03
  option remote-port 6993
end-volume

volume stripe4
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume stripe          # name of the remote volume
  option remote-host nix04
  option remote-port 6993
end-volume

volume stripe5
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume stripe          # name of the remote volume
  option remote-host nix05
  option remote-port 6993
end-volume
############################################################################
#  STRIPE
#
volume stripe
  type cluster/stripe
  option block-size 1MB
  subvolumes stripe1 stripe2 stripe3 stripe4 stripe5
end-volume

############################################################################
#  NAMESPACE
#
volume ns-remote
  type protocol/client
  option transport-type tcp              # for TCP/IP transport
  option remote-subvolume ns-posix          # name of the remote volume
  option remote-host nix02
  option remote-port 6993
end-volume
############################################################################
#  UNIFY SWITCH
#
volume bricks
  type cluster/unify
  option scheduler switch
  option namespace ns-remote
  option self-heal off
  option switch.case *.big*:stripe;*:dht
  subvolumes dht stripe
end-volume

volume writebehind
  type performance/write-behind
  option aggregate-size 128KB # default is 0bytes
  option cache-size 1MB    # default is equal to aggregate-size
  option flush-behind on    # default is 'off'
  subvolumes bricks
end-volume

volume readahead
  type performance/read-ahead
  option page-size 1024kB        # 256KB is the default option
  option page-count 32           # 2 is default option
  option force-atime-update off # default is off
  subvolumes writebehind
end-volume

volume iothreads
  type performance/io-threads
  option thread-count 8  # deault is 1
  subvolumes readahead
end-volume