Re: [Gluster-devel] ha example

[Mickey Mazarick]

Our first test didn't go so well, we got the error below.
If you remind me how to display the stacktrace using the core file I'll 
send that as well :-)

This was a complicated first test with 6 servers 
Unify(AFR(1,2),AFR(3,4),AFR(5,6))
We just put in HA for each server to itself over an alternate protocol.

example:
volume brick1
 type cluster/ha
 subvolumes brick1ib brick1tcp
end-volume


pending frames:
frame : type(1) op(32)

Signal received: 11
configuration details:argp 1
backtrace 1
dlfcn 1
fdatasync 1
libpthread 1
llistxattr 1
setfsid 1
spinlock 1
epoll.h 1
xattr.h 1
tv_nsec 1
package-string: glusterfs 1.4.0pre6
/lib64/tls/libc.so.6[0x33c0a2e2b0]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/cluster/ha.so(ha_checksum+0x192)[0x2b514f2a8972]
/usr/local/lib/libglusterfs.so.0(default_checksum+0xb2)[0x2b514efe6962]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/cluster/unify.so(gf_unify_self_heal+0x404)[0x2b514f4cb364]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/cluster/unify.so(unify_lookup_cbk+0x752)[0x2b514f4befc2]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/cluster/afr.so(afr_lookup_cbk+0x160)[0x2b514f3b0580]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/cluster/ha.so[0x2b514f29ff04]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/protocol/client.so(client_lookup_cbk+0xd8)[0x2b514f1967c8]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/protocol/client.so(protocol_client_interpret+0x38d)[0x2b514f1993ed]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/protocol/client.so(protocol_client_pollin+0xbe)[0x2b514f199b0e]
/usr/local/lib/glusterfs/1.4.0pre6/xlator/protocol/client.so(notify+0x1cb)[0x2b514f199d1b]
/usr/local/lib/glusterfs/1.4.0pre6/transport/socket.so[0x2aaaad22c13e]
/usr/local/lib/libglusterfs.so.0[0x2b514eff54d1]
/usr/local/sbin/glusterfs(main+0x69b)[0x402d2b]
/lib64/tls/libc.so.6(__libc_start_main+0xdb)[0x33c0a1c3fb]
/usr/local/sbin/glusterfs[0x4021ea]
---------


SPECFILE:

####TCP######
volume brick1tcp
type protocol/client
option transport-type tcp     # for TCP/IP transport
option remote-host RTPST201      # IP address of the remote brick
option remote-subvolume brick        # name of the remote volume
end-volume

volume brick2tcp
type protocol/client
option transport-type tcp
option remote-host RTPST202
option remote-subvolume brick
end-volume

volume brick3tcp
type protocol/client
option transport-type tcp
option remote-host RTPST203
option remote-subvolume brick
end-volume

volume brick4tcp
type protocol/client
option transport-type tcp
option remote-host RTPST204
option remote-subvolume brick
end-volume

volume brick5tcp
type protocol/client
option transport-type tcp
option remote-host RTPST205
option remote-subvolume brick
end-volume

volume brick6tcp
type protocol/client
option transport-type tcp
option remote-host RTPST206
option remote-subvolume brick
end-volume

volume brick-ns1tcp
type protocol/client
option transport-type tcp
option remote-host RTPST201
option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

volume brick-ns2tcp
type protocol/client
option transport-type tcp
option remote-host RTPST202
option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

###IBVERBS####

volume brick1ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST201
option remote-subvolume brick
end-volume

volume brick2ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST202
option remote-subvolume brick
end-volume

volume brick3ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST203
option remote-subvolume brick
end-volume

volume brick4ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST204
option remote-subvolume brick
end-volume

volume brick5ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST205
option remote-subvolume brick
end-volume

volume brick6ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST206
option remote-subvolume brick
end-volume

volume brick-ns1ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST201
option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume

volume brick-ns2ib
type protocol/client
option transport-type ib-verbs
option transport-timeout 2
option remote-port 6997
option remote-host RTPST202
option remote-subvolume brick-ns  # Note the different remote volume name.
end-volume


###HA Section
volume brick1
 type cluster/ha
 subvolumes brick1ib brick1tcp
end-volume

volume brick2
 type cluster/ha
 subvolumes brick2ib brick2tcp
end-volume

volume brick3
 type cluster/ha
 subvolumes brick3ib brick3tcp
end-volume

volume brick4
 type cluster/ha
 subvolumes brick4ib brick4tcp
end-volume

volume brick5
 type cluster/ha
 subvolumes brick5ib brick5tcp
end-volume

volume brick6
 type cluster/ha
 subvolumes brick6ib brick6tcp
end-volume

volume brick-ns1
 type cluster/ha
 subvolumes brick-ns1ib brick-ns1tcp
end-volume

volume brick-ns2
 type cluster/ha
 subvolumes brick-ns2ib brick-ns2tcp
end-volume


###AFR SECTION

volume afr1
type cluster/afr
subvolumes brick1 brick4
end-volume

volume afr2
type cluster/afr
subvolumes brick2 brick5
end-volume

volume afr3
type cluster/afr
subvolumes brick3 brick6
end-volume

volume afr-ns
type cluster/afr
subvolumes brick-ns1 brick-ns2
end-volume

volume unify
type cluster/unify
option namespace afr-ns
option scheduler rr
subvolumes afr1 afr2 afr3
end-volume

###PERFORMANCE SECTION

volume io-cache
 type performance/io-cache
 subvolumes unify
end-volume

volume readahead
 type performance/read-ahead
 option force-atime-update on # default is off
 subvolumes io-cache
end-volume

volume writebehind
 type performance/write-behind
 option aggregate-size 1MB # default is 0bytes
 #option flush-behind on    # default is 'off'
 subvolumes readahead
end-volume




Krishna Srinivas wrote:
> oops .... forgot to CC gluster-devel...
>
> On Fri, Oct 17, 2008 at 12:58 AM, Krishna Srinivas
> <address@hidden> wrote:
>   
> > On Fri, Oct 17, 2008 at 12:50 AM, Mickey Mazarick
> > <address@hidden> wrote:
> >     
> > > Ultimately we want to use it permanently. We are looking for a system that
> > > lets our infiniband system fail over to gigabit ethernet.
> > > For us,
> > > subvolume one = server1 over ibverbs subvolume two = server1 over tcp
> > >
> > > How does it currently handle failure? Will it attempt to connect to one then
> > > fail over to two on every read? After a number of failures will it always
> > > use 2?
> > >       
> > HA will maintain the state of an "operation", if it fails on one
> > subvol because of connection failure, it tries the operaion on the
> > next subvol, application issuing the system call like read/write will
> > not see the connection loss.
> >
> >     
> > > Thanks!
> > >
> > > Krishna Srinivas wrote:
> > >       
> > > > Hi Mickey,
> > > >
> > > > config is simple, you give a list of subvolumes list:
> > > > volume ha1
> > > >  type cluster/ha
> > > >  subvolumes one two
> > > > end-volume
> > > >
> > > > Do you just want to test it or you are planning on a setup which needs
> > > > it? (if so can you give me the setup details?, I am just curious)
> > > >
> > > > The code is in the TLA, but "locking" code flow needs some changes.
> > > > You can try it out though.
> > > >
> > > > Thanks
> > > > Krishna
> > > >
> > > >
> > > > On Fri, Oct 17, 2008 at 12:10 AM, Mickey Mazarick
> > > > <address@hidden> wrote:
> > > >
> > > >         
> > > > > Can someone give me a quick example of how the ha xlator works?  I for
> > > > > one
> > > > > want to test it. :-)
> > > > >
> > > > > Thanks!
> > > > > --
> > > > >
> > > > >
> > > > > _______________________________________________
> > > > > Gluster-devel mailing list
> > > > > address@hidden
> > > > > http://lists.nongnu.org/mailman/listinfo/gluster-devel
> > > > >
> > > > >
> > > > >           
> > > --
> > >
> > >       
>
>
> _______________________________________________
> Gluster-devel mailing list
> address@hidden
> http://lists.nongnu.org/mailman/listinfo/gluster-devel
>   


--