[Gluster-devel] [client-protocol.c:5678:client_forget_cbk] : fop not implemented

[Ville Tuulos]

Hi

I upgraded to the latest Git snapshot (HEAD 57a6cb9806ad9d99e) yesterday, 
related to the trusted.glustefs.location issue (see thread "xattrs and bug 
9").

Results of this upgrade were less than stellar. First, I couldn't find 
the trusted.glusterfs.location attribute on any files. How should it work?

Today, a day after the upgrade, I noticed that several glusterfs processes 
had gone wild and consumed >20G of memory.

Client logs repeat the following message:

[2009-08-16 11:36:49] C [client-protocol.c:5678:client_forget_cbk] : fop 
not implemented
[2009-08-16 11:36:49] E [socket.c:1052:socket_connect] log-transport: 
connection attempt failed (Connection refused)

and the server logs contain 9G of the following messages:

[2009-08-16 04:09:43] C [client-protocol.c:5678:client_forget_cbk] : fop 
not implemented
[2009-08-16 04:12:24] E [saved-frames.c:165:saved_frames_unwind] 
log-transport: forced unwinding frame type(2) op(LOG)

What's your policy with the git HEAD? Is it supposed to be suitable for 
normal, non mission-critical, usage at all times?

Please find my volume file below.


Ville


-- clip --

# ------
# SERVER
# ------

# -- vol1

volume vol1-posix
    type storage/posix
    option directory /data1
end-volume

volume vol1-lock
    type features/locks
    subvolumes vol1-posix
end-volume

volume vol1-readahead
    type performance/read-ahead
    option page-count 4
    option page-size 512KB
    subvolumes vol1-lock
end-volume

volume vol1
    type performance/io-threads
    option thread-count 32
    subvolumes vol1-readahead
end-volume

# -- vol2

volume vol2-posix
    type storage/posix
    option directory /data2
end-volume

volume vol2-lock
    type features/locks
    subvolumes vol2-posix
end-volume

volume vol2-readahead
    type performance/read-ahead
    option page-count 4
    option page-size 512KB
    subvolumes vol2-lock
end-volume

volume vol2
    type performance/io-threads
    option thread-count 32
    subvolumes vol2-readahead
end-volume

# -- server

volume server
    type protocol/server
    option auth.addr.vol1.allow *
    option auth.addr.vol2.allow *
    option transport-type tcp
    option transport.socket.listen-port 9900
    subvolumes vol1 vol2
end-volume


# -----
# NODES
# -----

# -- dx02

volume dx02-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx02
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx02-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx02
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx03

volume dx03-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx03
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx03-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx03
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx04

volume dx04-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx04
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx04-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx04
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx05

volume dx05-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx05
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx05-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx05
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx06

volume dx06-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx06
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx06-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx06
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx07

volume dx07-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx07
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx07-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx07
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx08

volume dx08-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx08
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx08-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx08
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx09

volume dx09-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx09
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx09-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx09
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx10

volume dx10-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx10
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx10-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx10
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx13

volume dx13-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx13
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx13-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx13
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx14

volume dx14-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx14
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx14-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx14
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx15

volume dx15-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx15
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx15-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx15
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx16

volume dx16-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx16
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx16-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx16
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx17

volume dx17-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx17
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx17-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx17
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx18

volume dx18-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx18
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx18-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx18
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx19

volume dx19-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx19
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx19-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx19
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx20

volume dx20-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx20
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx20-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx20
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx21

volume dx21-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx21
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx21-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx21
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx22

volume dx22-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx22
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx22-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx22
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx23

volume dx23-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx23
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx23-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx23
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx24

volume dx24-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx24
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx24-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx24
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx25

volume dx25-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx25
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx25-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx25
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx26

volume dx26-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx26
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx26-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx26
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx27

volume dx27-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx27
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx27-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx27
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx28

volume dx28-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx28
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx28-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx28
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx29

volume dx29-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx29
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx29-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx29
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx30

volume dx30-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx30
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx30-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx30
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx31

volume dx31-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx31
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx31-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx31
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx32

volume dx32-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx32
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx32-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx32
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx33

volume dx33-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx33
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx33-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx33
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx34

volume dx34-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx34
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx34-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx34
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx35

volume dx35-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx35
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx35-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx35
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume

# -- dx37

volume dx37-vol1
    type protocol/client
    option ping-timeout 60
    option remote-host dx37
    option remote-port 9900
    option remote-subvolume vol1
    option transport-type tcp
end-volume

volume dx37-vol2
    type protocol/client
    option ping-timeout 60
    option remote-host dx37
    option remote-port 9900
    option remote-subvolume vol2
    option transport-type tcp
end-volume


# -----------
# REPLICATION
# -----------

# replication group 1
#    dx35 (22768f11e0d76ce73cc63a1b016b15f)
#    dx08 (304b14c2a158b7504bca52ac5a6c418)
#    dx34 (af5d544395bbad6170deddc88cafa5d)

volume repl1-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx35-vol1 dx08-vol1 dx34-vol1
end-volume

volume repl1-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx35-vol2 dx08-vol2 dx34-vol2
end-volume

# replication group 2
#    dx04 (dc1a1598b1179fc438fe64bb45a620e)
#    dx29 (155f6d242985603418d89cd4ecd2433c)
#    dx19 (17b17414eddfc2f1e60bc8be2375c602)

volume repl2-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx04-vol1 dx29-vol1 dx19-vol1
end-volume

volume repl2-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx04-vol2 dx29-vol2 dx19-vol2
end-volume

# replication group 3
#    dx15 (18888fcf091a7607599532b97e051e61)
#    dx23 (1a0d213507c3ee4d63ed0cdbbc4815ec)
#    dx27 (1e9ffba57b99473199d425db1a6539d5)

volume repl3-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx15-vol1 dx23-vol1 dx27-vol1
end-volume

volume repl3-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx15-vol2 dx23-vol2 dx27-vol2
end-volume

# replication group 4
#    dx37 (23c0b19e3c7b36d0150db91451cfe12b)
#    dx05 (2e5012609403826ad9ef4c684bce9066)
#    dx03 (41a05dea3b673d111b3bf070523881cb)

volume repl4-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx37-vol1 dx05-vol1 dx03-vol1
end-volume

volume repl4-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx37-vol2 dx05-vol2 dx03-vol2
end-volume

# replication group 5
#    dx18 (68026a43fa1f37b491c5601843a75845)
#    dx14 (87959142f56ee156596aa648183fe8d8)
#    dx16 (9310d179b87b9a61fe53626608167bc5)

volume repl5-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx18-vol1 dx14-vol1 dx16-vol1
end-volume

volume repl5-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx18-vol2 dx14-vol2 dx16-vol2
end-volume

# replication group 6
#    dx09 (9cf09374a1b2659f9d8d2b77562fcabd)
#    dx07 (a52ae25b8b90a07e2f960f81f2bc0085)
#    dx31 (a930413a8ce473e3ad229d8b0fbffa4b)

volume repl6-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx09-vol1 dx07-vol1 dx31-vol1
end-volume

volume repl6-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx09-vol2 dx07-vol2 dx31-vol2
end-volume

# replication group 7
#    dx30 (b1f4f4f59f82a779ccc0237dfc6ca99f)
#    dx10 (b391d9c4829dc247f8b1cd476217f6c7)
#    dx32 (b4ec1bc4ca396a0e106ee52d04663289)

volume repl7-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx30-vol1 dx10-vol1 dx32-vol1
end-volume

volume repl7-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx30-vol2 dx10-vol2 dx32-vol2
end-volume

# replication group 8
#    dx28 (b8b2e4dbb10f4d0fd4a0e5687045eb65)
#    dx25 (bfeeda403fcc9a774959b014d6c831d6)
#    dx02 (c25b161d6cda8a437fd4b527e7cdd185)

volume repl8-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx28-vol1 dx25-vol1 dx02-vol1
end-volume

volume repl8-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx28-vol2 dx25-vol2 dx02-vol2
end-volume

# replication group 9
#    dx33 (cda4e9f6a9d8ea1a49431375349648af)
#    dx13 (d6e9834b895340f48e08ed3207255518)
#    dx17 (d84950ffd383c64a4ad24de2e722b947)

volume repl9-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx33-vol1 dx13-vol1 dx17-vol1
end-volume

volume repl9-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx33-vol2 dx13-vol2 dx17-vol2
end-volume

# replication group 10
#    dx26 (d995c43b17435569077aff9fc59b0757)
#    dx20 (d9a7ad05ebd30c52ac33b2d470880433)
#    dx24 (e86547f4e3c6a16decc6c182816a5326)

volume repl10-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx26-vol1 dx20-vol1 dx24-vol1
end-volume

volume repl10-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx26-vol2 dx20-vol2 dx24-vol2
end-volume

# replication group 11
#    dx22 (f480ce4cba11ae4a4bc5db32e67945f3)
#    dx21 (fa4f3d023e340fc24c8c5b04a785718b)
#    dx06 (fa88b6a797b37ce1618b8a98331830e1)

volume repl11-vol1
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol1"`
    subvolumes dx22-vol1 dx21-vol1 dx06-vol1
end-volume

volume repl11-vol2
    type cluster/replicate
    option read-subvolume `echo "$(hostname)-vol2"`
    subvolumes dx22-vol2 dx21-vol2 dx06-vol2
end-volume


# ------
# CLIENT
# ------

volume distribute
    type cluster/distribute
    subvolumes repl1-vol1 repl1-vol2 repl2-vol1 repl2-vol2 repl3-vol1 
repl3-vol2 repl4-vol1 repl4-vol2 repl5-vol1 repl5-vol2 repl6-vol1 repl6-vol2 
repl7-vol1 repl7-vol2 repl8-vol1 repl8-vol2 repl9-vol1 repl9-vol2 repl10-vol1 
repl10-vol2 repl11-vol1 repl11-vol2
end-volume