# # patch "dumb.py" # from [f27c38a9c6c0949562af24eb429ef70655e387ec] # to [20b825f392c97aadca36cf47a3a107baca3fd066] # # patch "merkle_dir.py" # from [83b65f2294aa41de080fdf12c3345ff70f6204ca] # to [a7bca4b74638296707f43c49f36bb8da7b475a84] # # patch "remote.py" # from [9f1f211f97f2e689b647cd30dd6cacaa22b16a1e] # to [8ed5efc2ed70f8b430eec15b81a8c2a62bd167f4] # ======================================================================== --- dumb.py f27c38a9c6c0949562af24eb429ef70655e387ec +++ dumb.py 20b825f392c97aadca36cf47a3a107baca3fd066 @@ -101,29 +101,12 @@ # it and see what happens) # "delete file" -# locking: -# writers: -# -- if the lockfile exists, abort -# -- append a unique nonce to the lockfile -# -- fetch the lockfile; if our nonce is at the beginning proceed -# otherwise, abort -# (with ftp, can use APPE command: ftp.storebinary("APPE ", fileobj)) -# (with sftp, can open in mode "a") -# -- when done, delete lockfile +# also write a file called _lock_info or something containing info on who +# created the lock, to ease in cleaning things up. # -# readers: -# -- fetches VERSION, then root hashfile, then child hashes, then data -# -- data is not referenced until fully written, so that's okay -# -- as long as hash files are updated atomically (!), they will always -# contain a list of things that really exist and may be wanted; and the -# root file will always contain a list of things that may be interesting -# (even if yet more interesting things are being written at the same -# time) -# so the worst thing that can happen is that we get some stuff without its -# prerequisites, which will be fixed at next pull anyway. -# should probably check the push version before and after the pull anyway, -# so we can tell the user when they raced and there's more stuff to get...? -# or even just start the pull over...? +# there is no atomic replace in sftp (probably not ftp either). can write, +# delete, rename, to minimize window. but clients should be prepared to retry +# if a file fetch fails. # TODO: # -- cat revision and packet commands -> automate? ======================================================================== --- merkle_dir.py 83b65f2294aa41de080fdf12c3345ff70f6204ca +++ merkle_dir.py a7bca4b74638296707f43c49f36bb8da7b475a84 @@ -11,7 +11,8 @@ data_file = "DATA" index_file = "INDEX" data_length_file = "DATA_LENGTH" - lock_file = "__lock" + lock_file = "_lock" + lock_info_file = "_lock_info" hashes_prefix = "HASHES_" def __init__(self, directory): ======================================================================== --- remote.py 9f1f211f97f2e689b647cd30dd6cacaa22b16a1e +++ remote.py 8ed5efc2ed70f8b430eec15b81a8c2a62bd167f4 @@ -6,6 +6,11 @@ class ReadableServer: # All operators are blocking + # returns an object that supports read(), close() with standard file + # object semantics + def open_read(self, filename): + raise NotImplementedError + # takes an iterable of filenames # returns a map {filename -> contents of file} def fetch(self, filenames): @@ -22,8 +27,9 @@ class WriteableServer (ReadableServer): - # returns None - def append(self, filename, data): + # returns an object that supports write(), flush(), close() with standard + # file object semantics + def open_append(self, filename): raise NotImplementedError # files is a map {filename -> contents of file} @@ -39,35 +45,39 @@ def __init__(self, dir): self.dir = dir + def _fname(self, filename): + return os.path.join(self.dir, filename) + + def open_read(self, filename): + return open(self._fname(filename), "rb") + def fetch(self, filenames): files = {} for fn in filenames: - f = open(os.path.join(self.dir, fn), "rb") + f = open(self._fname(fn), "rb") files[fn] = f.read() f.close() return files def fetch_bytes(self, filename, bytes): - f = open(os.path.join(self.dir, filename), "rb") + f = open(self._fname(filename), "rb") for offset, length in bytes: f.seek(offset) yield ((offset, length), f.read(length)) def exists(self, filename): - return os.path.exists(os.path.join(self.dir, filename)) + return os.path.exists(self._fname(filename)) - def append(self, filename, data): - f = open(os.path.join(self.dir, filename), "ab") - f.write(data) - f.close() + def open_append(self, filename): + return open(self._fname(filename), "ab") def replace(self, filenames): for fn, data in filenames.iteritems(): - tmpname = os.path.join(self.dir, "_tmp") + tmpname = self._fname("__tmp") tmph = open(tmpname, "wb") tmph.write(data) tmph.close() - os.rename(tmpname, os.path.join(self.dir, fn)) + os.rename(tmpname, self._fname(fn)) def delete(self, filename): + os.unlink(self._fname(filename)) - os.unlink(os.path.join(self.dir, filename))