# # add_file "string_queue.cc" # # add_file "string_queue.hh" # # patch "ChangeLog" # from [c81f05a228d690fe72ca7938c04ed46e997f1f57] # to [acb9b81e92fd4b1dd67d14cb12270dbf07803a7c] # # patch "Makefile.am" # from [c8f56d086950c437c10c455b623b7fb4766a8bdf] # to [1a1de954d58c4511fc0a01186c70ac86164c51bf] # # patch "database.cc" # from [cae2141a55c09e75754a2887d5c89d258218896a] # to [8d0f3f9f112b640c2eb877595fbce56e8b04ac02] # # patch "hmac.cc" # from [db9feaa73b21d5ab1b088fcb1d2fe214fa20a1e5] # to [6abde45d788c47d50bb104e76e26ee903b93be7f] # # patch "hmac.hh" # from [416753e704f4a6294e4f76b57e732dfa36a733e0] # to [759ade0869017270c8f10bf4307e7ab463f18868] # # patch "netcmd.cc" # from [b18f03651d0f0256957f47afcc96f5a121598d96] # to [24df93e3c3d6ae51a8ee69d972f10392609af1d9] # # patch "netcmd.hh" # from [f6262bdd5719defdd5ef6237e090687701ef4006] # to [8bea78d5e3429c65cc158c90450a7473528ceb07] # # patch "netio.hh" # from [6d9731e34a43726d756fdc9e2724bd962ed46942] # to [e973154d8e441e5d4446d75a0112f1450b826b7f] # # patch "netsync.cc" # from [c5406e41e4aec3b81907d638db40e805bde7edfb] # to [d1db1316c8a727e34dc4f285bc20d1beed0c8fc7] # # patch "string_queue.cc" # from [] # to [8e101f4569d10559b9bcac8b08677d5332e96ac1] # # patch "string_queue.hh" # from [] # to [c97430d992a71fe7f05eda42d852a93db6f04db3] # # patch "unit_tests.cc" # from [049f2a37c29874fb26d6cb5fff1e3417303bc2e3] # to [5088d7c068c79caf69f553a295b8dbf496b6b442] # # patch "unit_tests.hh" # from [7213d23add3f0715300ca0348d9c16b1d136cc3e] # to [fa58467ac77ae1ee84f966c620eecf917be527b7] # ======================================================================== --- ChangeLog c81f05a228d690fe72ca7938c04ed46e997f1f57 +++ ChangeLog acb9b81e92fd4b1dd67d14cb12270dbf07803a7c @@ -224,6 +224,26 @@ * monotone.spec: include zlib-devel and texinfo as build requirements, zlib as a runtime requirement. +2005-08-09 Eric Anderson + + * Changes to significantly improve network pull performance + * string_queue.hh: created to store pending data and allow for + efficient removal from the front. The string queue automatically + reduces its buffer size if it is very empty. + * hmac.{cc,hh}: Add in a version of chained_hmac::process that can + operate on a string_queue for use during read. + * netcmd.{cc,hh}: update netcmd::read to use a string_queue rather + than a string, update all the regression tests also. This required + the somewhat ugly creation of a read_string function because the + netcmd read and write functions are no longer using the same type. + * netio.hh: introduce functions for operating on a string_queue. They + are identical to the equivalent string functions except for the type + of the argument. + * netsync.cc: Use a string_queue rather than a string for storing the + input and output buffers. + + * string_queue.cc: unit tests (Matt Johnston) + 2005-08-09 Richard Li * std_hooks.lua (merge2, merge3): explain a little better why ======================================================================== --- Makefile.am c8f56d086950c437c10c455b623b7fb4766a8bdf +++ Makefile.am 1a1de954d58c4511fc0a01186c70ac86164c51bf @@ -42,6 +42,7 @@ restrictions.cc restrictions.hh \ hmac.cc hmac.hh \ globish.cc globish.hh \ + string_queue.cc string_queue.hh \ \ cleanup.hh unit_tests.hh interner.hh \ cycle_detector.hh randomfile.hh adler32.hh quick_alloc.hh \ ======================================================================== --- database.cc cae2141a55c09e75754a2887d5c89d258218896a +++ database.cc 8d0f3f9f112b640c2eb877595fbce56e8b04ac02 @@ -1362,7 +1362,7 @@ new_id.inner()().c_str()); } - check_sane_history(new_id, constants::verify_depth, *__app); + //check_sane_history(new_id, constants::verify_depth, *__app); guard.commit(); } ======================================================================== --- hmac.cc db9feaa73b21d5ab1b088fcb1d2fe214fa20a1e5 +++ hmac.cc 6abde45d788c47d50bb104e76e26ee903b93be7f @@ -42,3 +42,26 @@ return chain_val; } + +std::string +chained_hmac::process(string_queue const & str, size_t pos, size_t n) +{ + I(pos < str.size()); + if (n == std::string::npos) + n = str.size() - pos; + + I(pos + n <= str.size()); + + Botan::Pipe p(new Botan::MAC_Filter("HMAC(SHA-1)", key, + constants::sha1_digest_length)); + p.start_msg(); + p.write(chain_val); + p.write(reinterpret_cast(str.front_pointer(n) + pos), n); + + p.end_msg(); + + chain_val = p.read_all_as_string(); + I(chain_val.size() == constants::sha1_digest_length); + + return chain_val; +} ======================================================================== --- hmac.hh 416753e704f4a6294e4f76b57e732dfa36a733e0 +++ hmac.hh 759ade0869017270c8f10bf4307e7ab463f18868 @@ -6,6 +6,7 @@ #include "botan/botan.h" #include "vocab.hh" #include "constants.hh" +#include "string_queue.hh" struct chained_hmac { @@ -14,6 +15,8 @@ void set_key(netsync_session_key const & session_key); std::string process(std::string const & str, size_t pos = 0, size_t n = std::string::npos); + std::string process(string_queue const & str, size_t pos = 0, + size_t n = std::string::npos); size_t const hmac_length; ======================================================================== --- netcmd.cc b18f03651d0f0256957f47afcc96f5a121598d96 +++ netcmd.cc 24df93e3c3d6ae51a8ee69d972f10392609af1d9 @@ -77,7 +77,7 @@ } bool -netcmd::read(string & inbuf, chained_hmac & hmac) +netcmd::read(string_queue & inbuf, chained_hmac & hmac) { size_t pos = 0; @@ -126,35 +126,25 @@ // there might not be enough data yet in the input buffer if (inbuf.size() < pos + payload_len + constants::netsync_hmac_value_length_in_bytes) { - inbuf.reserve(pos + payload_len + constants::netsync_hmac_value_length_in_bytes + constants::bufsz); return false; } -// out.payload = extract_substring(inbuf, pos, payload_len, "netcmd payload"); - // Do this ourselves, so we can swap the strings instead of copying. - require_bytes(inbuf, pos, payload_len, "netcmd payload"); - // grab it before the data gets munged I(hmac.hmac_length == constants::netsync_hmac_value_length_in_bytes); string digest = hmac.process(inbuf, 0, pos + payload_len); - payload = inbuf.substr(pos + payload_len); - inbuf.erase(pos + payload_len, inbuf.npos); - inbuf.swap(payload); - size_t payload_pos = pos; - pos = 0; + payload = extract_substring(inbuf, pos, payload_len, "netcmd payload"); // they might have given us bogus data string cmd_digest = extract_substring(inbuf, pos, constants::netsync_hmac_value_length_in_bytes, "netcmd HMAC"); - inbuf.erase(0, pos); + inbuf.pop_front(pos); if (cmd_digest != digest) throw bad_decode(F("bad HMAC checksum (got %s, wanted %s)\n" "this suggests data was corrupted in transit\n") % encode_hexenc(cmd_digest) % encode_hexenc(digest)); - payload.erase(0, payload_pos); return true; } @@ -571,7 +561,7 @@ chained_hmac mac(key); // mutates mac out_cmd.write(buf, mac); - BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + BOOST_CHECK_THROW(in_cmd.read_string(buf, mac), bad_decode); } { @@ -581,7 +571,7 @@ buf[0] ^= 0xff; { chained_hmac mac(key); - BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + BOOST_CHECK_THROW(in_cmd.read_string(buf, mac), bad_decode); } { @@ -591,7 +581,7 @@ buf[buf.size() - 1] ^= 0xff; { chained_hmac mac(key); - BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + BOOST_CHECK_THROW(in_cmd.read_string(buf, mac), bad_decode); } { @@ -601,7 +591,7 @@ buf += '\0'; { chained_hmac mac(key); - BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + BOOST_CHECK_THROW(in_cmd.read_string(buf, mac), bad_decode); } } @@ -615,7 +605,7 @@ } { chained_hmac mac(key); - BOOST_CHECK(in_cmd.read(buf, mac)); + BOOST_CHECK(in_cmd.read_string(buf, mac)); } BOOST_CHECK(in_cmd == out_cmd); } ======================================================================== --- netcmd.hh f6262bdd5719defdd5ef6237e090687701ef4006 +++ netcmd.hh 8bea78d5e3429c65cc158c90450a7473528ceb07 @@ -13,6 +13,7 @@ #include "numeric_vocab.hh" #include "vocab.hh" #include "hmac.hh" +#include "string_queue.hh" typedef enum { @@ -65,9 +66,20 @@ // basic cmd i/o (including checksums) void write(std::string & out, chained_hmac & hmac) const; - bool read(std::string & inbuf, + bool read(string_queue & inbuf, chained_hmac & hmac); - + bool read_string(std::string & inbuf, + chained_hmac & hmac) { + // this is here only for the regression tests because they want to + // read and write to the same type, but we want to have reads from + // a string queue so that when data is read in from the network it + // can be processed efficiently + string_queue tmp(inbuf.size()); + tmp.append(inbuf); + bool ret = read(tmp, hmac); + inbuf = tmp.substr(0,tmp.size()); + return ret; + } // i/o functions for each type of command payload void read_error_cmd(std::string & errmsg) const; void write_error_cmd(std::string const & errmsg); ======================================================================== --- netio.hh 6d9731e34a43726d756fdc9e2724bd962ed46942 +++ netio.hh e973154d8e441e5d4446d75a0112f1450b826b7f @@ -14,6 +14,7 @@ #include "numeric_vocab.hh" #include "sanity.hh" +#include "string_queue.hh" struct bad_decode { bad_decode(boost::format const & fmt) : what(fmt.str()) {} @@ -22,6 +23,24 @@ inline void require_bytes(std::string const & str, + size_t pos, + size_t len, + std::string const & name) +{ + // if you've gone past the end of the buffer, there's a logic error, + // and this program is not safe to keep running. shut down. + I(pos < str.size() || (pos == str.size() && len == 0)); + // otherwise make sure there's room for this decode operation, but + // use a recoverable exception type. + if (len == 0) + return; + if (str.size() < pos + len) + throw bad_decode(F("need %d bytes to decode %s at %d, only have %d") + % len % name % pos % (str.size() - pos)); +} + +inline void +require_bytes(string_queue const & str, size_t pos, size_t len, std::string const & name) @@ -41,6 +60,41 @@ template inline bool try_extract_datum_uleb128(std::string const & in, + size_t & pos, + std::string const & name, + T & out) +{ + BOOST_STATIC_ASSERT(std::numeric_limits::is_signed == false); + size_t shift = 0; + size_t maxbytes = sizeof(T) + 1 + (sizeof(T) / 8); + out = 0; + while (maxbytes > 0) + { + if (pos >= in.size()) + return false; + T curr = widen(in[pos]); + ++pos; + out |= ((static_cast(curr) + & static_cast(0x7f)) << shift); + bool finished = ! static_cast(static_cast(curr) + & static_cast(0x80)); + if (finished) + break; + else if (maxbytes == 1) + throw bad_decode(F("uleb128 decode for '%s' into %d-byte datum overflowed") + % name % maxbytes); + else + { + --maxbytes; + shift += 7; + } + } + return true; +} + +template +inline bool +try_extract_datum_uleb128(string_queue const & in, size_t & pos, std::string const & name, T & out) @@ -100,6 +154,31 @@ T remainder = in >> 7; bool finished = ! static_cast(remainder); if (finished) + { + out += item; + break; + } + else + { + out += (item | static_cast(0x80)); + --maxbytes; + in = remainder; + } + } +} + +template +inline void +insert_datum_uleb128(T in, string_queue & out) +{ + BOOST_STATIC_ASSERT(std::numeric_limits::is_signed == false); + size_t maxbytes = sizeof(T) + 1 + (sizeof(T) / 8); + while (maxbytes > 0) + { + u8 item = (static_cast(in) & static_cast(0x7f)); + T remainder = in >> 7; + bool finished = ! static_cast(remainder); + if (finished) { out += item; break; @@ -116,6 +195,27 @@ template inline T extract_datum_lsb(std::string const & in, + size_t & pos, + std::string const & name) +{ + size_t nbytes = sizeof(T); + T out = 0; + size_t shift = 0; + + require_bytes(in, pos, nbytes, name); + + while (nbytes > 0) + { + out |= widen(in[pos++]) << shift; + shift += 8; + --nbytes; + } + return out; +} + +template +inline T +extract_datum_lsb(string_queue const & in, size_t & pos, std::string const & name) { @@ -148,7 +248,21 @@ out.append(std::string(tmp, tmp+nbytes)); } +template inline void +insert_datum_lsb(T in, string_queue & out) +{ + size_t const nbytes = sizeof(T); + char tmp[nbytes]; + for (size_t i = 0; i < nbytes; ++i) + { + tmp[i] = static_cast(in) & static_cast(0xff); + in >>= 8; + } + out.append(std::string(tmp, tmp+nbytes)); +} + +inline void extract_variable_length_string(std::string const & buf, std::string & out, size_t & pos, @@ -174,9 +288,29 @@ buf.append(in); } +inline void +insert_variable_length_string(std::string const & in, + string_queue & buf) +{ + size_t len = in.size(); + insert_datum_uleb128(len, buf); + buf.append(in); +} inline std::string extract_substring(std::string const & buf, + size_t & pos, + size_t len, + std::string const & name) +{ + require_bytes(buf, pos, len, name); + std::string tmp = buf.substr(pos, len); + pos += len; + return tmp; +} + +inline std::string +extract_substring(string_queue const & buf, size_t & pos, size_t len, std::string const & name) ======================================================================== --- netsync.cc c5406e41e4aec3b81907d638db40e805bde7edfb +++ netsync.cc d1db1316c8a727e34dc4f285bc20d1beed0c8fc7 @@ -269,7 +269,7 @@ Netxx::socket_type fd; Netxx::Stream str; - string inbuf; + string_queue inbuf; // deque of pair deque< pair > outbuf; // the total data stored in outbuf - this is @@ -500,7 +500,7 @@ peer_id(peer), fd(sock), str(sock, to), - inbuf(""), + inbuf(), outbuf_size(0), armed(false), remote_peer_key_hash(""), @@ -1355,7 +1355,7 @@ L(F("in error unwind mode, so throwing them into the bit bucket\n")); return true; } - inbuf.append(string(tmp, tmp + count)); + inbuf.append(tmp,count); mark_recent_io(); if (byte_in_ticker.get() != NULL) (*byte_in_ticker) += count; @@ -1598,7 +1598,16 @@ L(F("queueing %d bytes of data for %s item '%s'\n") % dat.size() % typestr % hid); + netcmd cmd; + // TODO: This pair of functions will make two copies of a large + // file, the first in cmd.write_data_cmd, and the second in + // write_netcmd_and_try_flush when the data is copied from the + // cmd.payload variable to the string buffer for output. This + // double copy should be collapsed out, it may be better to use + // a string_queue for output as well as input, as that will reduce + // the amount of mallocs that happen when the string queue is large + // enough to just store the data. cmd.write_data_cmd(type, item, dat); write_netcmd_and_try_flush(cmd); note_item_sent(type, item); ======================================================================== --- string_queue.cc +++ string_queue.cc 8e101f4569d10559b9bcac8b08677d5332e96ac1 @@ -0,0 +1,78 @@ +// unit tests for string_queue + +#include + +#include "string_queue.hh" + +using namespace std; + +#ifdef BUILD_UNIT_TESTS + +#include +#include "unit_tests.hh" + +void +test_string_queue() +{ + string_queue sq1; + + BOOST_CHECKPOINT( "append" ); + + sq1.append("123"); + sq1.append("45z", 2); // 'z' shall be ignored + sq1.append('6'); + + BOOST_CHECK( sq1.size() == 6 ); + + BOOST_CHECKPOINT( "retrieve" ); + + BOOST_CHECK( sq1.substr(0, 6) == "123456" ); + BOOST_CHECK( sq1.substr(3, 2) == "45" ); + + BOOST_CHECK( sq1[5] == '6' ); + BOOST_CHECK( sq1[0] == '1' ); + + BOOST_CHECK( *(sq1.front_pointer(6)) == '1'); + + BOOST_CHECK( sq1.size() == 6); + + BOOST_CHECKPOINT( "failures" ); + + // check a few things will fail + BOOST_CHECK_THROW( sq1.substr(3, 4), logic_error ); + BOOST_CHECK_THROW( sq1.front_pointer(7), logic_error ); + + // modification + BOOST_CHECKPOINT( "modification" ); + + sq1[5] = 'r'; + BOOST_CHECK_THROW( sq1[6], logic_error ); + + BOOST_CHECK( sq1[5] == 'r' ); + BOOST_CHECK( sq1.substr(3, 3) == "45r" ); + + // empty it out + BOOST_CHECKPOINT( "emptying" ); + + BOOST_CHECK_THROW( sq1.pop_front( 7 ), logic_error ); + sq1.pop_front(1); + BOOST_CHECK( sq1.size() == 5 ); + BOOST_CHECK(sq1[0] == '2'); + + BOOST_CHECK(sq1[4] == 'r'); + BOOST_CHECK_THROW( sq1[5], logic_error ); + BOOST_CHECK_THROW( sq1.pop_front( 6 ), logic_error ); + sq1.pop_front(5); + BOOST_CHECK_THROW( sq1.pop_front( 1 ), logic_error ); + + // it's empty again + BOOST_CHECK( sq1.size() == 0 ); +} + +void +add_string_queue_tests(test_suite * suite) +{ + suite->add(BOOST_TEST_CASE(&test_string_queue)); +} + +#endif // BUILD_UNIT_TESTS ======================================================================== --- string_queue.hh +++ string_queue.hh c97430d992a71fe7f05eda42d852a93db6f04db3 @@ -0,0 +1,249 @@ +#ifndef __STRING_QUEUE_HH__ +#define __STRING_QUEUE_HH__ + +// copyright (C) 2005 Eric Anderson +// all rights reserved. +// licensed to the public under the terms of the GNU GPL (>= 2) +// see the file COPYING for details + +#include + +#include "sanity.hh" + +// a class for handling inserting at the back of a string and removing +// from the front efficiently while still preserving the data as +// contiguous; could also do the reverse, but that wasn't needed, and +// so hasn't been implemented. + +class string_queue +{ +public: + string_queue (size_t default_size = 8192) + { + buf = new char[default_size]; + front = back = buf; + end = buf + default_size; + } + + ~string_queue () + { + delete[]buf; + } + + void append (const std::string & v) + { + selfcheck (); + reserve_additional (v.size ()); + simple_append (v); + if (do_selfcheck) + { + selfcheck_str.append (v); + selfcheck (); + } + }; + + void append (const char *str, size_t bytes) + { + selfcheck (); + reserve_additional (bytes); + simple_append (str, bytes); + if (do_selfcheck) + { + selfcheck_str.append (std::string (str, bytes)); + selfcheck (); + } + }; + + void append (const char v) + { + selfcheck (); + if (available_size () >= 1) + { + *back = v; + ++back; + } + else + { + std::string tmp; + tmp += v; + I (tmp.size () == 1 && tmp[0] == v); + append (tmp); + } + if (do_selfcheck) + { + selfcheck_str += v; + selfcheck (); + } + } + + string_queue & operator+= (const char v) + { + append (v); + return *this; + } + + string_queue & operator+= (const std::string & v) + { + append (v); + return *this; + } + + char &operator[] (size_t pos) + { + I (pos < used_size ()); + return front[pos]; + } + + const char &operator[] (size_t pos) const + { + I (pos < used_size ()); + return front[pos]; + } + + void pop_front (size_t amount) + { + selfcheck (); + I (used_size () >= amount); + front += amount; + if (front == back) + { + front = back = buf; + } + if (used_size () * 3 < buffer_size () && buffer_size () > (1024 * 1024)) + { + // don't bother shrinking unless it will help a lot, and + // we're using enough memory to care. + size_t a_new_size = (size_t) (used_size () * 1.1); // leave some headroom + resize_buffer (std::max ((size_t) 8192, a_new_size)); + } + if (do_selfcheck) + { + selfcheck_str.erase (0, amount); + selfcheck (); + } + } + + std::string substr (size_t pos, size_t size) const + { + I (size <= max_string_queue_incr); + I (pos <= max_string_queue_size); + I (used_size () >= (pos + size)); + return std::string (front + pos, size); + } + + const char *front_pointer (size_t strsize) const + { + I (strsize <= max_string_queue_size); + I (used_size () >= strsize); + return front; + } + + size_t size () const + { + return used_size (); + } + size_t used_size () const + { + return (size_t) (back - front); + } + size_t buffer_size () const + { + return (size_t) (end - buf); + } + size_t available_size () const + { + return (size_t) (end - back); + } + bool empty () const + { + return front == back; + } + + void selfcheck () + { + if (do_selfcheck) + { + I (buf <= front && front <= back && back <= end); + I (selfcheck_str.size () == used_size () + && memcmp (selfcheck_str.data (), front, used_size ()) == 0); + } + } + + void reserve_total (size_t amount) + { + if ((size_t) (end - front) >= amount) + { + return; + } + reserve_additional (amount - available_size ()); + } + + void reserve_additional (size_t amount) + { + I(amount <= max_string_queue_incr); + if (available_size () >= amount) + return; + if (1.25 * (used_size () + amount) < buffer_size ()) + { + // 1.25* to make sure that we don't do a lot of remove 1 byte from + // beginning, move entire array, append a byte, etc. + size_t save_used_size = used_size (); + memmove (buf, front, save_used_size); + front = buf; + back = front + save_used_size; + selfcheck (); + return; + } + // going to expand the buffer, increase by at least 1.25x so that + // we don't have a pathological case of reserving a little extra + // a whole bunch of times + size_t new_buffer_size = + std::max ((size_t) (1.25 * buffer_size ()), + used_size () + amount); + resize_buffer (new_buffer_size); + selfcheck (); + } + +protected: + void simple_append (const std::string & v) + { + I ((size_t) (end - back) >= v.size ()); + I (v.size() <= max_string_queue_incr); + memcpy (back, v.data (), v.size ()); + back += v.size (); + } + + void simple_append (const char *str, size_t bytes) + { + I ((size_t) (end - back) >= bytes); + I (bytes <= max_string_queue_incr); + memcpy (back, str, bytes); + back += bytes; + } + + void resize_buffer (size_t new_buffer_size) + { + I (new_buffer_size <= max_string_queue_size); + size_t save_used_size = used_size (); + char *newbuf = new char[new_buffer_size]; + memcpy (newbuf, front, save_used_size); + delete[]buf; + buf = front = newbuf; + back = front + save_used_size; + end = buf + new_buffer_size; + } + +private: + static const bool do_selfcheck = false; + // used to avoid integer wraparound, 500 megs should be enough: + static const size_t max_string_queue_size = 500 * 1024 * 1024; + static const size_t max_string_queue_incr = 500 * 1024 * 1024; + string_queue (string_queue & from) + { + abort (); + } + char *buf, *front, *back, *end; + std::string selfcheck_str; +}; + +#endif ======================================================================== --- unit_tests.cc 049f2a37c29874fb26d6cb5fff1e3417303bc2e3 +++ unit_tests.cc 5088d7c068c79caf69f553a295b8dbf496b6b442 @@ -83,6 +83,9 @@ if (t.empty() || t.find("crypto") != t.end()) add_crypto_tests(suite); + if (t.empty() || t.find("string_queue") != t.end()) + add_string_queue_tests(suite); + // all done, add our clean-shutdown-indicator suite->add(BOOST_TEST_CASE(&clean_shutdown_dummy_test)); ======================================================================== --- unit_tests.hh 7213d23add3f0715300ca0348d9c16b1d136cc3e +++ unit_tests.hh fa58467ac77ae1ee84f966c620eecf917be527b7 @@ -34,5 +34,6 @@ void add_path_component_tests(test_suite * suite); void add_globish_tests(test_suite * suite); void add_crypto_tests(test_suite * suite); +void add_string_queue_tests(test_suite * suite); #endif