# # # add_dir "tests/schema_migration_error_recovery" # # add_file "tests/schema_migration_error_recovery/__driver__.lua" # content [69671b8cf2ddaeb74716c2f8d01dbe2c82a1f48b] # # add_file "tests/schema_migration_error_recovery/bad_base64.dump" # content [b6a7afa91f57cf8a1f20ac342c6acf8af8605129] # # patch "ChangeLog" # from [23e8ec657c8562daf5420597f6674400be498e54] # to [9307f0c3384b2c4ea7c19994392ffa8060ff2c3e] # # patch "schema_migration.cc" # from [e65188fd724fa36f580292185541216ce83cc281] # to [0c24ca599347dc4d5b97c97c5b95b8601dc05061] # # patch "testsuite.lua" # from [5360e416548d4328291c3c6a19f86feffc6cdd0e] # to [31665a2c0d4546535acc5470c28b2b73f827d6b3] # ============================================================ --- tests/schema_migration_error_recovery/__driver__.lua 69671b8cf2ddaeb74716c2f8d01dbe2c82a1f48b +++ tests/schema_migration_error_recovery/__driver__.lua 69671b8cf2ddaeb74716c2f8d01dbe2c82a1f48b @@ -0,0 +1,13 @@ +-- This test exercises monotone's ability to detect and recover from errors +-- while migrating to new database schemata. + +function test_one(tag, diagnostic) + dump = tag .. ".dump" + db = tag .. ".mtn" + check(get(dump)) + check(raw_mtn("db", "load", "-d", db), 0, nil, nil, {dump}) + check(raw_mtn("db", "migrate", "-d", db), 1, nil, true) + check(qgrep(diagnostic, "stderr")) +end + +test_one("bad_base64", "invalid base64 character") ============================================================ --- tests/schema_migration_error_recovery/bad_base64.dump b6a7afa91f57cf8a1f20ac342c6acf8af8605129 +++ tests/schema_migration_error_recovery/bad_base64.dump b6a7afa91f57cf8a1f20ac342c6acf8af8605129 @@ -0,0 +1,307 @@ +BEGIN EXCLUSIVE; +CREATE TABLE branch_epochs + ( + hash not null unique, -- hash of remaining fields separated by ":" + branch not null unique, -- joins with revision_certs.value + epoch not null -- random hex-encoded id + ); +CREATE TABLE db_vars + ( + domain not null, -- scope of application of a var + name not null, -- var key + value not null, -- var value + unique(domain, name) + ); +CREATE TABLE file_deltas + ( + id not null, -- strong hash of file contents + base not null, -- joins with files.id or file_deltas.id + delta not null, -- rdiff to construct current from base + unique(id, base) + ); +INSERT INTO file_deltas VALUES('a9ca701697adae066b96d07aabb30f0d6245692c','d4929f246d23a51eba6799685e28f9ab077b483a','H4sIAAAAAAAA//NUMOVKMywz5OICAOrIolkKAAAA +'); +INSERT INTO file_deltas VALUES('36f92840dcffa22064b2dd9e0848d14350f07c5c','f9d518a4e1308cbe8503bdd8f578b16de4407491','H4sIAAAAAAAA//NUMOVKMyoz5OICADqyAh4KAAAA +'); +INSERT INTO file_deltas VALUES('09848c4631a20ac166344f58a23fee04a6c646a4','1ece609689fb9462de25716110769bad1a80e8d8','H4sIAAAAAAAA//NUMOVKMykz5OICAJpHQpEKAAAA +'); +CREATE TABLE files + ( + id primary key, -- strong hash of file contents + data not null -- compressed, encoded contents of a file + ); +INSERT INTO files VALUES('d4929f246d23a51eba6799685e28f9ab077b483a','H4sIAAAAAAAA/0szLDPiAgC5Qx7FBQAAAA== +'); +INSERT INTO files VALUES('bbeadf8e35428c9e5333e71caf25851498306eb6','H4sIAAAAAAAA/0szLjPkAgDx2DpEBQAAAA== +'); +INSERT INTO files VALUES('f9d518a4e1308cbe8503bdd8f578b16de4407491','H4sIAAAAAAAA/0szKjPiAgBX7KvXBQAAAA== +'); +INSERT INTO files VALUES('1ece609689fb9462de25716110769bad1a80e8d8','H4sIAAAAAAAA/0szKTPiAgCLs8DyBQAAAA== +'); +CREATE TABLE manifest_certs + ( + hash not null unique, -- hash of remaining fields separated by ":" + id not null, -- joins with manifests.id or manifest_deltas.id + name not null, -- opaque string chosen by user + value not null, -- opaque blob + keypair not null, -- joins with public_keys.id + signature not null, -- RSA/SHA1 signature of "address@hidden:val]" + unique(name, id, value, keypair, signature) + ); +CREATE TABLE manifest_deltas + ( + id not null, -- strong hash of all the entries in a manifest + base not null, -- joins with either manifest.id or manifest_deltas.id + delta not null, -- rdiff to construct current from base + unique(id, base) + ); +CREATE TABLE manifests + ( + id primary key, -- strong hash of all the entries in a manifest + data not null -- compressed, encoded contents of a manifest + ); +CREATE TABLE next_roster_node_number + ( + node primary key -- only one entry in this table, ever + ); +INSERT INTO next_roster_node_number VALUES('5'); +CREATE TABLE public_keys + ( + hash not null unique, -- hash of remaining fields separated by ":" + id primary key, -- key identifier chosen by user + keydata not null -- RSA public params + ); +INSERT INTO public_keys VALUES('de84b575d5e47254393eba49dce9dc4db98ed42d','address@hidden','MIGdMA0GCSqGSIb3DQEBAQUAA4GLADCBhwKBgQC54vVjrrqYoTfPTgWm6JpuL+kOERcN2OSc +BsWq6cb4Wm3nlymwVqJJywq6cbfygUYwmqyiRLPxRosfLGu228AhEzaM4JbAH1pgg7CwvvVd +fHRXNAXEMgO89gBjkkecxLi4U/T67DrLjkRPAilCgWLZNv8YeOG9XAPegWyr7hNA9wIBEQ=='); +INSERT INTO public_keys VALUES('c9d80250e944708aab7fe960c1136b517fd30772','address@hidden','MIGdMA0GCSqGSIb3DQEBAQUAA4GLADCBhwKBgQCfN/cAMabgb6T7m8ksGnpQ7LO6hOdnc/7V +yivrRGtmpwSItljht1bmgLQF37KiSPoMEDUb1stfKxaMsYiy8iTyoQ+M2EVFP37n2rtnNZ0H +oVcQd2sRsCerQFh9nslRPymlkQXUlOiNFN6RlFNcdjkucqNe+YorFX21EYw7XuT5XwIBEQ=='); +CREATE TABLE revision_ancestry + ( + parent not null, -- joins with revisions.id + child not null, -- joins with revisions.id + unique(parent, child) + ); +INSERT INTO revision_ancestry VALUES('','bf468e6c22dec9203af6441ad7d20b6ad8af049a'); +INSERT INTO revision_ancestry VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','c81722b0236303685e341e16f0073d665090fb73'); +INSERT INTO revision_ancestry VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','43a2235616452dca74eecf39d645a69da8e0bdd0'); +INSERT INTO revision_ancestry VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0','4a1274f35812a695e357c6e7c7cd60f449f0cada'); +INSERT INTO revision_ancestry VALUES('c81722b0236303685e341e16f0073d665090fb73','4a1274f35812a695e357c6e7c7cd60f449f0cada'); +INSERT INTO revision_ancestry VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada','75810233cc39b62341d669b610e9416fd6352869'); +CREATE TABLE revision_certs + ( + hash not null unique, -- hash of remaining fields separated by ":" + id not null, -- joins with revisions.id + name not null, -- opaque string chosen by user + value not null, -- opaque blob + keypair not null, -- joins with public_keys.id + signature not null, -- RSA/SHA1 signature of "address@hidden:val]" + unique(name, id, value, keypair, signature) + ); +INSERT INTO revision_certs VALUES('6f938572483d4f73bf952c3666dace09f95ebd50','bf468e6c22dec9203af6441ad7d20b6ad8af049a','branch','dGVzdGJyYW5jaDE= +','address@hidden','CjgQqP1r/1DVkgSCaz7jKvgdPJ1WJ3EbC8jyeARxqs2w1tgM7iAGNs0961Y8+rBVZuCBcGLl +S/W1F1ZVAlseVT2NmBVOhu0OlmWhZ2V1JPklyIkFk3krJjSJxP1bt8D6IHCVxxdnhEDUrf8O +Cc4Z0gOzDznJ2qUnFfM3ZZUCAjI= +'); +INSERT INTO revision_certs VALUES('6ac4524843235b44ca2dfc2696d38f2b90239109','bf468e6c22dec9203af6441ad7d20b6ad8af049a','date','MTk5OS0wMS0wMVQxMjowMDowMA== +','address@hidden','ZBHfnu/Gi6S90RN4GpIuQsflSL2JPU3QISuVIejxWCxK54V6zieOZ6ZHI8GECfWCqJWtD3L+ +wFEhpgg3oSSsZrQRM8mdpqZM5sTEOKja5td72dPkISp0ysJE4KLmuDVv88aSCrcsXDxyEZU8 +jUEGaii+JwAfFdP4OTrfL2sH1JI= +'); +INSERT INTO revision_certs VALUES('d77b687d8c619078e80721658a6d99dcfe7e32e5','bf468e6c22dec9203af6441ad7d20b6ad8af049a','author','dGVzdGVyQHRlc3QubmV0 +','address@hidden','HZsa3yyBrQGi+Gl4/jL6pJhrjz6ef3ASrg6YNjONW5ypegj2DIcZpQN4jLGJGKzTPQUI+RBg +SFn3G2dTEA1T/ul8STkuIc/mqZmKvMeMPStq3ezjSKYCyLey5QbTvm0HkpISY3nTmsXetcsc +mhBDuUA9GjZHgj9CIEVOrYfkKQc= +'); +INSERT INTO revision_certs VALUES('2dda1c13436be7c2f10271c210a2bd0c885f313c','bf468e6c22dec9203af6441ad7d20b6ad8af049a','changelog','YmxhaC1ibGFo +','address@hidden','NKIdKEogyd3DKwjJfUk0BoqfgRfE8e5HrRahLsMSf/j6XD8XQ00qVW9hmrn/CaBTdByZ5ZZu +TH+ByLcXDAHVhITHYaRALxTx34Wl+GpPBdguXgI3hgy5V+FI9JN1m2cBl89nkVG6GW0o78nG +Fo1x73vpmYqa5WPWwCrAOwJm7ZQ= +'); +INSERT INTO revision_certs VALUES('c50226beb7e3d289d38e9f613021c380f89ab011','bf468e6c22dec9203af6441ad7d20b6ad8af049a','somekey','c29tZXZhbHVl +','address@hidden','GAfv6ovoLebquWhqqUnZmHCpnzdLCfneH4mWmfIajdivhBu+hPFV6yi++xSv83crepDtVJYe +Fd1RI50PYZntUF4rzW3JOOlJlwAhairzx0saEpHJiSY+zBLNXDWbsFRnDwWwACd5TsSwdPVV +AF4ZtPvUfCg2oG4qinL+JoUNcWI= +'); +INSERT INTO revision_certs VALUES('628a294256cbb30fa29665cba0ce9a58c02e57b0','c81722b0236303685e341e16f0073d665090fb73','branch','dGVzdGJyYW5jaDI= +','address@hidden','l8RLneTmbvJraUlwzqqNi4pjoMDIV4rQlR2ShOIbmFR1FjyOvRaN6Q4rj36kYaeMlVasCjHR +K5dMMnmhEGcjwUmtc6z4+HKbvO5BjbHC7HeMsc7e+nKpz94cmbVVT0fJ5/vCqzI3awnKi3jT +9TT9v384x5OTWhukmV7C7VSwrns= +'); +INSERT INTO revision_certs VALUES('70c013d2ba0e97f5e706dcc27e8eb32920f09c6f','c81722b0236303685e341e16f0073d665090fb73','date','MjAwMC0wMS0wMVQxMjowMDowMA== +','address@hidden','L77126IRXEDbZjhtv5FFNoTF2zxZdCTWMb0r9X2FD//BX5uPPfQLR70dk19KzkZPHcWzNyRd +iwcSYJmDSuV7blTtLvrvG66RjLcZRHJdrBL6u7heyhMqL+7lWJq273aFvvP+XjbuJq5LbXyr +jQeu69/Demh7LWJlnzfwel4KUgo= +'); +INSERT INTO revision_certs VALUES('e70e1cc3843fe732d6ee20d25749f98c8862166b','c81722b0236303685e341e16f0073d665090fb73','author','dGVzdGVyQHRlc3QubmV0 +','address@hidden','hqMs692uyOaM7i3vI71OrU/0Y0a+wmFBnIs5kfH/iJqH7phcRYm34WDFFnZCLaKuBkzaj5wu +6ftkEPwNYFwlhzKzoGf1XkNGj50vGuo2ZR7ksAO6gtQLhU8FSBUhc1im6tmBvFDWQa0h9J9g +b4mifWttew/gS8b5Sv3pq4FxewI= +'); +INSERT INTO revision_certs VALUES('55d537c81d3e6db852813076cbb476aa7bfb8e6d','c81722b0236303685e341e16f0073d665090fb73','changelog','YmxhaC1ibGFo +','address@hidden','MprtSWnltZ1+tRKIwOku8QM0+yJA1x22UiFuDgnF3iVuR3lePSNWilzObMzGuYRLvbJ5fXY9 +blnKDPZe8JlCQVgamunYCjpok4u8SEcSa0abGvCPSfIjX8UY4YBY+hNj0zmQBfrVvOORwSVR +SUQmoqylhIUgDr1+azI7w8OgEXM= +'); +INSERT INTO revision_certs VALUES('b8207be46ed175205466998210bab3b6c30fa06b','43a2235616452dca74eecf39d645a69da8e0bdd0','branch','dGVzdGJyYW5jaDE= +','address@hidden','aOq1Ecb9GZfbcUHQyRcbLa+AeEs4dfnAUyP008eIYkt2tX+nHgoEPiY8k87cpo8KurjDZlt7 +6Rjom0NEH8vkfml57WbcejYxq/7TSG7qlIiQ0uKmkON+sT6MR8k+1yCcmkK4e0pjfCtK/TCh +Ac7oy1iF0WgbfS6dmQ3zPpmqrGI= +'); +INSERT INTO revision_certs VALUES('c2e926d430da9903c918ee15b2b2dfe99ee58463','43a2235616452dca74eecf39d645a69da8e0bdd0','date','MjAwMS0wMS0wMVQxMjowMDowMA== +','address@hidden','CKFR6o8w5ewVdO27QwTs2Q0MvnZSWWtWrnJO/FNGre0CTpVwAo0cj1ZfAt4oQYN9+bYOiFJv +so886hP7b65h/gQlBjY4hVTKgsXfW+CYhJcO/xicA8dSuHyGQMqWzd0wKkORRassF8FYQ7aN +w9I2aW54oWHc5kYHlnkNnFcnxCw= +'); +INSERT INTO revision_certs VALUES('eaa39fc163b0d38cded5270f60bc9570e7f4e1b0','43a2235616452dca74eecf39d645a69da8e0bdd0','author','dGVzdGVyQHRlc3QubmV0 +','address@hidden','OsgLCmG7U9ZPHM4QgiwNFHB9taJmo1F08bCNXO7vUNhTJygeR6by6jlxIZzZLdVwlDZ2QlNY +l4KluIftqFpYj81w0gxA0cNgf2YSfpLDUU6YlsmvjtGhUDu8m2EX46oDhWC+kV1uDhQjuZQW +niDaS0V3tAr7151DiBU4CivDoCM= +'); +INSERT INTO revision_certs VALUES('9d9472e1be171031fee0ff82f41d752e8124742e','43a2235616452dca74eecf39d645a69da8e0bdd0','changelog','YmxhaC1ibGFo +','address@hidden','CZdbbLXvcETdez6wt9Jz1l+IX5b8xOcEcgNNO4IhE5qL92jY0uJYD19YT7krv4wSEyqGr+UM +vS71Lns1A87MnvaaYqLDzWJKLfpKCTLbXmvXkxVFot+d1NjrapHyxDWkzsWYvpMDv+QsLFjB +JYBFpdy5EDj1g5cKN2+dbYojLWg= +'); +INSERT INTO revision_certs VALUES('884d60e16e31cf3621191bd2dab0caf39a0283fe','4a1274f35812a695e357c6e7c7cd60f449f0cada','date','MjAwMi0wMS0wMVQxMjowMDowMA== +','address@hidden','DPd17dv2lhHUrM5XBA5/28ygytUeoHs12/SDB9mzoUpfSByEAgR6ONCy8RRcboz58V/5sc02 +gMDwaQ6VFpMqeQs8mQ1ng/Y3RCbPhgDwilatA4vrn7bn4J3vxuTr8giy9m1R6EDAGIdLBt6y +shx9joHN4G7zutOmRgtr7U5xYUY= +'); +INSERT INTO revision_certs VALUES('2724dd7883fb44a58d40eba12808ed5b9e09f7b6','4a1274f35812a695e357c6e7c7cd60f449f0cada','author','dGVzdGVyQHRlc3QubmV0 +','address@hidden','bKe9iSJ7+z3xoq1GqC4VvO4DqnIaYKK3EGdXPxtk4xRdIGNkNRbcwIr9+YvxknZaCBhyDJQy +q+5Smfw34N59CEO5CcfylWcAS4gcpmjDNWFMDDxGxof/vN+OwDQTNFRpAn6PVPnuHbKkBs6J +F3fuS5ooLxfVLKF6Iu8ljXcyPtI= +'); +INSERT INTO revision_certs VALUES('1288aa1a3c351fd0528aa95ddaae309cc5eec3e2','4a1274f35812a695e357c6e7c7cd60f449f0cada','branch','dGVzdGJyYW5jaDE= +','address@hidden','l2qifuBN5jBbGr04fu/xC7HGbp1h890kboHv7VVw+rejuWM46DLluAE4NaoaE1qxlAZmWiCV +l382q/q9vAR4nWFkmsXInj1myKGyvK2CHpTtaxDSUAORNnrkYOOc4uSbjRWuiQtefhSUNWyx +1WgxAsCSsEksUkE3gntXBcGZ5Mw= +'); +INSERT INTO revision_certs VALUES('27e8a71b0fa56a551e72aed26861ba404d02fd32','4a1274f35812a695e357c6e7c7cd60f449f0cada','changelog','cHJvcGFnYXRlIGZyb20gYnJhbmNoICd0ZXN0YnJhbmNoMicgKGhlYWQgYzgxNzIyYjAyMzYz +MDM2ODVlMzQxZTE2ZjAwNzNkNjY1MDkwZmI3MykKICAgICAgICAgICAgdG8gYnJhbmNoICd0 +ZXN0YnJhbmNoMScgKGhlYWQgNDNhMjIzNTYxNjQ1MmRjYTc0ZWVjZjM5ZDY0NWE2OWRhOGUw +YmRkMCkK +','address@hidden','ilQqoBOBBjdPbRC0UOplwSuv1wHAv4HKAyZL40QspeWv/qlG3fqYZTRfOcXiG/Ey43/lRUPg +Q9KqaNmDAfD1sQzBLiWmpdi31Qdzb2XI383VSc2kZxwwrK7GzCc7WXTWW6Mm9vyUb7N4Xo1U +Tc7Y3obkFE+VPmYLTjD+6S1xwf0= +'); +INSERT INTO revision_certs VALUES('5b44cd5cd32f3e8f268ac1cd30ccfa27d16667c6','75810233cc39b62341d669b610e9416fd6352869','branch','dGVzdGJyYW5jaDM= +','address@hidden','AgZdq7S2Skb6vekOGl/GknzDlVsCsQShdAClWGSjxfNeu8ISaO91MgFXeT1QQZZ9Vmu2ln9H +8YFu5NlAlcgL1xX2d8JMTDadlnk48xWxKhLS2pKNpFzfA5LS6yO5t2NOVGEI5016SfU3UWxD +pWg/TyjvXAl9LHkve8bT2k6wGek= +'); +INSERT INTO revision_certs VALUES('b0f324986982bdf443036f061977a7895ae94f84','75810233cc39b62341d669b610e9416fd6352869','date','MjAwMy0wMS0wMVQxMjowMDowMA== +','address@hidden','O3ORIe1YVt/fEz0uHf4wZKZ2K7HfjGJHeSPyhf1RGfdSCv2pfWhfGlfagM2V1Lzsyl4TkikZ +YS+H+6sC6FbN32zpeE/9+iisdaRHGACWXdOebKWdEJaBGRTNc6w3ablmkhvXAqK36FjFJSbj +wD1XGZbqq/VilnTZ5cr5GqYRGNs= +'); +INSERT INTO revision_certs VALUES('7a8a589d30fc17ee317ff9bacecac0d30524003a','75810233cc39b62341d669b610e9416fd6352869','author','dGVzdGVyQHRlc3QubmV0 +','address@hidden','H/C6jJ1fyAdu3QF+rLt3FuhWJm5lsapfV8mYkETKuu2gjbTecx3So8MCsP2WhE+Itvc7LMu4 +j0oAk9VePu5q4o1h3x5Nezc8zx9L2igJqko9HFenu6QAj3SbE6AUMYNolk+nmAj/jUPvT3R8 +1/+SlGe42S1TzlDxTv8Exk7UcUY= +'); +INSERT INTO revision_certs VALUES('98640622c14a2ecbe23b7251b960d8bc78dec4a2','75810233cc39b62341d669b610e9416fd6352869','changelog','YmxhaC1ibGFo +','address@hidden','Xrbz1t1qJYpFKoWfRP1ijeGXiS3Y0Mxy/address@hidden@$()lb8du939pUuv +jtKDDdK/nm5Byo+svmz6eP0orqLJIh5VIAmY9tN8h78eMdLrm+yUmN41OwVxHv3oNrUbwo3S +euGYk7NjxhWm+XFPoQbayRLZfgM= +'); +CREATE TABLE revision_roster + ( + rev_id primary key, -- joins with revisions.id + roster_id not null -- joins with either rosters.id or roster_deltas.id + ); +INSERT INTO revision_roster VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','e63f3fbaa205e1d3117d3ee9b056b1edcd95cbbe'); +INSERT INTO revision_roster VALUES('c81722b0236303685e341e16f0073d665090fb73','330548314b7c980863e65195d0f82aab4cd7e355'); +INSERT INTO revision_roster VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0','cb81e62f9e18a8b3f16fb714fd8b564813e5ab28'); +INSERT INTO revision_roster VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada','74185a2d2c138df730096e0e92b8e43dfec2aa94'); +INSERT INTO revision_roster VALUES('75810233cc39b62341d669b610e9416fd6352869','c8b35398ca2282fb9b893c47245920870a475c4b'); +CREATE TABLE revisions + ( + id primary key, -- SHA1(text of revision) + data not null -- compressed, encoded contents of a revision + ); +INSERT INTO revisions VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','H4sIAAAAAAAA/32PUYrDMAxE/3MK4xM4jq3YZynFKJYEZlMHUjfL3n6dFko/lv0bCc2bkWz7 +DVs6eL+XrSo96mGo/J1uWIvwvanL6HkiDxkmycYAGRcEA2UHQnbx3rEQxjBfh2FbKe18lCfq +0hdIlKjsSuuXlrKy0q1zT9WzVN5q49pjMGaczQhxRkI2AEvsWTPiskxGDIF1HqLN179I9pM0 +gUQbnKEsgtYacIslimyCCzS6yXfcnP1JUurO7WynsLVes3dNX/yjhwPXR+eXWlrBNT1H/b7/ +fOBlfG/+s/8CTv6BAGwBAAA= +'); +INSERT INTO revisions VALUES('c81722b0236303685e341e16f0073d665090fb73','H4sIAAAAAAAA/1VQW27EIAz85xQoJwADBs4SrSKDjRo1m1QJ3aq3b7KV+vgbeeR5tW2/U58e +sh/zturBDkqt8jHdaZ2bHF2PJTp0JMBRPGAhSGDAJK4tBrKlXhQbgJtS28LTLo/5KTWW5jEJ +VgCWmsE4aui9JY4MpiBxomZ8pvORmKc2L6KHfnpeyA1K123tsl4RihC3JC54SDVLcM5JtJUa +hBSsz8kZlIKn0hv1+vIrc9bRbd/ueqRcKRqLORKTGMSSkU0kKsWZZhjBB8xQb0pr3Tc9ss+Q +G3hkcBSsFMKYM6YgkFqmYmIsPrkrfl2E9v+m1Pvfy/Qqn+eyWh/S9fDD87x/Mw9a3s/21/JP +OKgvDhQZ/pkBAAA= +'); +INSERT INTO revisions VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0','H4sIAAAAAAAA/0XPQW7EMAgF0H1OYeUEGGNinyWqImxAjTSTVJloev263XTH5r//8fN6yr29 +7Xrt5xHmOE/TYd/bU47d7XWH1UDRoqbc2oIgtWBsS+kZIRfmxtAxdVzixzSdD90ue+9/1Nqc +uBh3RLVeEZI4E0XRRREaixZxoCojKKqb7w8L8z06fy+ap9DP47ZjTIBaqHTiFGUs6JE5EXku +gsnNgIQ7EwsN6Uvu/vnP4GD8Op9hTewVC4F2d0EEpoaq1WDQGillcFh67h9TCOE+w+pVcyxC +FhOU3qxkSE21eF5Ki6xGBAvV8fcPP3EP60QBAAA= +'); +INSERT INTO revisions VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada','H4sIAAAAAAAA/22S3Y4bIQyF7+cp0DyBwcaYZ4lWkcFGjZrNVJNpqr59yVba3f7cGSzOOf7M +2PZXPc4P3++X7RbWuC7LzX+cX/V2GX4/wql6qzU69gaxinjuERhLL2OwcSQRGZIgvizLdrXz +7o/Lm9SJUFPCzJEpJ+tayL0PrDbPytVUHJoZzIdqdh6Xq4f1mJ7PCtcl9O12+G1GaM3Vhjhm +StKrZ0T0EruOlCVHqoLA3ngqfdOjf/mQmeOEsW+v4aR1JoDItaipA3OrbFBUW0MYYJwoc039 +ZQkhHFs4GdVURyK2hJqjN+VSK0v2JKNqg1IaCeo07VfX/U9TPY7PN+ev/nOSDeHuR1jf+3bZ +f3ceev0+p3+SfyvXv2F2iSWlBgkZAZ8pkKJHHgAFjTlDhdEK/hcmfYYJVUg6MUZNoD0yI9HI +ogmHO5ByZ2Klf2Gmd5jIoyYhsD7GXDIwtWRWHaa0RcI8iZaeP2COajmKkkcE6c0lA87dy8hF +WmRzIihU5yf6Bb/ZGwSRAgAA +'); +INSERT INTO revisions VALUES('75810233cc39b62341d669b610e9416fd6352869','H4sIAAAAAAAA/0WOS27DMAwF9zqF4BNQFkVRZzECg5JI1IA/hSMk12/aTbcPmJln133IWF96 +P7fr9FOYnDv1vR5ybqbP4ZeeEDkkwciZGCDWZia1citSsWsWxgIED+euva+3vrY/1YIS5owW +E4dZqCSNKTfS3HLrBIZYDJp0+YBddx3qp/Ep2rbr74tvGe3rf8LJebuvwy9QGLkhxSAzSAtE +EdESyxxNFVCoEZLgw3nvx+WXoE0JCnGxWpDmrnPKgUKATKVKD8Kg3PnhfgC2AzoCDgEAAA== + +'); +CREATE TABLE roster_deltas + ( + id not null, -- strong hash of the roster + base not null, -- joins with either rosters.id or roster_deltas.id + delta not null, -- rdiff to construct current from base + unique(id, base) + ); +INSERT INTO roster_deltas VALUES('e63f3fbaa205e1d3117d3ee9b056b1edcd95cbbe','330548314b7c980863e65195d0f82aab4cd7e355','H4sIAAAAAAAA/5WPTWoDMQxG9z6F8AkkjSOP11n1DCEM8h+YTKaQuIXevjN0GgLdNFoJCX1P +7wgIMpo38KYtrTedzRE8AQ2HdUijM7E6GYsk5lxSYBy0inOk2WfGKJpHreiCno2BvWqbC9he +7n3ryP4s0vvSy9LhpCGpR5LgNWtBkRgko1eNccCKWdgdJHA6/wa2vN1Ztg+C9n57IkyX8mXB +7gbTp84fZfUYvAPizY6IXxDZAdNVb5c/lNN/c9YHhBywBPMNbVnlwmYBAAA= +'); +INSERT INTO roster_deltas VALUES('e63f3fbaa205e1d3117d3ee9b056b1edcd95cbbe','cb81e62f9e18a8b3f16fb714fd8b564813e5ab28','H4sIAAAAAAAA/xXLsRUEIQgFwJwqKAHxixpvtEVsgCD9l3DvJp+HhW0avQyhbrV1QTKqXFUM +RzP3lYWVDX1IyYwR9PBsxg3j/5ROwda1UM0bW6V7GdA8Z6oc81xegu0f0Q+YPBTccAAAAA== + +'); +INSERT INTO roster_deltas VALUES('cb81e62f9e18a8b3f16fb714fd8b564813e5ab28','74185a2d2c138df730096e0e92b8e43dfec2aa94','H4sIAAAAAAAA/5WPTWrEMAxG9z6F8AksWSPH61n1DMMQ5D8wk6Yw4xZ6+yY0LYVuWq0+JKSn +dwYHMpknCKavfXRdzBkCAvrT1sSJTWosU5VMVGqO5Lw2YUYtoZBLomXS5jjq1Rg4qvWlgh31 +MfaE9nOQX9ZR1wEXjVmDQ4lBi1YnkqIUF1RT8q65IsQniZSvXwd72fcs2W+CjnH/QZhv9d2C +PQzmN11e6+bhAwPSbodI/xA5APOz3m+/KJe/3tkeEGSgSFtCDB6I2XwA8hU3MHEBAAA= +'); +INSERT INTO roster_deltas VALUES('330548314b7c980863e65195d0f82aab4cd7e355','74185a2d2c138df730096e0e92b8e43dfec2aa94','H4sIAAAAAAAA/yXLsREDMQgEwFxVUAJC6B7ij1wGAtF/CfaMN9+XmLAxPqQ8FtrFlCu7Q4Sh +R6r8sqnV1LW5+cmd4yU4aOr+v9MKu0iRuunCKxqqM+op4YMoi2b1+D2zSQIfX8XZRk14AAAA + +'); +INSERT INTO roster_deltas VALUES('74185a2d2c138df730096e0e92b8e43dfec2aa94','c8b35398ca2282fb9b893c47245920870a475c4b','H4sIAAAAAAAA/03NPW7DMAyG4V2nIHwC/omS5kw9QxEYtH6AII0DJFp6+9rtUm4c3ue7AAIn +Ch9AKIGWAOfV5z77PuGzaeEyWK2xeKS+uaVSLMfOeRTfMKVNs/j1rwO4tbNbeAnt+Xr4Plef +8wXL7O85bl+d1nv/DhcQjUCcj12LoWZKzBuymKCcuih1soGYpJlFLDi2JL8rp7c+/HX/hx4g +lQLGcoCKAUvWXNWEnNErmYnqiNlZRu+obtXUXI8uGwNpPDsOKs4s0cg0cquetPc6pLTjdyvN +c8etNbyG8APg9u/sOQEAAA== +'); +CREATE TABLE rosters + ( + id primary key, -- strong hash of the roster + data not null -- compressed, encoded contents of the roster + ); +INSERT INTO rosters VALUES('c8b35398ca2282fb9b893c47245920870a475c4b','H4sIAAAAAAAA/6WSYWojMQyF/+cUw5xAsmxZPksIQbZlOrRNymTaZW9fJ00oGVhIt/4l+fGM +3ie34/yqy/7D5tN0PAwjjpvNcDl1modxvDRTtcNy0S6KLkuXur5/tr/jMB7sz/5DX97tS8/T +vDwN29w8i3FxrlpJDkgbe49aY3WQWatoA590t3nT5Wn/qvPzT0znIb5M35Nsi2B0LoMjJiCW +YOTRkBtApMocIEHLkXa3kMPQphcbxsVOy7ly14jleFjOmbct1YCi3pBASjYJQLlWaSFKRq7m +PUSfcHd78MqKxtvFj3F0z38Ruc589XlS5ygwsg+uFo3erDRKtffKqaoY9CDwTxS0RpGzaW1i +FLyTkiwQkUUs2lyQgD4JAVvmNQq/RvH4lu5RPO67R/HrX+HXKNCKMSSW1HLy3FfjQkRGhMgp +a0UVMKmyRhHWKB7f0j2Kx333KGIQ7ByoFEqZXefQ8/cKwZLvRCpTcMJpt/kEwROOuxkEAAA= + +'); +CREATE INDEX revision_ancestry__child ON revision_ancestry (child); +CREATE INDEX revision_certs__id ON revision_certs (id); +CREATE INDEX revision_certs__name_value ON revision_certs (name, value); +COMMIT; ============================================================ --- ChangeLog 23e8ec657c8562daf5420597f6674400be498e54 +++ ChangeLog 9307f0c3384b2c4ea7c19994392ffa8060ff2c3e @@ -1,5 +1,18 @@ 2007-01-13 Zack Weinberg + * schema_migration.cc: Replace logged_sqlite3_* with a new, tidier + class-based API. (logged_sqlite3_exec remains for transitional + purposes.) Globally change "sql" local variables to "db" to avoid + conflict with new "sql" class. Introduce RAII class to ensure that + database rollback happens when necessary. Report errors with + exceptions (migrator functions have not been fully updated for this). + Take the database lock at the beginning of migrate_monotone_schema, + not after calculating migration steps. Trap Botan errors in + sqlite3_unbase64_fn. Give helpful additional message for more + sqlite error codes. + * tests/schema_migration_error_recovery: New test. + * testsuite.lua: Update. + * sqlite/pager.c, sqlite/btree.c: Apply upstream change #3563 - fixes crash when db is writable but directory isn't. * database.cc (assert_sqlite_ok): Give helpful additional message ============================================================ --- schema_migration.cc e65188fd724fa36f580292185541216ce83cc281 +++ schema_migration.cc 0c24ca599347dc4d5b97c97c5b95b8601dc05061 @@ -33,101 +33,173 @@ using std::vector; // in this file is easier to write and understand if it speaks directly // to sqlite. -// we do not use sqlite3_exec because we want the better error handling that -// sqlite3_prepare_v2 gives us. +// Wrappers around the bare sqlite3 API. We do not use sqlite3_exec because +// we want the better error handling that sqlite3_prepare_v2 gives us. -static sqlite3_stmt * -logged_sqlite3_prepare(sqlite3 * sql, char const * cmd) +namespace { - sqlite3_stmt * stmt; - char const * after; + struct sql + { + sql(sqlite3 * db, int cols, char const *cmd) + : stmt(0), ncols(cols) + { + sqlite3_stmt * s; + char const * after; - L(FL("executing SQL '%s'") % cmd); + L(FL("executing SQL '%s'") % cmd); - int res = sqlite3_prepare_v2(sql, cmd, strlen(cmd), &stmt, &after); - if (res != SQLITE_OK) + if (sqlite3_prepare_v2(db, cmd, strlen(cmd), &s, &after)) + error(db); + + I(s); + I(*after == 0); + I(sqlite3_column_count(s) == ncols); + stmt = s; + } + ~sql() { - L(FL("prepare failure: %s") % sqlite3_errmsg(sql)); - return 0; + if (stmt) + sqlite3_finalize(stmt); } - //I(stmt); - //I(after == 0); - return stmt; -} + bool step() + { + int res = sqlite3_step(stmt); + if (res == SQLITE_ROW) + return true; + if (res == SQLITE_DONE) + { + L(FL("success")); + return false; + } + // Diagnostics from sqlite3_result_error show up in sqlite3_errmsg + // only after sqlite3_finalize or sqlite3_reset are called on the + // stmt object. See SQLite ticket #1640. + sqlite3 * db = sqlite3_db_handle(stmt); + sqlite3_finalize(stmt); + stmt = 0; + error(db); + } + int column_int(int col) + { + I(col >= 0 && col < ncols); + return sqlite3_column_int(stmt, col); + } + string column_string(int col) + { + I(col >= 0 && col < ncols); + return string(reinterpret_cast + (sqlite3_column_text(stmt, col))); + } -// this function can only be used with statements that do not return rows. -static int -logged_sqlite3_exec(sqlite3 * sql, char const * cmd, - void* d1, void* d2, char **errmsg) -{ - I(d1 == 0); - I(d2 == 0); + // convenience API if you don't need to get rows + static void exec(sqlite3 * db, char const * cmd) + { + sql stmt(db, 0, cmd); + I(stmt.step() == false); + } - sqlite3_stmt * stmt = logged_sqlite3_prepare(sql, cmd); - if (stmt == 0) - return SQLITE_ERROR; - //I(sqlite3_column_count(stmt) == 0); + static int value(sqlite3 * db, char const * cmd) + { + sql stmt(db, 1, cmd); - int res = sqlite3_step(stmt); + I(stmt.step() == true); + int res = stmt.column_int(0); + I(stmt.step() == false); - //I(res != SQLITE_ROW); - if (res == SQLITE_DONE) + return res; + } + + // convenience for making functions + static void create_function(sqlite3 * db, char const * name, + void (*fn)(sqlite3_context *, + int, sqlite3_value **)) { - // callers expect OK - L(FL("success")); - res = SQLITE_OK; + if (sqlite3_create_function(db, name, -1, SQLITE_UTF8, 0, fn, 0, 0)) + error(db); } - else if (errmsg) - *errmsg = const_cast(sqlite3_errmsg(sql)); - sqlite3_finalize(stmt); - return res; -} + private: + sqlite3_stmt * stmt; + int ncols; -static void NORETURN -report_sqlite_error(sqlite3 * sql) -{ - // note: useful error messages should be kept consistent with - // assert_sqlite3_ok() in database.cc - char const * errmsg = sqlite3_errmsg(sql); - char const * auxiliary_message = ""; + static void NORETURN + error(sqlite3 * db) + { + // note: useful error messages should be kept consistent with + // assert_sqlite3_ok() in database.cc + char const * errmsg = sqlite3_errmsg(db); + int errcode = sqlite3_errcode(db); - L(FL("sqlite error: %s") % errmsg); + L(FL("sqlite error: %d: %s") % errcode % errmsg); + + // Check the string to see if it looks like an informative_failure + // thrown from within an SQL extension function, caught, and turned + // into a call to sqlite3_result_error. (Extension functions have to + // do this to avoid corrupting sqlite's internal state.) If it is, + // rethrow it rather than feeding it to E(), lest we get "error: + // sqlite error: error: " ugliness. + char const *pfx = _("error: "); + if (!std::strncmp(errmsg, pfx, strlen(pfx))) + throw informative_failure(errmsg); - if (sqlite3_errcode(sql) == SQLITE_ERROR) - auxiliary_message - = _("make sure database and containing directory are writeable\n" - "and you have not run out of disk space"); + char const * auxiliary_message = ""; + switch (errcode) + { + case SQLITE_ERROR: + case SQLITE_IOERR: + case SQLITE_CANTOPEN: + case SQLITE_PROTOCOL: + auxiliary_message + = _("make sure database and containing directory are writeable\n" + "and you have not run out of disk space"); + break; + default: break; + } - logged_sqlite3_exec(sql, "ROLLBACK", 0, 0, 0); - E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message); + E(false, F("sqlite error: %s\n%s") % errmsg % auxiliary_message); + } + }; + + struct transaction + { + transaction(sqlite3 * s) : db(s), committed(false) + { + sql::exec(db, "BEGIN EXCLUSIVE"); + } + void commit() + { + I(committed == false); + committed = true; + } + ~transaction() + { + if (committed) + sql::exec(db, "COMMIT"); + else + sql::exec(db, "ROLLBACK"); + } + private: + sqlite3 * db; + bool committed; + }; } -// execute an sql statement and return the single integer value that it -// should produce. +// transitional static int -logged_sqlite3_exec_int(sqlite3 * sql, char const * cmd) +logged_sqlite3_exec(sqlite3 * db, char const * cmd, + void* d1 = 0, void* d2 = 0, char **errmsg = 0) { - sqlite3_stmt * stmt = logged_sqlite3_prepare(sql, cmd); - if (stmt == 0) - report_sqlite_error(sql); - //I(sqlite3_column_count(stmt) == 1); + I(d1 == 0); + I(d2 == 0); + I(errmsg == 0); - if (sqlite3_step(stmt) != SQLITE_ROW) - report_sqlite_error(sql); - - int res = sqlite3_column_int(stmt, 0); - - if (sqlite3_step(stmt) != SQLITE_DONE) - report_sqlite_error(sql); - - L(FL("success")); - - sqlite3_finalize(stmt); - return res; + sql::exec(db, cmd); + return SQLITE_OK; } +// SQL extension functions. + // sqlite3_value_text returns unsigned char const *, which is inconvenient inline char const * sqlite3_value_cstr(sqlite3_value * arg) @@ -135,13 +207,6 @@ sqlite3_value_cstr(sqlite3_value * arg) return reinterpret_cast(sqlite3_value_text(arg)); } -// sqlite3_column_text also returns unsigned char const * -inline string -sqlite3_column_string(sqlite3_stmt * stmt, int col) -{ - return string(reinterpret_cast(sqlite3_column_text(stmt, col))); -} - inline bool is_ws(char c) { return c == '\r' || c == '\n' || c == '\t' || c == ' '; @@ -182,50 +247,28 @@ sqlite_sha1_fn(sqlite3_context *f, int n sqlite3_result_text(f, sha().c_str(), sha().size(), SQLITE_TRANSIENT); } -void -calculate_schema_id(sqlite3 *sql, string & ident) +static void +sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args) { - sqlite3_stmt * stmt - = logged_sqlite3_prepare(sql, - "SELECT sql FROM sqlite_master " - "WHERE (type = 'table' OR type = 'index') " - // filter out NULL sql statements, because - // those are auto-generated indices (for - // UNIQUE constraints, etc.). - "AND sql IS NOT NULL " - "AND name not like 'sqlite_stat%' " - "ORDER BY name"); - if (!stmt) - report_sqlite_error(sql); - //I(sqlite3_column_count(stmt) == 1); + if (nargs != 1) + { + sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1); + return; + } + data decoded; - int res; - string schema; - using boost::char_separator; - typedef boost::tokenizer > tokenizer; - char_separator sep(" \r\n\t", "(),;"); - - while ((res = sqlite3_step(stmt)) == SQLITE_ROW) + // This operation may throw informative_failure. We must intercept that + // and turn it into a call to sqlite3_result_error, or rollback will fail. + try { - string table_schema(sqlite3_column_string(stmt, 0)); - tokenizer tokens(table_schema, sep); - for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++) - { - if (schema.size() != 0) - schema += " "; - schema += *i; - } + decode_base64(base64(string(sqlite3_value_cstr(args[0]))), decoded); } - - if (res != SQLITE_DONE) - report_sqlite_error(sql); - - sqlite3_finalize(stmt); - L(FL("success")); - - hexenc tid; - calculate_ident(data(schema), tid); - ident = tid(); + catch (informative_failure & e) + { + sqlite3_result_error(f, e.what(), -1); + return; + } + sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT); } // these must be listed in order so that ones listed earlier override ones @@ -244,7 +287,7 @@ set_regime(upgrade_regime new_regime, up regime = std::min(new_regime, regime); } -static bool move_table(sqlite3 *sql, char **errmsg, +static bool move_table(sqlite3 *db, char **errmsg, char const * srcname, char const * dstname, char const * dstschema) @@ -254,7 +297,7 @@ static bool move_table(sqlite3 *sql, cha create += " "; create += dstschema; - int res = logged_sqlite3_exec(sql, create.c_str(), NULL, NULL, errmsg); + int res = logged_sqlite3_exec(db, create.c_str(), NULL, NULL, errmsg); if (res != SQLITE_OK) return false; @@ -263,14 +306,14 @@ static bool move_table(sqlite3 *sql, cha insert += " SELECT * FROM "; insert += srcname; - res = logged_sqlite3_exec(sql, insert.c_str(), NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, insert.c_str(), NULL, NULL, errmsg); if (res != SQLITE_OK) return false; string drop = "DROP TABLE "; drop += srcname; - res = logged_sqlite3_exec(sql, drop.c_str(), NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, drop.c_str(), NULL, NULL, errmsg); if (res != SQLITE_OK) return false; @@ -279,14 +322,14 @@ static bool static bool -migrate_client_merge_url_and_group(sqlite3 * sql, +migrate_client_merge_url_and_group(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { // migrate the posting_queue table - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "posting_queue", "tmp", "(" @@ -296,7 +339,7 @@ migrate_client_merge_url_and_group(sqlit ")")) return false; - int res = logged_sqlite3_exec(sql, "CREATE TABLE posting_queue " + int res = logged_sqlite3_exec(db, "CREATE TABLE posting_queue " "(" "url not null, -- URL we are going to send this to\n" "content not null -- the packets we're going to send\n" @@ -304,7 +347,7 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO posting_queue " + res = logged_sqlite3_exec(db, "INSERT INTO posting_queue " "SELECT " "(url || '/' || groupname), " "content " @@ -312,13 +355,13 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // migrate the incoming_queue table - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "incoming_queue", "tmp", "(" @@ -328,7 +371,7 @@ migrate_client_merge_url_and_group(sqlit ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE incoming_queue " + res = logged_sqlite3_exec(db, "CREATE TABLE incoming_queue " "(" "url not null, -- URL we got this bundle from\n" "content not null -- the packets we're going to read\n" @@ -336,7 +379,7 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO incoming_queue " + res = logged_sqlite3_exec(db, "INSERT INTO incoming_queue " "SELECT " "(url || '/' || groupname), " "content " @@ -344,13 +387,13 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // migrate the sequence_numbers table - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "sequence_numbers", "tmp", "(" @@ -363,7 +406,7 @@ migrate_client_merge_url_and_group(sqlit )) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE sequence_numbers " + res = logged_sqlite3_exec(db, "CREATE TABLE sequence_numbers " "(" "url primary key, -- URL to read from\n" "major not null, -- 0 in news servers, may be higher in depots\n" @@ -372,7 +415,7 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO sequence_numbers " + res = logged_sqlite3_exec(db, "INSERT INTO sequence_numbers " "SELECT " "(url || '/' || groupname), " "major, " @@ -381,13 +424,13 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // migrate the netserver_manifests table - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "netserver_manifests", "tmp", "(" @@ -399,7 +442,7 @@ migrate_client_merge_url_and_group(sqlit )) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE netserver_manifests " + res = logged_sqlite3_exec(db, "CREATE TABLE netserver_manifests " "(" "url not null, -- url of some server\n" "manifest not null, -- manifest which exists on url\n" @@ -408,7 +451,7 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO netserver_manifests " + res = logged_sqlite3_exec(db, "INSERT INTO netserver_manifests " "SELECT " "(url || '/' || groupname), " "manifest " @@ -416,7 +459,7 @@ migrate_client_merge_url_and_group(sqlit if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; @@ -424,14 +467,14 @@ static bool } static bool -migrate_client_add_hashes_and_merkle_trees(sqlite3 * sql, +migrate_client_add_hashes_and_merkle_trees(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { // add the column to manifest_certs - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "manifest_certs", "tmp", "(" @@ -444,7 +487,7 @@ migrate_client_add_hashes_and_merkle_tre ")")) return false; - int res = logged_sqlite3_exec(sql, "CREATE TABLE manifest_certs\n" + int res = logged_sqlite3_exec(db, "CREATE TABLE manifest_certs\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" "id not null, -- joins with manifests.id or manifest_deltas.id\n" @@ -457,7 +500,7 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO manifest_certs " + res = logged_sqlite3_exec(db, "INSERT INTO manifest_certs " "SELECT " "sha1(':', id, name, value, keypair, signature), " "id, name, value, keypair, signature " @@ -465,12 +508,12 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // add the column to file_certs - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "file_certs", "tmp", "(" @@ -483,7 +526,7 @@ migrate_client_add_hashes_and_merkle_tre ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE file_certs\n" + res = logged_sqlite3_exec(db, "CREATE TABLE file_certs\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" "id not null, -- joins with files.id or file_deltas.id\n" @@ -496,7 +539,7 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO file_certs " + res = logged_sqlite3_exec(db, "INSERT INTO file_certs " "SELECT " "sha1(':', id, name, value, keypair, signature), " "id, name, value, keypair, signature " @@ -504,12 +547,12 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // add the column to public_keys - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "public_keys", "tmp", "(" @@ -518,7 +561,7 @@ migrate_client_add_hashes_and_merkle_tre ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE public_keys\n" + res = logged_sqlite3_exec(db, "CREATE TABLE public_keys\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" "id primary key, -- key identifier chosen by user\n" @@ -527,7 +570,7 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO public_keys " + res = logged_sqlite3_exec(db, "INSERT INTO public_keys " "SELECT " "sha1(':', id, keydata), " "id, keydata " @@ -535,12 +578,12 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // add the column to private_keys - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "private_keys", "tmp", "(" @@ -549,7 +592,7 @@ migrate_client_add_hashes_and_merkle_tre ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE private_keys\n" + res = logged_sqlite3_exec(db, "CREATE TABLE private_keys\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" "id primary key, -- as in public_keys (same identifiers, in fact)\n" @@ -558,7 +601,7 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO private_keys " + res = logged_sqlite3_exec(db, "INSERT INTO private_keys " "SELECT " "sha1(':', id, keydata), " "id, keydata " @@ -566,13 +609,13 @@ migrate_client_add_hashes_and_merkle_tre if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // add the merkle tree stuff - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE merkle_nodes\n" "(\n" "type not null, -- \"key\", \"mcert\", \"fcert\", \"manifest\"\n" @@ -589,42 +632,42 @@ static bool } static bool -migrate_client_to_revisions(sqlite3 * sql, +migrate_client_to_revisions(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, "DROP TABLE schema_version;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE schema_version;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE posting_queue;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE posting_queue;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE incoming_queue;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE incoming_queue;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE sequence_numbers;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE sequence_numbers;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE file_certs;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE file_certs;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE netserver_manifests;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE netserver_manifests;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE merkle_nodes\n" "(\n" "type not null, -- \"key\", \"mcert\", \"fcert\", \"rcert\"\n" @@ -637,7 +680,7 @@ migrate_client_to_revisions(sqlite3 * sq if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE revision_certs\n" + res = logged_sqlite3_exec(db, "CREATE TABLE revision_certs\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" "id not null, -- joins with revisions.id\n" @@ -650,7 +693,7 @@ migrate_client_to_revisions(sqlite3 * sq if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE revisions\n" + res = logged_sqlite3_exec(db, "CREATE TABLE revisions\n" "(\n" "id primary key, -- SHA1(text of revision)\n" "data not null -- compressed, encoded contents of a revision\n" @@ -658,7 +701,7 @@ migrate_client_to_revisions(sqlite3 * sq if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE revision_ancestry\n" + res = logged_sqlite3_exec(db, "CREATE TABLE revision_ancestry\n" "(\n" "parent not null, -- joins with revisions.id\n" "child not null, -- joins with revisions.id\n" @@ -674,19 +717,19 @@ static bool static bool -migrate_client_to_epochs(sqlite3 * sql, +migrate_client_to_epochs(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE merkle_nodes;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE branch_epochs\n" "(\n" "hash not null unique, -- hash of remaining fields separated by \":\"\n" @@ -700,14 +743,14 @@ static bool } static bool -migrate_client_to_vars(sqlite3 * sql, +migrate_client_to_vars(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE db_vars\n" "(\n" "domain not null, -- scope of application of a var\n" @@ -722,28 +765,28 @@ static bool } static bool -migrate_client_to_add_indexes(sqlite3 * sql, +migrate_client_to_add_indexes(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE INDEX revision_ancestry__child " "ON revision_ancestry (child)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE INDEX revision_certs__id " "ON revision_certs (id);", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE INDEX revision_certs__name_value " "ON revision_certs (name, value);", NULL, NULL, errmsg); @@ -754,7 +797,7 @@ static bool } static bool -migrate_client_to_external_privkeys(sqlite3 * sql, +migrate_client_to_external_privkeys(sqlite3 * db, char ** errmsg, app_state *app, upgrade_regime & regime) @@ -762,47 +805,20 @@ migrate_client_to_external_privkeys(sqli int res; map pub, priv; vector pairs; - sqlite3_stmt * stmt; - stmt = logged_sqlite3_prepare(sql, "SELECT id, keydata FROM private_keys;"); - if (stmt == 0) - { - *errmsg = const_cast(sqlite3_errmsg(sql)); - return false; - } - //I(sqlite3_column_count(stmt) == 2); + { + sql stmt(db, 2, "SELECT id, keydata FROM private_keys"); - while ((res = sqlite3_step(stmt)) == SQLITE_ROW) - priv.insert(make_pair(sqlite3_column_string(stmt, 0), - sqlite3_column_string(stmt, 1))); + while (stmt.step()) + priv.insert(make_pair(stmt.column_string(0), stmt.column_string(1))); + } + { + sql stmt(db, 2, "SELECT id, keydata FROM public_keys"); - if (res != SQLITE_DONE) - { - *errmsg = const_cast(sqlite3_errmsg(sql)); - return false; - } - sqlite3_finalize(stmt); + while (stmt.step()) + pub.insert(make_pair(stmt.column_string(0), stmt.column_string(1))); + } - stmt = logged_sqlite3_prepare(sql, "SELECT id, keydata FROM public_keys;"); - if (stmt == 0) - { - *errmsg = const_cast(sqlite3_errmsg(sql)); - return false; - } - //I(sqlite3_column_count(stmt) == 2); - - while ((res = sqlite3_step(stmt)) == SQLITE_ROW) - pub.insert(make_pair(sqlite3_column_string(stmt, 0), - sqlite3_column_string(stmt, 1))); - - if (res != SQLITE_DONE) - { - *errmsg = const_cast(sqlite3_errmsg(sql)); - return false; - } - sqlite3_finalize(stmt); - - for (map::const_iterator i = priv.begin(); i != priv.end(); ++i) { @@ -825,7 +841,7 @@ migrate_client_to_external_privkeys(sqli app->keys.put_key_pair(ident, kp); } - res = logged_sqlite3_exec(sql, "DROP TABLE private_keys;", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE private_keys;", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; @@ -833,14 +849,14 @@ static bool } static bool -migrate_client_to_add_rosters(sqlite3 * sql, +migrate_client_to_add_rosters(sqlite3 * db, char ** errmsg, app_state *, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE rosters\n" "(\n" "id primary key, -- strong hash of the roster\n" @@ -850,7 +866,7 @@ migrate_client_to_add_rosters(sqlite3 * if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE roster_deltas\n" "(\n" "id not null, -- strong hash of the roster\n" @@ -862,7 +878,7 @@ migrate_client_to_add_rosters(sqlite3 * if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE revision_roster\n" "(\n" "rev_id primary key, -- joins with revisions.id\n" @@ -872,7 +888,7 @@ migrate_client_to_add_rosters(sqlite3 * if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE next_roster_node_number\n" "(\n" "node primary key -- only one entry in this table, ever\n" @@ -886,33 +902,18 @@ migrate_client_to_add_rosters(sqlite3 * return true; } -static void -sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args) -{ - if (nargs != 1) - { - sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1); - return; - } - data decoded; - decode_base64(base64(string(sqlite3_value_cstr(args[0]))), decoded); - sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT); -} - // I wish I had a form of ALTER TABLE COMMENT on sqlite3 static bool -migrate_files_BLOB(sqlite3 * sql, +migrate_files_BLOB(sqlite3 * db, char ** errmsg, app_state *app, upgrade_regime & regime) { int res; - I(sqlite3_create_function(sql, "unbase64", -1, - SQLITE_UTF8, NULL, - &sqlite3_unbase64_fn, - NULL, NULL) == 0); + sql::create_function(db, "unbase64", sqlite3_unbase64_fn); + // change the encoding of file(_delta)s - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "files", "tmp", "(" @@ -921,7 +922,7 @@ migrate_files_BLOB(sqlite3 * sql, ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE files\n" + res = logged_sqlite3_exec(db, "CREATE TABLE files\n" "\t(\n" "\tid primary key, -- strong hash of file contents\n" "\tdata not null -- compressed contents of a file\n" @@ -929,17 +930,17 @@ migrate_files_BLOB(sqlite3 * sql, if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO files " + res = logged_sqlite3_exec(db, "INSERT INTO files " "SELECT id, unbase64(data) " "FROM tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - if (!move_table(sql, errmsg, + if (!move_table(db, errmsg, "file_deltas", "tmp", "(" @@ -949,7 +950,7 @@ migrate_files_BLOB(sqlite3 * sql, ")")) return false; - res = logged_sqlite3_exec(sql, "CREATE TABLE file_deltas\n" + res = logged_sqlite3_exec(db, "CREATE TABLE file_deltas\n" "\t(\n" "\tid not null, -- strong hash of file contents\n" "\tbase not null, -- joins with files.id or file_deltas.id\n" @@ -959,57 +960,57 @@ migrate_files_BLOB(sqlite3 * sql, if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "INSERT INTO file_deltas " + res = logged_sqlite3_exec(db, "INSERT INTO file_deltas " "SELECT id, base, unbase64(delta) " "FROM tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE tmp", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE tmp", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; // migrate other contents which are accessed by get|put_version - res = logged_sqlite3_exec(sql, "UPDATE manifests SET data=unbase64(data)", + res = logged_sqlite3_exec(db, "UPDATE manifests SET data=unbase64(data)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE manifest_deltas " + res = logged_sqlite3_exec(db, "UPDATE manifest_deltas " "SET delta=unbase64(delta)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE rosters SET data=unbase64(data) ", + res = logged_sqlite3_exec(db, "UPDATE rosters SET data=unbase64(data) ", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE roster_deltas " + res = logged_sqlite3_exec(db, "UPDATE roster_deltas " "SET delta=unbase64(delta)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE db_vars " + res = logged_sqlite3_exec(db, "UPDATE db_vars " "SET value=unbase64(value),name=unbase64(name)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE public_keys " + res = logged_sqlite3_exec(db, "UPDATE public_keys " "SET keydata=unbase64(keydata)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE revision_certs " + res = logged_sqlite3_exec(db, "UPDATE revision_certs " "SET value=unbase64(value),signature=unbase64(signature)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE manifest_certs " + res = logged_sqlite3_exec(db, "UPDATE manifest_certs " "SET value=unbase64(value),signature=unbase64(signature) ", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE revisions " + res = logged_sqlite3_exec(db, "UPDATE revisions " "SET data=unbase64(data)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "UPDATE branch_epochs " + res = logged_sqlite3_exec(db, "UPDATE branch_epochs " "SET branch=unbase64(branch)", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; @@ -1017,24 +1018,24 @@ static bool } static bool -migrate_rosters_no_hash(sqlite3 * sql, +migrate_rosters_no_hash(sqlite3 * db, char ** errmsg, app_state * app, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, "DROP TABLE rosters", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE rosters", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE roster_deltas", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE roster_deltas", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, "DROP TABLE revision_roster", NULL, NULL, errmsg); + res = logged_sqlite3_exec(db, "DROP TABLE revision_roster", NULL, NULL, errmsg); if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE rosters\n" "\t(\n" "\tid primary key, -- a revision id\n" @@ -1045,7 +1046,7 @@ migrate_rosters_no_hash(sqlite3 * sql, if (res != SQLITE_OK) return false; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE roster_deltas\n" "\t(\n" "\tid primary key, -- a revision id\n" @@ -1064,14 +1065,14 @@ static bool static bool -migrate_add_heights(sqlite3 *sql, +migrate_add_heights(sqlite3 *db, char ** errmsg, app_state *app, upgrade_regime & regime) { int res; - res = logged_sqlite3_exec(sql, + res = logged_sqlite3_exec(db, "CREATE TABLE heights\n" "(\n" "revision not null, -- joins with revisions.id\n" @@ -1139,6 +1140,41 @@ const size_t n_migration_events = (sizeo const size_t n_migration_events = (sizeof migration_events / sizeof migration_events[0]); +void +calculate_schema_id(sqlite3 *db, string & ident) +{ + sql stmt(db, 1, + "SELECT sql FROM sqlite_master " + "WHERE (type = 'table' OR type = 'index') " + // filter out NULL statements, because + // those are auto-generated indices (for + // UNIQUE constraints, etc.). + "AND sql IS NOT NULL " + "AND name not like 'sqlite_stat%' " + "ORDER BY name"); + + string schema; + using boost::char_separator; + typedef boost::tokenizer > tokenizer; + char_separator sep(" \r\n\t", "(),;"); + + while (stmt.step()) + { + string table_schema(stmt.column_string(0)); + tokenizer tokens(table_schema, sep); + for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++) + { + if (schema.size() != 0) + schema += " "; + schema += *i; + } + } + + hexenc tid; + calculate_ident(data(schema), tid); + ident = tid(); +} + // Look through the migration_events table and return the index of the // entry corresponding to schema ID, or -1 if it isn't there (i.e. if // the database schema is not one we know). @@ -1156,7 +1192,7 @@ static void NORETURN // Provide sensible diagnostics for a database schema whose hash we do not // recognize. static void NORETURN -diagnose_unrecognized_schema(sqlite3 * sql, system_path const & filename, +diagnose_unrecognized_schema(sqlite3 * db, system_path const & filename, string const & id) { // Give a special message for an utterly empty sqlite3 database, such as @@ -1173,12 +1209,11 @@ diagnose_unrecognized_schema(sqlite3 * s // 'manifest_deltas'. // ??? Use PRAGMA user_version to record an additional magic number in // monotone databases. - int n = logged_sqlite3_exec_int(sql, - "SELECT COUNT(*) FROM sqlite_master " - "WHERE type = 'table' AND sql IS NOT NULL " - "AND (name = 'files' OR name = 'file_deltas'" - " OR name = 'manifests'" - " OR name = 'manifest_deltas')"); + int n = sql::value(db, + "SELECT COUNT(*) FROM sqlite_master " + "WHERE type = 'table' AND sql IS NOT NULL " + "AND (name = 'files' OR name = 'file_deltas'" + " OR name = 'manifests' OR name = 'manifest_deltas')"); N(n == 4, F("%s does not appear to be a monotone database\n" "(schema %s, core tables missing)") @@ -1190,22 +1225,22 @@ diagnose_unrecognized_schema(sqlite3 * s "you probably need a newer version of monotone.") % filename % id); } - + // check_sql_schema is called by database.cc on open, to determine whether // the schema is up to date. If it returns at all, the schema is indeed // up to date (otherwise it throws a diagnostic). void -check_sql_schema(sqlite3 * sql, system_path const & filename) +check_sql_schema(sqlite3 * db, system_path const & filename) { - I(sql != NULL); + I(db != NULL); string id; - calculate_schema_id(sql, id); + calculate_schema_id(db, id); int migration = schema_to_migration(id); if (migration == -1) - diagnose_unrecognized_schema(sql, filename, id); + diagnose_unrecognized_schema(db, filename, id); N(migration_events[migration].migrator == 0, F("database %s is laid out according to an old schema, %s\n" @@ -1215,74 +1250,66 @@ void } void -migrate_monotone_schema(sqlite3 * sql, app_state * app) +migrate_monotone_schema(sqlite3 * db, app_state * app) { - I(sql != NULL); - I(!sqlite3_create_function(sql, "sha1", -1, SQLITE_UTF8, NULL, - &sqlite_sha1_fn, NULL, NULL)); + I(db != NULL); + sql::create_function(db, "sha1", sqlite_sha1_fn); - string init; - calculate_schema_id(sql, init); + upgrade_regime regime = upgrade_none; + + // Take an exclusive lock on the database before we try to read anything + // from it. If we don't take this lock until the beginning of the + // "migrating data" phase, two simultaneous "db migrate" processes could + // race through the "calculating migration" phase; then one of them would + // wait for the other to finish all the migration steps, and trip over the + // invariant check inside the for loop. + { + transaction guard(db); - P(F("calculating migration for schema %s") % init); + string init; + calculate_schema_id(db, init); - int i = schema_to_migration(init); + P(F("calculating migration for schema %s") % init); - if (i == -1) - diagnose_unrecognized_schema(sql, app->db.get_filename(), init); + int i = schema_to_migration(init); - // We really want 'db migrate' on an up-to-date schema to be a no-op - // (no vacuum or anything, even), so that automated scripts can fire - // one off optimistically and not have to worry about getting their - // administrators to do it by hand. - if (migration_events[i].migrator == 0) - { - P(F("no migration performed; database schema already up-to-date")); - return; - } + if (i == -1) + diagnose_unrecognized_schema(db, app->db.get_filename(), init); - upgrade_regime regime = upgrade_none; - P(F("migrating data")); + // We really want 'db migrate' on an up-to-date schema to be a no-op + // (no vacuum or anything, even), so that automated scripts can fire + // one off optimistically and not have to worry about getting their + // administrators to do it by hand. + if (migration_events[i].migrator == 0) + { + P(F("no migration performed; database schema already up-to-date")); + return; + } - E(logged_sqlite3_exec(sql, "BEGIN EXCLUSIVE", NULL, NULL, NULL) == SQLITE_OK, - F("error at transaction BEGIN statement: %s") % sqlite3_errmsg(sql)); + P(F("migrating data")); - for (;;) - { - // confirm that we are where we ought to be - string curr; - calculate_schema_id(sql, curr); - if (curr != migration_events[i].id) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - I(false); - } + for (;;) + { + // confirm that we are where we ought to be + string curr; + calculate_schema_id(db, curr); + I(curr == migration_events[i].id); - if (migration_events[i].migrator == 0) - break; + if (migration_events[i].migrator == 0) + break; - char * errmsg; - if (! migration_events[i].migrator(sql, &errmsg, app, regime)) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - E(false, F("migration step failed: %s") - % (errmsg ? errmsg : "unknown error")); - } + migration_events[i].migrator(db, 0, app, regime); - i++; - if ((size_t)i >= n_migration_events) - { - logged_sqlite3_exec(sql, "ROLLBACK", NULL, NULL, NULL); - I(false); - } - } + i++; + I((size_t)i < n_migration_events); + } - P(F("committing changes to database")); - E(logged_sqlite3_exec(sql, "COMMIT", NULL, NULL, NULL) == SQLITE_OK, - F("failure on COMMIT")); + P(F("committing changes to database")); + guard.commit(); + } P(F("optimizing database")); - logged_sqlite3_exec(sql, "VACUUM", NULL, NULL, NULL); + logged_sqlite3_exec(db, "VACUUM"); switch (regime) { ============================================================ --- testsuite.lua 5360e416548d4328291c3c6a19f86feffc6cdd0e +++ testsuite.lua 31665a2c0d4546535acc5470c28b2b73f827d6b3 @@ -386,6 +386,7 @@ table.insert(tests, "schema_migration_wi table.insert(tests, "schema_migration") table.insert(tests, "schema_migration_bad_schema") table.insert(tests, "schema_migration_with_rosterify") +table.insert(tests, "schema_migration_error_recovery") table.insert(tests, "database_dump_load") table.insert(tests, "no-change_deltas_disappear") table.insert(tests, "merge((),_(drop_a,_rename_b_a,_patch_a))")