# # # add_file "contrib/mashschema.cc" # content [009d7602100f5cd713207854bc4eb8b3098e4397] # # add_file "tests/schema_migration/2881277287f6ee9bfc5ee255a503a6dc20dd5994.mtn.dumped" # content [b660fa27065bf53d12b210dc785c33e6b45e0db1] # # patch "ChangeLog" # from [84a29f98ca8c2f76e1e2a0b7c7ae50be47eef5d6] # to [153dba95c7b8e906e7d4c9e18d8f30cfb3ddbea2] # # patch "README.changesets" # from [5d0304f2673662993c53df4026d150b2889dcab8] # to [490acbb65d7c12452e9d0161c619b690de30bda3] # # patch "database.cc" # from [688eaa61e7485a72cc4593c8f90f6d0b5cf6f832] # to [f0fea1672a7803f2725c81e0b2eeebb1c47e2f15] # # patch "schema.sql" # from [9078e155d91e3f75159816ce77c57218ea7f6016] # to [e98092bdf486d6d237a1e87f4ce6b404100ba6fd] # # patch "schema_migration.cc" # from [642fd28cf4c1e900dd1d747cbe1c8d1acec7ca02] # to [d99b978978e754e603e533613f5a59aa4136e03e] # # patch "schema_migration.hh" # from [7d49cb153197c91b0572b76cdc72748cadfda57f] # to [75ea83a18e754dc5f3a7302bfa95de9eadbd643a] # # patch "tests/schema_migration/__driver__.lua" # from [747b54fe959db4efd3588f495dbe627dda2174c1] # to [faccb4ba0fb263719495780b5ea830065edda37c] # # patch "tests/schema_migration_bad_schema/possible.dump" # from [99f9120eef90c5600b0afd4f85653b815b0ea4cb] # to [87858caeefa73d827b8a072fd8578448140b88eb] # ============================================================ --- contrib/mashschema.cc 009d7602100f5cd713207854bc4eb8b3098e4397 +++ contrib/mashschema.cc 009d7602100f5cd713207854bc4eb8b3098e4397 @@ -0,0 +1,62 @@ +/* This program helps take a sequence of CREATE TABLE statements as + written in schema.sql and mash them into the format used when + computing schema hashes. Use it when you need to add an entry to + the temporarily_allowed_tables array in schema_migration.cc. + + Here's how to use it: Cut the relevant CREATE TABLE statements out + of schema.sql. Paste them into a scratch file. Make sure they are + in alphabetical order by table name. Remove all terminating + semicolons. Then run this program as follows: + + g++ mashschema.cc + ./a.out < scratchfile | fmt | + sed -e 's/\\/\\\\/g + s/"/\\"/g + s/^/ "/ + s/$/ "/ + $s/ "$/",/' > scratchfile2 + + Insert the text in scratchfile2 *verbatim* into the array, just before + the 0-terminator. */ + +#include +#include +#include + +using std::cin; +using std::cout; +using std::string; +using boost::char_separator; +typedef boost::tokenizer > tokenizer; + +int +main(void) +{ + string schema; + string line; + char_separator sep(" \r\n\t", "(),;"); + + while (!cin.eof()) + { + std::getline(cin, line); + tokenizer tokens(line, sep); + for (tokenizer::iterator i = tokens.begin(); i != tokens.end(); i++) + { + if (schema.size() != 0) + schema += " "; + schema += *i; + } + } + + cout << schema << "\n"; + + return 0; +} + +// Local Variables: +// mode: C++ +// fill-column: 76 +// c-file-style: "gnu" +// indent-tabs-mode: nil +// End: +// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s: ============================================================ --- tests/schema_migration/2881277287f6ee9bfc5ee255a503a6dc20dd5994.mtn.dumped b660fa27065bf53d12b210dc785c33e6b45e0db1 +++ tests/schema_migration/2881277287f6ee9bfc5ee255a503a6dc20dd5994.mtn.dumped b660fa27065bf53d12b210dc785c33e6b45e0db1 @@ -0,0 +1,133 @@ +BEGIN EXCLUSIVE; +CREATE TABLE branch_epochs + ( + hash not null unique, -- hash of remaining fields separated by ":" + branch not null unique, -- joins with revision_certs.value + epoch not null -- random hex-encoded id + ); +CREATE TABLE db_vars + ( + domain not null, -- scope of application of a var + name not null, -- var key + value not null, -- var value + unique(domain, name) + ); +CREATE TABLE file_deltas + ( + id not null, -- strong hash of file contents + base not null, -- joins with files.id or file_deltas.id + delta not null, -- compressed rdiff to construct current from base + unique(id, base) + ); +INSERT INTO file_deltas VALUES('a9ca701697adae066b96d07aabb30f0d6245692c','d4929f246d23a51eba6799685e28f9ab077b483a',X'1f8b08000000000000fff35430e54a332c33e4e20200eac8a2590a000000'); +INSERT INTO file_deltas VALUES('36f92840dcffa22064b2dd9e0848d14350f07c5c','f9d518a4e1308cbe8503bdd8f578b16de4407491',X'1f8b08000000000000fff35430e54a332a33e4e202003ab2021e0a000000'); +INSERT INTO file_deltas VALUES('09848c4631a20ac166344f58a23fee04a6c646a4','1ece609689fb9462de25716110769bad1a80e8d8',X'1f8b08000000000000fff35430e54a332933e4e202009a4742910a000000'); +CREATE TABLE files + ( + id primary key, -- strong hash of file contents + data not null -- compressed contents of a file + ); +INSERT INTO files VALUES('bbeadf8e35428c9e5333e71caf25851498306eb6',X'1f8b08000000000000ff4b332e33e40200f1d83a4405000000'); +INSERT INTO files VALUES('d4929f246d23a51eba6799685e28f9ab077b483a',X'1f8b08000000000000ff4b332c33e20200b9431ec505000000'); +INSERT INTO files VALUES('f9d518a4e1308cbe8503bdd8f578b16de4407491',X'1f8b08000000000000ff4b332a33e2020057ecabd705000000'); +INSERT INTO files VALUES('1ece609689fb9462de25716110769bad1a80e8d8',X'1f8b08000000000000ff4b332933e202008bb3c0f205000000'); +CREATE TABLE heights + ( + revision not null, -- joins with revisions.id + height not null, -- complex height, array of big endian u32 integers + unique(revision, height) + ); +INSERT INTO heights VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a',X'00000001'); +INSERT INTO heights VALUES('c81722b0236303685e341e16f0073d665090fb73',X'00000002'); +INSERT INTO heights VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0',X'000000010000000000000000'); +INSERT INTO heights VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada',X'00000003'); +INSERT INTO heights VALUES('75810233cc39b62341d669b610e9416fd6352869',X'00000004'); +CREATE TABLE next_roster_node_number + ( + node primary key -- only one entry in this table, ever + ); +INSERT INTO next_roster_node_number VALUES('5'); +CREATE TABLE public_keys + ( + hash not null unique, -- hash of remaining fields separated by ":" + id primary key, -- key identifier chosen by user + keydata not null -- RSA public params + ); +INSERT INTO public_keys VALUES('de84b575d5e47254393eba49dce9dc4db98ed42d','address@hidden',X'30819d300d06092a864886f70d010101050003818b0030818702818100b9e2f563aeba98a137cf4e05a6e89a6e2fe90e11170dd8e49c06c5aae9c6f85a6de79729b056a249cb0aba71b7f28146309aaca244b3f1468b1f2c6bb6dbc02113368ce096c01f5a6083b0b0bef55d7c74573405c43203bcf6006392479cc4b8b853f4faec3acb8e444f0229428162d936ff1878e1bd5c03de816cabee1340f7020111'); +INSERT INTO public_keys VALUES('c9d80250e944708aab7fe960c1136b517fd30772','address@hidden',X'30819d300d06092a864886f70d010101050003818b00308187028181009f37f70031a6e06fa4fb9bc92c1a7a50ecb3ba84e76773fed5ca2beb446b66a70488b658e1b756e680b405dfb2a248fa0c10351bd6cb5f2b168cb188b2f224f2a10f8cd845453f7ee7dabb67359d07a15710776b11b027ab40587d9ec9513f29a59105d494e88d14de9194535c76392e72a35ef98a2b157db5118c3b5ee4f95f020111'); +CREATE TABLE revision_ancestry + ( + parent not null, -- joins with revisions.id + child not null, -- joins with revisions.id + unique(parent, child) + ); +INSERT INTO revision_ancestry VALUES('','bf468e6c22dec9203af6441ad7d20b6ad8af049a'); +INSERT INTO revision_ancestry VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','c81722b0236303685e341e16f0073d665090fb73'); +INSERT INTO revision_ancestry VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','43a2235616452dca74eecf39d645a69da8e0bdd0'); +INSERT INTO revision_ancestry VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0','4a1274f35812a695e357c6e7c7cd60f449f0cada'); +INSERT INTO revision_ancestry VALUES('c81722b0236303685e341e16f0073d665090fb73','4a1274f35812a695e357c6e7c7cd60f449f0cada'); +INSERT INTO revision_ancestry VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada','75810233cc39b62341d669b610e9416fd6352869'); +CREATE TABLE revision_certs + ( + hash not null unique, -- hash of remaining fields separated by ":" + id not null, -- joins with revisions.id + name not null, -- opaque string chosen by user + value not null, -- opaque blob + keypair not null, -- joins with public_keys.id + signature not null, -- RSA/SHA1 signature of "address@hidden:val]" + unique(name, id, value, keypair, signature) + ); +INSERT INTO revision_certs VALUES('6f938572483d4f73bf952c3666dace09f95ebd50','bf468e6c22dec9203af6441ad7d20b6ad8af049a','branch',X'746573746272616e636831','address@hidden',X'0a3810a8fd6bff50d59204826b3ee32af81d3c9d5627711b0bc8f2780471aacdb0d6d80cee200636cd3deb563cfab05566e0817062e54bf5b5175655025b1e553d8d98154e86ed0e9665a167657524f925c8890593792b263489c4fd5bb7c0fa207095c717678440d4adff0e09ce19d203b30f39c9daa52715f3376595020232'); +INSERT INTO revision_certs VALUES('2dda1c13436be7c2f10271c210a2bd0c885f313c','bf468e6c22dec9203af6441ad7d20b6ad8af049a','changelog',X'626c61682d626c6168','address@hidden',X'34a21d284a20c9ddc32b08c97d4934068a9f8117c4f1ee47ad16a12ec3127ff8fa5c3f17434d2a556f619ab9ff09a053741c99e5966e4c7f81c8b7170c01d58484c761a4402f14f1df85a5f86a4f05d82e5e0237860cb957e148f493759b670197cf679151ba196d28efc9c6168d71ef7be9998a9ae563d6c02ac03b0266ed94'); +INSERT INTO revision_certs VALUES('6ac4524843235b44ca2dfc2696d38f2b90239109','bf468e6c22dec9203af6441ad7d20b6ad8af049a','date',X'313939392d30312d30315431323a30303a3030','address@hidden',X'6411df9eefc68ba4bdd113781a922e42c7e548bd893d4dd0212b9521e8f1582c4ae7857ace278e67a64723c18409f582a895ad0f72fec05121a60837a124ac66b41133c99da6a64ce6c4c438a8dae6d77bd9d3e4212a74cac244e0a2e6b8356ff3c6920ab72c5c3c7211953c8d41066a28be27001f15d3f8393adf2f6b07d492'); +INSERT INTO revision_certs VALUES('d77b687d8c619078e80721658a6d99dcfe7e32e5','bf468e6c22dec9203af6441ad7d20b6ad8af049a','author',X'74657374657240746573742e6e6574','address@hidden',X'1d9b1adf2c81ad01a2f86978fe32faa4986b8f3e9e7f7012ae0e9836338d5b9ca97a08f60c8719a503788cb18918acd33d0508f910604859f71b6753100d53fee97c49392e21cfe6a9998abcc78c3d2b6addece348a602c8b7b2e506d3be6d079292126379d39ac5deb5cb1c9a1043b9403d1a3647823f4220454ead87e42907'); +INSERT INTO revision_certs VALUES('c50226beb7e3d289d38e9f613021c380f89ab011','bf468e6c22dec9203af6441ad7d20b6ad8af049a','somekey',X'736f6d6576616c7565','address@hidden',X'1807efea8be82de6eab9686aa949d99870a99f374b09f9de1f899699f21a8dd8af841bbe84f155eb28befb14aff3772b7a90ed54961e15dd51239d0f6199ed505e2bcd6dc938e9499700216a2af3c74b1a1291c989263ecc12cd5c359bb054670f05b00027794ec4b074f555005e19b4fbd47c2836a06e2a8a72fe26850d7162'); +INSERT INTO revision_certs VALUES('628a294256cbb30fa29665cba0ce9a58c02e57b0','c81722b0236303685e341e16f0073d665090fb73','branch',X'746573746272616e636832','address@hidden',X'97c44b9de4e66ef26b694970ceaa8d8b8a63a0c0c8578ad0951d9284e21b985475163c8ebd168de90e2b8f7ea461a78c9556ac0a31d12b974c3279a1106723c149ad73acf8f8729bbcee418db1c2ec778cb1cedefa72a9cfde1c99b5554f47c9e7fbc2ab32376b09ca8b78d3f534fdbf7f38c793935a1ba4995ec2ed54b0ae7b'); +INSERT INTO revision_certs VALUES('55d537c81d3e6db852813076cbb476aa7bfb8e6d','c81722b0236303685e341e16f0073d665090fb73','changelog',X'626c61682d626c6168','address@hidden',X'329aed4969e5b59d7eb51288c0e92ef10334fb2240d71db652216e0e09c5de256e47795e3d23568a5cce6cccc6b9844bbdb2797d763d6e59ca0cf65ef0994241581a9ae9d80a3a68938bbc4847126b469b1af08f49f2235fc518e18058fa1363d3399005fad5bce391c12551494426a2aca58485200ebd7e6b323bc3c3a01173'); +INSERT INTO revision_certs VALUES('70c013d2ba0e97f5e706dcc27e8eb32920f09c6f','c81722b0236303685e341e16f0073d665090fb73','date',X'323030302d30312d30315431323a30303a3030','address@hidden',X'2fbef5dba2115c40db66386dbf91453684c5db3c597424d631bd2bf57d850fffc15f9b8f3df40b47bd1d935f4ace464f1dc5b337245d8b07126099834ae57b6e54ed2efaef1bae918cb71944725dac12fabbb85eca132a2feee5589ab6ef7685bef3fe5e36ee26ae4b6d7cab8d07aeebdfc37a687b2d62659f37f07a5e0a520a'); +INSERT INTO revision_certs VALUES('e70e1cc3843fe732d6ee20d25749f98c8862166b','c81722b0236303685e341e16f0073d665090fb73','author',X'74657374657240746573742e6e6574','address@hidden',X'86a32cebddaec8e68cee2def23bd4ead4ff46346bec261419c8b3991f1ff889a87ee985c4589b7e160c51676422da2ae064cda8f9c2ee9fb6410fc0d605c258732b3a067f55e43468f9d2f1aea36651ee4b003ba82d40b854f054815217358a6ead981bc50d641ad21f49f606f89a27d6b6d7b0fe04bc6f94afde9ab81717b02'); +INSERT INTO revision_certs VALUES('b8207be46ed175205466998210bab3b6c30fa06b','43a2235616452dca74eecf39d645a69da8e0bdd0','branch',X'746573746272616e636831','address@hidden',X'68eab511c6fd1997db7141d0c9171b2daf80784b3875f9c05323f4d3c788624b76b57fa71e0a043e263c93cedca68f0abab8c3665b7be918e89b43441fcbe47e6979ed66dc7a3631abfed3486eea948890d2e2a690e37eb13e8c47c93ed7209c9a42b87b4a637c2b4afd30a101cee8cb5885d1681b7d2e9d990df33e99aaac62'); +INSERT INTO revision_certs VALUES('9d9472e1be171031fee0ff82f41d752e8124742e','43a2235616452dca74eecf39d645a69da8e0bdd0','changelog',X'626c61682d626c6168','address@hidden',X'09975b6cb5ef7044dd7b3eb0b7d273d65f885f96fcc4e70472034d3b8221139a8bf768d8d2e2580f5f584fb92bbf8c12132a86afe50cbd2ef52e7b3503cecc9ef69a62a2c3cd624a2dfa4a0932db5e6bd7931545a2df9dd4d8eb6a91f2c435a4cec598be9303bfe42c2c58c1258045a5dcb91038f583970a376f9d6d8a232d68'); +INSERT INTO revision_certs VALUES('c2e926d430da9903c918ee15b2b2dfe99ee58463','43a2235616452dca74eecf39d645a69da8e0bdd0','date',X'323030312d30312d30315431323a30303a3030','address@hidden',X'08a151ea8f30e5ec1574edbb4304ecd90d0cbe7652596b56ae724efc5346aded024e9570028d1c8f565f02de2841837df9b60e88526fb28f3cea13fb6fae61fe04250636388554ca82c5df5be09884970eff189c03c752b87c8640ca96cddd302a439145ab2c17c15843b68dc3d236696e78a161dce6460796790d9c5727c42c'); +INSERT INTO revision_certs VALUES('eaa39fc163b0d38cded5270f60bc9570e7f4e1b0','43a2235616452dca74eecf39d645a69da8e0bdd0','author',X'74657374657240746573742e6e6574','address@hidden',X'3ac80b0a61bb53d64f1cce10822c0d14707db5a266a35174f1b08d5ceeef50d85327281e47a6f2ea3971219cd92dd5709436764253589782a5b887eda85a588fcd70d20c40d1c3607f66127e92c3514e9896c9af8ed1a1503bbc9b6117e3aa038560be915d6e0e1423b994169e20da4b4577b40afbd79d438815380a2bc3a023'); +INSERT INTO revision_certs VALUES('1288aa1a3c351fd0528aa95ddaae309cc5eec3e2','4a1274f35812a695e357c6e7c7cd60f449f0cada','branch',X'746573746272616e636831','address@hidden',X'976aa27ee04de6305b1abd387eeff10bb1c66e9d61f3dd246e81efed5570fab7a3b96338e832e5b8013835aa1a135ab19406665a2095977f36abfabdbc04789d61649ac5c89e3d66c8a1b2bcad821e94ed6b10d2500391367ae460e39ce2e49b8d15ae890b5e7e1494356cb1d5683102c092b0492c524137827b5705c199e4cc'); +INSERT INTO revision_certs VALUES('27e8a71b0fa56a551e72aed26861ba404d02fd32','4a1274f35812a695e357c6e7c7cd60f449f0cada','changelog',X'70726f7061676174652066726f6d206272616e63682027746573746272616e636832272028686561642063383137323262303233363330333638356533343165313666303037336436363530393066623733290a202020202020202020202020746f206272616e63682027746573746272616e636831272028686561642034336132323335363136343532646361373465656366333964363435613639646138653062646430290a','address@hidden',X'8a542aa0138106374f6d10b450ea65c12bafd701c0bf81ca03264be3442ca5e5affea946ddfa9865345f39c5e21bf132e37fe54543e043d2aa68d98301f0f5b10cc12e25a6a5d8b7d507736f65c8dfcdd549cda4671c30acaec6cc273b5974d65ba326f6fc946fb3785e8d544dced8de86e4144f953e660b4e30fee92d71c1fd'); +INSERT INTO revision_certs VALUES('884d60e16e31cf3621191bd2dab0caf39a0283fe','4a1274f35812a695e357c6e7c7cd60f449f0cada','date',X'323030322d30312d30315431323a30303a3030','address@hidden',X'0cf775eddbf69611d4acce57040e7fdbcca0cad51ea07b35dbf48307d9b3a14a5f481c8402047a38d0b2f1145c6e8cf9f15ff9b1cd3680c0f0690e9516932a790b3c990d6783f6374426cf8600f08a56ad038beb9fb6e7e09defc6e4ebf208b2f66d51e840c018874b06deb2b21c7d8e81cde06ef3bad3a6460b6bed4e716146'); +INSERT INTO revision_certs VALUES('2724dd7883fb44a58d40eba12808ed5b9e09f7b6','4a1274f35812a695e357c6e7c7cd60f449f0cada','author',X'74657374657240746573742e6e6574','address@hidden',X'6ca7bd89227bfb3df1a2ad46a82e15bcee03aa721a60a2b71067573f1b64e3145d2063643516dcc08afdf98bf192765a0818720c9432abee5299fc37e0de7d0843b909c7f29567004b881ca668c335614c0c3c46c687ffbcdf8ec03413345469027e8f54f9ee1db2a406ce891777ee4b9a282f17d52ca17a22ef258d77323ed2'); +INSERT INTO revision_certs VALUES('5b44cd5cd32f3e8f268ac1cd30ccfa27d16667c6','75810233cc39b62341d669b610e9416fd6352869','branch',X'746573746272616e636833','address@hidden',X'02065dabb4b64a46fabde90e1a5fc6927cc3955b02b104a17400a55864a3c5f35ebbc21268ef75320157793d5041967d566bb6967f47f1816ee4d94095c80bd715f677c24c4c369d967938f315b12a12d2da928da45cdf0392d2eb23b9b7634e546108e74d7a49f537516c43a5683f4f28ef5c097d2c792f7bc6d3da4eb019e9'); +INSERT INTO revision_certs VALUES('98640622c14a2ecbe23b7251b960d8bc78dec4a2','75810233cc39b62341d669b610e9416fd6352869','changelog',X'626c61682d626c6168','address@hidden',X'5eb6f3d6dd6a258a452a859f44fd628de197892dd8d0cc72fe8739e02bd9d71fb206270c48f95d3dbec666c2e495bf1dbbddfda54baf8ed2830dd2bf9e6e41ca8facbe6cfa78fd28aea2c9221e55200998f6d37c87bf1e31d2eb9bec9498de353b05711efde836b51bc28dd27ae19893b363c615a6f9714fa106dac912d97e03'); +INSERT INTO revision_certs VALUES('b0f324986982bdf443036f061977a7895ae94f84','75810233cc39b62341d669b610e9416fd6352869','date',X'323030332d30312d30315431323a30303a3030','address@hidden',X'3b739121ed5856dfdf133d2e1dfe3064a6762bb1df8c62477923f285fd5119f7520afda97d685f1a57da80cd95d4bcecca5e13922919612f87fbab02e856cddf6ce9784ffdfa28ac75a4471800965dd39e6ca59d1096811914cd73ac3769b966921bd702a2b7e858c52526e3c03d571996eaabf5629674d9e5caf91aa61118db'); +INSERT INTO revision_certs VALUES('7a8a589d30fc17ee317ff9bacecac0d30524003a','75810233cc39b62341d669b610e9416fd6352869','author',X'74657374657240746573742e6e6574','address@hidden',X'1ff0ba8c9d5fc8076edd017eacbb7716e856266e65b1aa5f57c9989044cabaeda08db4de731dd2a3c302b0fd96844f88b6f73b2ccbb88f4a0093d55e3eee6ae28d61df1e4d7b373ccf1f4bda2809aa4a3d1c57a7bba4008f749b13a014318368964fa79808ff8d43ef4f747cd7ff929467b8d92d53ce50f14eff04c64ed47146'); +CREATE TABLE revisions + ( + id primary key, -- SHA1(text of revision) + data not null -- compressed, encoded contents of a revision + ); +INSERT INTO revisions VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a',X'1f8b08000000000000ff7d8f516ec4200c44ff730a941310020e9c655521838d849a25524253f5f675b6ab6afbd33f7b64bf9929db7ec71e4fde8fba35354ee33034fe8c776cb5f0d1d56d723c93830c73c95a0369eb0b7aca160a99e49ce54218fcf2360cdb4a71e7b33e503711902852ddd528d46b2e75653576e15e9378a9bcb5ce4d6c30645cf404614142d6002988d78298d2ac8b2630d64130f949fd4b32afa4194a30de6acaa5a0311a6c324481b5b79e263b3bc12dd95d24a50eee573a85bd4b4cc91adff96b1c4e5c3f24696db5575ce363950ecffbd7023f8fbfca7fefdf4efe81006c010000'); +INSERT INTO revisions VALUES('c81722b0236303685e341e16f0073d665090fb73',X'1f8b08000000000000ff55504b6ec4200cdde7142827000306ce128d22838d1a359354099daab72f9959b4dd3df9c9ef57f7e34e6d7ec8712efba646330ec3265ff39db6a5cad9d49483454b021cc4016682081a74e452832793cb45b106b80dc3bef27cc863794a4db93a8c820580a524d0962a3a67880383ce481ca96a97a83f12f35c9755d4d8bae785ec38a8b26f4db62b4216e21ac57a07b124f1d65a09a650051fbd71295a8d92b12b7d502b6fbf32bd8eaac77e5713a542411b4c81984423e684ac0351ce5657cd08ce6382721b94526d5713bb04a98243064bde48260c2961f402b126ca3a84eca2bde29755e8f86f4aadfdbdcceff2dd9755ea94a6c61eeac5f372bc9807ad9fbdfdb5fc138ec30f0e1419fe99010000'); +INSERT INTO revisions VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0',X'1f8b08000000000000ff45cf416ec4300805d07d4e61e5041863629f25aa226c408d34935499687afdbadd74c7e6bffff1f37acabdbded7aede711e6384fd361dfdb538edded7587d540d1a2a6dcda8220b5606c4be9192117e6c6d031755ce2c7349d0fdd2e7bef7fd4da9cb8187744b55e1192381345d145111a8b1671a02a2328aa9bef0f0bf33d3a7f2f9aa7d0cfe3b6634c805aa874e214652ce8913911792e82c9cd80843b130b0de94beefef9cfe060fc3a9f614dec150b8176774104a686aad560d01a296570587aee1f5308e13ec3ea55732c42161394deac64484db5785e4a8bac46040bd5f1f70f3f710feb44010000'); +INSERT INTO revisions VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada',X'1f8b08000000000000ff6d925d8e5b310885dfef2aacac001b8c612dd128c23656a326b9d5cd6daaeebece8c343f9abe61830f870f8f75bbda7e7af8763fafb770888765b9f99fd3d56ee7e1f73d1cd5ab6a746c15a28a786e11184b2b6370e748223224417c5996f5d24f9b3fceaf5247424b093347a69c7ab342ee6da0f67936d66ee2507b87f9d07a3f8df3c5c3619f3d9f111e96d0d6dbeeb769a156b73ec4315392a69e11d14b6c3652961c490581bdf254fa657bfbf12133c709635bafe1683a1d40642dd6cd81b92a772866b5220ce89c28b3a6f6b28410f6351c3b69d291887b42cbd1ab715165c99e64a85528a592a0cda6ede2b67d6d6afbfef9e6f4d3ff4eb221dc7d0f8769ea2ddfcfdb5be66197df73fa27f9d770d67e85d92496942a2464047cba408a1e790014eccc1914462df85f98f41926a89034628c96c05a6446a291c5120e7720e3c6c446df61a67798c8439310f436c65c3230d5d4bb3a4ce91e09f3245a5afe8039b4e728461e11a455970c38772f2317a991bb1341219d9fe81fbfd91b0491020000'); +INSERT INTO revisions VALUES('75810233cc39b62341d669b610e9416fd6352869',X'1f8b08000000000000ff458e4b6ec3300c05f73a85e01350164551673102839248d4803f852324d76fda4db70f989967d77dc8585f7a3fb7ebf453989c3bf5bd1e726ea6cfe1979e103924c1c8991820d66626b5722b52b16b16c602040fe7aebdafb7beb63fd58212e68c16138759a8248d2937d2dc72eb0486580c9a74f9805d771deaa7f129dab6ebef8b6f19edeb7fc2c979bbafc32f5018b921c52033480b4411d112cb1c4d1550a81192e0c379efc7e597a04d090a71b15a90e6ae73ca8142804ca54a0fc2a0dcf9e17e00b6033a020e010000'); +CREATE TABLE roster_deltas + ( + id primary key, -- a revision id + checksum not null, -- checksum of 'delta', to protect against disk corruption + base not null, -- joins with either rosters.id or roster_deltas.id + delta not null -- rdiff to construct current from base + ); +INSERT INTO roster_deltas VALUES('bf468e6c22dec9203af6441ad7d20b6ad8af049a','e27f3cde4a1ab7074b37f9a95e2827a5d1c54a98','c81722b0236303685e341e16f0073d665090fb73',X'1f8b08000000000000ff9d92516ec3201044ff3905e20498501cce1245d6c22e310aa595bbadd4db173b955bff95f2b9308f9901a4424c2895554248895418a4324ac497ca54595ec04718f5e0fc0808a49d0bdea11e014238e9a4d119fbe4bc895721807999e20cf5b61207d5808fb56e48857999eef4b98f3fa0bcd37a4eaa5c336728d3366a4e8ea466e748627ae3940b0d7fe549f90ccb3dd7db6e2be485677909c9ba33b9680c52f4469f20396b07c0118d0e0ef00c495b0f57f10a3c4f2ba547b4e5d8447bfa1ef923f66efda7876ef7adbf7f05f8fe06ddc1db7dbfb21fdfaba7812f78156ffba0020000'); +INSERT INTO roster_deltas VALUES('43a2235616452dca74eecf39d645a69da8e0bdd0','e27f3cde4a1ab7074b37f9a95e2827a5d1c54a98','4a1274f35812a695e357c6e7c7cd60f449f0cada',X'1f8b08000000000000ff9d92516ec3201044ff3905e20498501cce1245d6c22e310aa595bbadd4db173b955bff95f2b9308f9901a4424c2895554248895418a4324ac497ca54595ec04718f5e0fc0808a49d0bdea11e014238e9a4d119fbe4bc895721807999e20cf5b61207d5808fb56e48857999eef4b98f3fa0bcd37a4eaa5c336728d3366a4e8ea466e748627ae3940b0d7fe549f90ccb3dd7db6e2be485677909c9ba33b9680c52f4469f20396b07c0118d0e0ef00c495b0f57f10a3c4f2ba547b4e5d8447bfa1ef923f66efda7876ef7adbf7f05f8fe06ddc1db7dbfb21fdfaba7812f78156ffba0020000'); +INSERT INTO roster_deltas VALUES('c81722b0236303685e341e16f0073d665090fb73','8f1ef559aafa23fd0ac6812bfb37359aca147026','4a1274f35812a695e357c6e7c7cd60f449f0cada',X'1f8b08000000000000ff958ed10d83300c05ff99c2ca04c6312199055595133b0595d20a657fb5a02ec0fb3ce94e4f6db5660a6e705d07a0b63601e75d57de5bb3adc1e4434d1419b5d42a441838936a328c1cb5673f60c5b10ce576f8bfbd647f2edbe38c9c00202f7b9b61ca9543b45088d44a22f45203732f3a2a610ea2512a7292db11fa489bef47eb8af73f7dd9fb02912a55d206010000'); +INSERT INTO roster_deltas VALUES('4a1274f35812a695e357c6e7c7cd60f449f0cada','c0e9c55669954cdd7b1fdbcec6dd2acc34cf4bf3','75810233cc39b62341d669b610e9416fd6352869',X'1f8b08000000000000ff9551d16ec3200c7ccf5720be008c63e05baa293218da685d3a756cd2fe7e248bb6ae4fddbd59d69defce2c32d5f95c94063d9c2f99db7c5994b65ae956dedabab27a50f9b2b4b23475108c102b2009381e6d494c3e460a6381502327e37dc2e0f869189492726eacf4a8871fbe890143467296c170b6440eb18e81c1d5520c32654262ec7c6eed3ae5132fc7229b3db5635ddcd89b9ecb67b7f88d0f3ebff730a6fbd7ab858e17be3ecfcbf15622cdd7765287549142a10c202547308e2b215a162f6012b104ae06630fd3655eb99da655eb3fbc3df7cecbc17a8064c091336e2dcda12d96aa31de09d168a2a9c9bbedde167fbbf7fb892daa3a3cae73d740ffc45689527b03e818c08d64094790cc1e4bc9d545e93353140ec5241173dfc0e3bcbf0d3ccefb02ae00bba999020000'); +CREATE TABLE rosters + ( + id primary key, -- a revision id + checksum not null, -- checksum of 'data', to protect against disk corruption + data not null -- compressed, encoded contents of the roster + ); +INSERT INTO rosters VALUES('75810233cc39b62341d669b610e9416fd6352869','9c5d47769e8f04ad777bd574a6a6e2bcb658c4bc',X'1f8b08000000000000ff9d93dd6a23310c85eff314669e40b26c597e9610827f643ab44dca74dab26fbf4e32e9928185697d25231fa3f3496ae7e935cdc74f9ddec7f3c90c38ec76e67aea389961b85ec6aaa7f99abb66d23cf754cf1f9ff5cf6086937e1d3fd3cb87de5ee7719a9fcc3e37c7a25cacad5aa2054a8d9dc35443b5903955490d5c4c87dd5b9a9f8eaf697afe89e852c44df4af927d110cd666b0c404c4e2951c2a72030854993d446839d0e16ed29836bea819667d9f2f91bd5930e57c9a2f9ef72d568f929c228194ace28172add27c908c5cd539082ee261a1661656b47c64cc8f7174c6bf22b2d4bc907494ac25cfc8cedb5a5270aaa551acfd9e38d6240add08fc17c5ddc1378a9c35d5264ade5929513d1169c0929af5e2d1452160cdbc46e1d628b677e911c576dd238aedba7b1357537177f08d02b5284364892d47c77dc2ad0fc8881038e6543109a85459a3f06b14dbbbf48862bbee1145f0827d3ba8148a996ddf8ebe153d42d0e8fa9e54266f85e361f717c1138ebb19040000'); +CREATE INDEX revision_ancestry__child ON revision_ancestry (child); +CREATE INDEX revision_certs__id ON revision_certs (id); +CREATE INDEX revision_certs__name_value ON revision_certs (name, value); +PRAGMA user_version = 1598903374; +COMMIT; ============================================================ --- ChangeLog 84a29f98ca8c2f76e1e2a0b7c7ae50be47eef5d6 +++ ChangeLog 153dba95c7b8e906e7d4c9e18d8f30cfb3ddbea2 @@ -1,3 +1,39 @@ +2007-01-21 Zack Weinberg + + * schema_migration.hh (mtn_creator_code): Define the creator code + we will henceforth insert into databases. + * schema.sql: Remove manifests, manifest_certs, and manifest_deltas. + * README.changesets: Old manifest certs are no longer preserved. + + * schema_migration.cc (migration_events): Update. + (migrate_add_ccode_and_drop_manifest_tables): New migrator. + (calculate_schema_id): Rename get_schema_data; don't compute the hash. + Include the creator code in the schema data. + (temporarily_allowed_tables, compare_schema_hash): New. + (schema_to_migration): Rename find_migration; operate on a db. + (classify_schema): Use creator code instead of ad-hoc check when + deciding if this is a monotone database at all. Count tables + instead of referring to the hash to decide if the database is empty. + (diagnose_unrecognized_schema): Remove schema id from diagnostics. + (describe_sql_schema, check_sql_schema, migrate_sql_schema) + (test_migration_step): Adjust to match. + + * database.cc (database::check_format): Manifest tables might not exist. + (database::initialize): Write the creator code into the database. + (dump_user_version_cb, format_creator_code): New functions. + (database::dump): Dump the creator code. + (database::info): Report the creator code. + (database::delete_existing_manifests): Drop the tables; don't just + delete all rows. Drop manifest_certs too. + + * tests/schema_migration: Fix logic to extract schema version from + "db version" output. Add migration test for schema + 2881277287f6ee9bfc5ee255a503a6dc20dd5994. + * tests/schema_migration_bad_schema/possible.dump: Adjust so that + it is still a possible future database. + + * contrib/mashschema.cc: New developer utility program. + 2007-01-20 Zack Weinberg * database.cc (sql_contexts): Delete write-only variable and all ============================================================ --- README.changesets 5d0304f2673662993c53df4026d150b2889dcab8 +++ README.changesets 490acbb65d7c12452e9d0161c619b690de30bda3 @@ -109,10 +109,10 @@ Please be sure to do so. After backing u $ cp mydatabase.db mydatabase.db.pre-changesets Please be sure to do so. After backing up, there are three required -steps (and one optional step) to the migration. The first step is to -dump and reload your database, to get something that the new version -of SQLite can read. Note that you need to keep your old version of -monotone around at least long enough to perform this step: +steps to the migration. The first step is to dump and reload your +database, to get something that the new version of SQLite can read. +Note that you need to keep your old version of monotone around at +least long enough to perform this step: $ old-monotone --db=mydatabase.db db dump > mydatabase.dump $ monotone --db=new-mydatabase.db db load < mydatabase.dump @@ -142,15 +142,10 @@ all the necessary certs. It will therefo This should build the new changeset and revision structure and issue all the necessary certs. It will therefore take a while on a large -database. It will not, however, delete the old manifest certs: the new -version of monotone will simply ignore them, but they continue to -exist in your database, should you wish to recover some of their -information for forensic purposes in the future. If you truly wish to -remove all manifest certs as well (they are all subsumed by revision -certs now, anyways) you can issue one further command: +database. +Older versions of these instructions said that this command would +preserve the old manifest certs. As of monotone 0.33 this is no +longer the case. If you wish to refer back to the old information, +you will need to keep the old database and the old version of monotone +around. - $ monotone --db=new-mydatabase.db db execute "delete from manifest_certs" - -This will purge the manifest_certs table of all its entries, but as we -said this step is purely optional. No harm will come from old manifest -certs remaining in your database. ============================================================ --- database.cc 688eaa61e7485a72cc4593c8f90f6d0b5cf6f832 +++ database.cc f0fea1672a7803f2725c81e0b2eeebb1c47e2f15 @@ -155,23 +155,39 @@ database::check_format() database::check_format() { results res; - query manifests_query("SELECT 1 FROM manifests LIMIT 1"); - query revisions_query("SELECT 1 FROM revisions LIMIT 1"); - query rosters_query("SELECT 1 FROM rosters LIMIT 1"); - query heights_query("SELECT 1 FROM heights LIMIT 1"); - fetch(res, one_col, any_rows, revisions_query); + // Check for revisions, rosters, and heights. We need these on both sides + // of the major decision below. + fetch(res, one_col, any_rows, query("SELECT 1 FROM revisions LIMIT 1")); bool have_revisions = !res.empty(); - fetch(res, one_col, any_rows, manifests_query); - bool have_manifests = !res.empty(); - fetch(res, one_col, any_rows, rosters_query); + fetch(res, one_col, any_rows, query("SELECT 1 FROM rosters LIMIT 1")); bool have_rosters = !res.empty(); - fetch(res, one_col, any_rows, heights_query); + fetch(res, one_col, any_rows, query("SELECT 1 FROM heights LIMIT 1")); bool have_heights = !res.empty(); - - if (have_manifests) + + // Find out if the manifest tables even exist. + fetch(res, one_col, any_rows, + query("SELECT name FROM sqlite_master WHERE name LIKE 'manifest%'")); + if (res.empty()) { - I(!have_rosters); + // Must have been changesetified and rosterified already. + // Or else the database was just created. + // Do we need to regenerate cached data? + E(!have_revisions || (have_rosters && have_heights), + F("database %s lacks some cached data\n" + "run '%s db regenerate_caches' to restore use of this database") + % filename % ui.prog_name); + } + else + { + // The manifest tables exist. They should not be empty. (Empty + // manifest tables are deleted during schema migration.) + fetch(res, one_col, any_rows, query("SELECT 1 FROM manifests LIMIT 1")); + I(!res.empty()); + + // The rosters and heights tables should be empty. + I(!have_rosters && !have_heights); + // they need to either changesetify or rosterify. which? if (have_revisions) E(false, @@ -190,19 +206,6 @@ database::check_format() "please see README.changesets for details") % filename); } - else - { - // no manifests - if (have_revisions && (!have_rosters || !have_heights)) - // must be an upgrade that requires rosters be regenerated - E(false, - F("database %s lacks some cached data\n" - "run '%s db regenerate_caches' to restore use of this database") - % filename % ui.prog_name); - else - // we're all good. - ; - } } static void @@ -260,6 +263,10 @@ database::initialize() sqlite3_exec(__sql, schema_constant, NULL, NULL, NULL); assert_sqlite3_ok(__sql); + sqlite3_exec(__sql, (FL("PRAGMA user_version = %u;") + % mtn_creator_code).str().c_str(), NULL, NULL, NULL); + assert_sqlite3_ok(__sql); + // make sure what we wanted is what we got check_sql_schema(__sql, filename); } @@ -362,6 +369,19 @@ dump_index_cb(void *data, int n, char ** return 0; } +static int +dump_user_version_cb(void *data, int n, char **vals, char **cols) +{ + dump_request *dump = reinterpret_cast(data); + I(dump != NULL); + I(dump->sql != NULL); + I(vals != NULL); + I(vals[0] != NULL); + I(n == 1); + *(dump->out) << "PRAGMA user_version = " << vals[0] << ";\n"; + return 0; +} + void database::dump(ostream & out) { @@ -387,6 +407,10 @@ database::dump(ostream & out) "ORDER BY name", dump_index_cb, &req, NULL); assert_sqlite3_ok(req.sql); + res = sqlite3_exec(req.sql, + "PRAGMA user_version;", + dump_user_version_cb, &req, NULL); + assert_sqlite3_ok(req.sql); out << "COMMIT;\n"; guard.commit(); } @@ -474,6 +498,35 @@ format_sqlite_error_for_info(informative return err; } +// Subroutine of info(). Pretty-print the database's "creator code", which +// is a 32-bit unsigned number that we interpret as a four-character ASCII +// string, provided that all four characters are graphic. (On disk, it's +// stored in the "user version" field of the database.) +static string +format_creator_code(uint32_t code) +{ + char buf[5]; + string result; + + if (code == 0) + return _("not set"); + + buf[4] = '\0'; + buf[3] = ((code & 0x000000ff) >> 0); + buf[2] = ((code & 0x0000ff00) >> 8); + buf[1] = ((code & 0x00ff0000) >> 16); + buf[0] = ((code & 0xff000000) >> 24); + + if (isgraph(buf[0]) && isgraph(buf[1]) && isgraph(buf[2]) && isgraph(buf[3])) + result = (FL("%s (0x%08x)") % buf % code).str(); + else + result = (FL("0x%08x") % code).str(); + if (code != mtn_creator_code) + result += _(" (not a monotone database)"); + return result; +} + + void database::info(ostream & out) { @@ -482,12 +535,21 @@ database::info(ostream & out) // do a dummy query to confirm that the database file is an sqlite3 // database. (this doesn't happen on open() because sqlite postpones the - // actual file open until the first access. we can't piggyback this on - // any of the real queries because they all trap errors in case tables are - // missing.) + // actual file open until the first access. we can't piggyback it on the + // query of the user version because there's a bug in sqlite 3.3.10: + // the routine that reads meta-values from the database header does not + // check the file format. reported as sqlite bug #2182.) sqlite3_exec(__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0); assert_sqlite3_ok(__sql); + uint32_t ccode; + { + results res; + fetch(res, one_col, one_row, query("PRAGMA user_version")); + I(res.size() == 1); + ccode = lexical_cast(res[0][0]); + } + vector counts; counts.push_back(count("rosters")); counts.push_back(count("roster_deltas")); @@ -558,7 +620,8 @@ database::info(ostream & out) } i18n_format form = - F("schema version : %s\n" + F("creator code : %s\n" + "schema version : %s\n" "counts:\n" " full rosters : %s\n" " roster deltas : %s\n" @@ -583,6 +646,7 @@ database::info(ostream & out) " cache size : %s" ); + form = form % format_creator_code(ccode); form = form % describe_sql_schema(__sql); for (vector::iterator i = counts.begin(); i != counts.end(); i++) @@ -1867,8 +1931,9 @@ database::delete_existing_manifests() void database::delete_existing_manifests() { - execute(query("DELETE FROM manifests")); - execute(query("DELETE FROM manifest_deltas")); + execute(query("DROP TABLE IF EXISTS manifests")); + execute(query("DROP TABLE IF EXISTS manifest_deltas")); + execute(query("DROP TABLE IF EXISTS manifest_certs")); } void @@ -2524,7 +2589,7 @@ void database::complete(selector_type ty completions.clear(); // step 1: the limit is transformed into an SQL select statement which - // selects a set of IDs from the manifest_certs table which match the + // selects a set of IDs from the revision_certs table which match the // limit. this is done by building an SQL select statement for each term // in the limit and then INTERSECTing them all. ============================================================ --- schema.sql 9078e155d91e3f75159816ce77c57218ea7f6016 +++ schema.sql e98092bdf486d6d237a1e87f4ce6b404100ba6fd @@ -35,20 +35,6 @@ CREATE TABLE file_deltas unique(id, base) ); -CREATE TABLE manifests - ( - id primary key, -- strong hash of all the entries in a manifest - data not null -- compressed, encoded contents of a manifest - ); - -CREATE TABLE manifest_deltas - ( - id not null, -- strong hash of all the entries in a manifest - base not null, -- joins with either manifest.id or manifest_deltas.id - delta not null, -- rdiff to construct current from base - unique(id, base) - ); - CREATE TABLE revisions ( id primary key, -- SHA1(text of revision) @@ -91,7 +77,7 @@ CREATE INDEX revision_ancestry__child ON CREATE INDEX revision_ancestry__child ON revision_ancestry (child); --- structures for managing RSA keys and file / manifest certs +-- structures for managing RSA keys and file / revision certs CREATE TABLE public_keys ( @@ -100,17 +86,6 @@ CREATE TABLE public_keys keydata not null -- RSA public params ); -CREATE TABLE manifest_certs - ( - hash not null unique, -- hash of remaining fields separated by ":" - id not null, -- joins with manifests.id or manifest_deltas.id - name not null, -- opaque string chosen by user - value not null, -- opaque blob - keypair not null, -- joins with public_keys.id - signature not null, -- RSA/SHA1 signature of "address@hidden:val]" - unique(name, id, value, keypair, signature) - ); - CREATE TABLE revision_certs ( hash not null unique, -- hash of remaining fields separated by ":" ============================================================ --- schema_migration.cc 642fd28cf4c1e900dd1d747cbe1c8d1acec7ca02 +++ schema_migration.cc d99b978978e754e603e533613f5a59aa4136e03e @@ -8,6 +8,7 @@ // PURPOSE. #include +#include #include #include "sanity.hh" @@ -606,6 +607,37 @@ char const migrate_add_heights[] = " );" ; +// this is a function because it has to refer to the numeric constant +// defined in schema_migration.hh; also because it has to carry out the +// removal of stale tables left behind by old versions of 'changesetify', +// but only if those tables still exist and are empty. +static void +migrate_add_ccode_and_drop_manifest_tables(sqlite3 * db, app_state &) +{ + string cmd = "PRAGMA user_version = "; + cmd += boost::lexical_cast(mtn_creator_code); + sql::exec(db, cmd.c_str()); + + try + { + if (sql::value(db, "SELECT COUNT(*) FROM manifests") + + sql::value(db, "SELECT COUNT(*) FROM manifest_certs") + + sql::value(db, "SELECT COUNT(*) FROM manifest_deltas") > 0) + return; + } + catch (informative_failure & e) + { + // this should be a "no such table" error + I(string(e.what()).find("no such table") != string::npos); + return; + } + + sql::exec(db, "DROP TABLE manifests;" + "DROP TABLE manifest_certs;" + "DROP TABLE manifest_deltas;"); +} + + // these must be listed in order so that ones listed earlier override ones // listed later enum upgrade_regime @@ -666,16 +698,20 @@ const migration_event migration_events[] { "ae196843d368d042f475e3dadfed11e9d7f9f01e", migrate_add_heights, 0, upgrade_regen_caches }, + { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d", + 0, migrate_add_ccode_and_drop_manifest_tables, upgrade_none }, + // The last entry in this table should always be the current // schema ID, with 0 for the migrators. - - { "48fd5d84f1e5a949ca093e87e5ac558da6e5956d", 0, 0, upgrade_none } + { "2881277287f6ee9bfc5ee255a503a6dc20dd5994", 0, 0, upgrade_none } }; const size_t n_migration_events = (sizeof migration_events / sizeof migration_events[0]); +// The next several functions are concerned with calculating the schema hash +// and determining whether a database is usable (with or without migration). static void -calculate_schema_id(sqlite3 *db, string & ident) +get_schema_data(sqlite3 * db, string & schema) { sql stmt(db, 1, "SELECT sql FROM sqlite_master " @@ -687,7 +723,6 @@ calculate_schema_id(sqlite3 *db, string "AND name not like 'sqlite_stat%' " "ORDER BY name"); - string schema; using boost::char_separator; typedef boost::tokenizer > tokenizer; char_separator sep(" \r\n\t", "(),;"); @@ -704,21 +739,92 @@ calculate_schema_id(sqlite3 *db, string } } - hexenc tid; - calculate_ident(data(schema), tid); - ident = tid(); + uint32_t code = sql::value(db, "PRAGMA user_version"); + if (code != 0) + { + schema += " PRAGMA user_version = "; + schema += boost::lexical_cast(code); + } } -// Look through the migration_events table and return a pointer to the -// entry corresponding to schema ID, or null if it isn't there (i.e. if -// the database schema is not one we know). +// On rare occasions, a migration event leaves tables behind that will be +// deleted by a post-migration conversion. When this happens, all the +// entries in the migration-event array after that point must record the +// schema hash as it will be *after* the tables are deleted, because that is +// the state during normal operation and during migrations that start from a +// later epoch. However, that means the schema will fail to match *before* +// the tables are deleted. To handle this, we have this array, which +// records blocks of text to try removing from the schema-as-we-hash-it. +// They are removed in sequence, and we stop as soon as we get a match. +// It is database.cc's responsibility to ensure that a database in +// "migrated but not converted" state is only usable for the conversion +// operation; we report it just as if it were normally usable. +// +// Text in this array has to be in a particular format; see +// contrib/mashschema.cc for instructions. + +char const * const temporarily_allowed_tables[] = { + // manifests - obsoleted by migrate_to_revisions, but not actually dropped + // until migrate_add_ccode_and_drop_manifest_tables. + + "CREATE TABLE manifest_certs ( hash not null unique , -- hash of remaining " + "fields separated by \":\" id not null , -- joins with manifests.id or " + "manifest_deltas.id name not null , -- opaque string chosen by user value " + "not null , -- opaque blob keypair not null , -- joins with public_keys.id " + "signature not null , -- RSA/SHA1 signature of \"address@hidden:val]\" unique ( " + "name , id , value , keypair , signature ) ) CREATE TABLE manifest_deltas " + "( id not null , -- strong hash of all the entries in a manifest base " + "not null , -- joins with either manifest.id or manifest_deltas.id delta " + "not null , -- rdiff to construct current from base unique ( id , base ) " + ") CREATE TABLE manifests ( id primary key , -- strong hash of all the " + "entries in a manifest data not null -- compressed , encoded contents of " + "a manifest )", + 0 +}; + +static bool +compare_schema_hash(string const & s, string const & hash) +{ + { + hexenc tid; + calculate_ident(data(s), tid); + if (hash == tid()) + return true; + } + + string schema = s; + for (char const * const * t = temporarily_allowed_tables; *t; t++) + { + string::size_type pos = schema.find(*t); + if (pos == string::npos) + break; + + string::size_type len = strlen(*t); + if (schema.at(pos + len) == ' ') + len ++; + + schema.erase(pos, len); + + hexenc tid; + calculate_ident(data(schema), tid); + if (hash == tid()) + return true; + } + return false; +} + +// Look through the migration_events table and return a pointer to the entry +// corresponding to database DB, or null if it isn't there (i.e. if the +// database schema is not one we know). static migration_event const * -schema_to_migration(string const & id) +find_migration(sqlite3 * db) { - migration_event const *p; - for (p = migration_events + n_migration_events - 1; + string schema; + get_schema_data(db, schema); + + for (migration_event const *p = migration_events + n_migration_events - 1; p >= migration_events; p--) - if (p->id == id) + if (compare_schema_hash(schema, p->id)) return p; return 0; @@ -736,8 +842,11 @@ static schema_mismatch_case }; static schema_mismatch_case -classify_schema(sqlite3 * db, string const & id, migration_event const * m) +classify_schema(sqlite3 * db, migration_event const * m = 0) { + if (!m) + m = find_migration(db); + if (m) { if (m->migrator_sql || m->migrator_func) @@ -750,19 +859,15 @@ classify_schema(sqlite3 * db, string con // Distinguish an utterly empty database, such as is created by // "mtn db load < /dev/null", or by the sqlite3 command line utility // if you don't give it anything to do. - if (id == "da39a3ee5e6b4b0d3255bfef95601890afd80709") + if (sql::value(db, "SELECT COUNT(*) FROM sqlite_master") == 0) return SCHEMA_EMPTY; - // Every version of the schema has included tables named 'files', - // 'file_deltas', 'manifests', and 'manifest_deltas'. - // FIXME: Instead, use PRAGMA user_version to record an additional - // magic number in monotone databases. - int n = sql::value(db, - "SELECT COUNT(*) FROM sqlite_master " - "WHERE type = 'table' AND sql IS NOT NULL AND (" - " name = 'files' OR name = 'file_deltas'" - "OR name = 'manifests' OR name = 'manifest_deltas')"); - if (n != 4) + // monotone started setting this value in database headers only with + // version 0.33, but all previous versions' databases are recognized + // by their schema hashes. + + uint32_t code = sql::value(db, "PRAGMA user_version"); + if (code != mtn_creator_code) return SCHEMA_NOT_MONOTONE; return SCHEMA_TOO_NEW; @@ -773,24 +878,23 @@ describe_sql_schema(sqlite3 * db) describe_sql_schema(sqlite3 * db) { I(db != NULL); - - string id; - calculate_schema_id(db, id); - migration_event const *m = schema_to_migration(id); - schema_mismatch_case cat = classify_schema(db, id, m); + string schema; + get_schema_data(db, schema); + hexenc hash; + calculate_ident(data(schema), hash); - switch (cat) + switch (classify_schema(db)) { case SCHEMA_MATCHES: - return (F("%s (usable)") % id).str(); + return (F("%s (usable)") % hash).str(); case SCHEMA_MIGRATION_NEEDED: - return (F("%s (migration needed)") % id).str(); + return (F("%s (migration needed)") % hash).str(); case SCHEMA_TOO_NEW: - return (F("%s (too new, cannot use)") % id).str(); + return (F("%s (too new, cannot use)") % hash).str(); case SCHEMA_NOT_MONOTONE: - return (F("%s (not a monotone database)") % id).str(); + return (F("%s (not a monotone database)") % hash).str(); case SCHEMA_EMPTY: - return (F("%s (database has no tables!)") % id).str(); + return (F("%s (database has no tables!)") % hash).str(); default: I(false); } @@ -800,8 +904,7 @@ diagnose_unrecognized_schema(schema_mism // recognize. (Shared between check_sql_schema and migrate_sql_schema.) static void diagnose_unrecognized_schema(schema_mismatch_case cat, - system_path const & filename, - string const & id) + system_path const & filename) { N(cat != SCHEMA_EMPTY, F("cannot use the empty sqlite database %s\n" @@ -809,15 +912,14 @@ diagnose_unrecognized_schema(schema_mism % filename % ui.prog_name); N(cat != SCHEMA_NOT_MONOTONE, - F("%s does not appear to be a monotone database\n" - "(schema %s, core tables missing)") - % filename % id); + F("%s does not appear to be a monotone database\n") + % filename); N(cat != SCHEMA_TOO_NEW, F("%s appears to be a monotone database, but this version of\n" - "monotone does not recognize its schema (%s).\n" + "monotone does not recognize its schema.\n" "you probably need a newer version of monotone.") - % filename % id); + % filename); } // check_sql_schema is called by database.cc on open, to determine whether @@ -828,26 +930,21 @@ check_sql_schema(sqlite3 * db, system_pa { I(db != NULL); - string id; - calculate_schema_id(db, id); - migration_event const *m = schema_to_migration(id); - schema_mismatch_case cat = classify_schema(db, id, m); + schema_mismatch_case cat = classify_schema(db); - diagnose_unrecognized_schema(cat, filename, id); + diagnose_unrecognized_schema(cat, filename); N(cat != SCHEMA_MIGRATION_NEEDED, - F("database %s is laid out according to an old schema, %s\n" + F("database %s is laid out according to an old schema\n" "try '%s db migrate' to upgrade\n" "(this is irreversible; you may want to make a backup copy first)") - % filename % id % ui.prog_name); + % filename % ui.prog_name); } void migrate_sql_schema(sqlite3 * db, app_state & app) { I(db != NULL); - sql::create_function(db, "sha1", sqlite_sha1_fn); - sql::create_function(db, "unbase64", sqlite3_unbase64_fn); upgrade_regime regime = upgrade_none; @@ -860,16 +957,13 @@ migrate_sql_schema(sqlite3 * db, app_sta { transaction guard(db); - string init; - calculate_schema_id(db, init); + P(F("calculating migration...")); - P(F("calculating migration for schema %s") % init); + migration_event const *m = find_migration(db); + schema_mismatch_case cat = classify_schema(db, m); - migration_event const *m = schema_to_migration(init); - schema_mismatch_case cat = classify_schema(db, init, m); + diagnose_unrecognized_schema(cat, app.db.get_filename()); - diagnose_unrecognized_schema(cat, app.db.get_filename(), init); - // We really want 'db migrate' on an up-to-date schema to be a no-op // (no vacuum or anything, even), so that automated scripts can fire // one off optimistically and not have to worry about getting their @@ -880,14 +974,18 @@ migrate_sql_schema(sqlite3 * db, app_sta return; } - P(F("migrating data")); + sql::create_function(db, "sha1", sqlite_sha1_fn); + sql::create_function(db, "unbase64", sqlite3_unbase64_fn); + P(F("migrating data...")); + for (;;) { // confirm that we are where we ought to be - string curr; - calculate_schema_id(db, curr); - I(curr == m->id); + string schema; + get_schema_data(db, schema); + + I(compare_schema_hash(schema, m->id)); I(!m->migrator_sql || !m->migrator_func); if (m->migrator_sql) @@ -901,6 +999,7 @@ migrate_sql_schema(sqlite3 * db, app_sta m++; I(m < migration_events + n_migration_events); + P(F("migrated to schema %s") % m->id); } P(F("committing changes to database")); @@ -948,11 +1047,18 @@ test_migration_step(sqlite3 * db, app_st transaction guard(db); - migration_event const *m = schema_to_migration(schema); - N(m, F("cannot test migration from unknown schema %s") % schema); + migration_event const *m; + for (m = migration_events + n_migration_events - 1; + m >= migration_events; m--) + if (schema == m->id) + break; - N(m->migrator_sql || m->migrator_func, F("schema %s is up to date") % schema); + N(m >= migration_events, + F("cannot test migration from unknown schema %s") % schema); + N(m->migrator_sql || m->migrator_func, + F("schema %s is up to date") % schema); + L(FL("testing migration from %s to %s\n in database %s") % schema % m[1].id % app.db.get_filename()); ============================================================ --- schema_migration.hh 7d49cb153197c91b0572b76cdc72748cadfda57f +++ schema_migration.hh 75ea83a18e754dc5f3a7302bfa95de9eadbd643a @@ -34,6 +34,17 @@ void test_migration_step(sqlite3 * db, a void test_migration_step(sqlite3 * db, app_state & app, std::string const & schema); +// this constant is part of the database schema, but it is not in schema.sql +// because sqlite expressions can't do arithmetic on character values. it +// is stored in the "user version" field of the database header. when we +// encounter a database whose schema hash we don't recognize, we look for +// this code in the header to decide whether it's a monotone database or +// some other sqlite3 database. the expectation is that it will never need +// to change. we call it a creator code because it has the same format and +// function as file creator codes in old-sk00l Mac OS. + +const unsigned int mtn_creator_code = ((('_'*256 + 'M')*256 + 'T')*256 + 'N'); + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- tests/schema_migration/__driver__.lua 747b54fe959db4efd3588f495dbe627dda2174c1 +++ tests/schema_migration/__driver__.lua faccb4ba0fb263719495780b5ea830065edda37c @@ -20,10 +20,10 @@ -- to the end of this file, with the being the same -- that was in the filename (it's the schema id, if you were wondering). --------------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------- ---- Do not touch this code; you'll have to regenerate all the test ---- databases if you do! --------------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------- mtn_setup() @@ -72,13 +72,13 @@ if debugging then check(mtn("--db=latest.mtn", "db", "dump"), 0, true, false) rename("stdout", "latest.mtn.dumped") check(mtn("--db=latest.mtn", "db", "version"), 0, true, false) - local ver = string.gsub(readfile("stdout"), "^.*: (.*)%s$", "%1") + local ver = string.gsub(readfile("stdout"), "^.*: ([0-9a-f]*).*$", "%1") rename("latest.mtn.dumped", ver..".mtn.dumped") end --------------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------- ---- End untouchable code --------------------------------------------------------------------------------------------------------------------------------------------- +---------------------------------------------------------------------------- function check_migrate_from(id, need_regen_rosters) -- id.dumped is a 'db dump' of a db with schema "id" @@ -98,6 +98,9 @@ function check_migrate_from(id, need_reg check(qgrep("regenerate_caches", "stderr")) -- and we should do the regeneration check(mtn("--db="..id..".mtn", "db", "regenerate_caches"), 0, false, false) + -- after which, normal commands should work again + check(mtn("--db="..id..".mtn", "ls", "keys"), 0, false, true) + check(not qgrep("regenerate_caches", "stderr")) else -- then the migrate should not have warned us check(string.find(readfile("stderr"), "regenerate_caches") == nil) @@ -112,3 +115,4 @@ check_migrate_from("48fd5d84f1e5a949ca09 check_migrate_from("9d2b5d7b86df00c30ac34fe87a3c20f1195bb2df", true) check_migrate_from("ae196843d368d042f475e3dadfed11e9d7f9f01e", true) check_migrate_from("48fd5d84f1e5a949ca093e87e5ac558da6e5956d", false) +check_migrate_from("2881277287f6ee9bfc5ee255a503a6dc20dd5994", false) ============================================================ --- tests/schema_migration_bad_schema/possible.dump 99f9120eef90c5600b0afd4f85653b815b0ea4cb +++ tests/schema_migration_bad_schema/possible.dump 87858caeefa73d827b8a072fd8578448140b88eb @@ -32,28 +32,6 @@ CREATE TABLE heights height not null, -- complex height, array of big endian u32 integers unique(revision, height) ); -CREATE TABLE manifest_certs - ( - hash not null unique, -- hash of remaining fields separated by ":" - id not null, -- joins with manifests.id or manifest_deltas.id - name not null, -- opaque string chosen by user - value not null, -- opaque blob - keypair not null, -- joins with public_keys.id - signature not null, -- RSA/SHA1 signature of "address@hidden:val]" - unique(name, id, value, keypair, signature) - ); -CREATE TABLE manifest_deltas - ( - id not null, -- strong hash of all the entries in a manifest - base not null, -- joins with either manifest.id or manifest_deltas.id - delta not null, -- rdiff to construct current from base - unique(id, base) - ); -CREATE TABLE manifests - ( - id primary key, -- strong hash of all the entries in a manifest - data not null -- compressed, encoded contents of a manifest - ); CREATE TABLE next_roster_node_number ( node primary key -- only one entry in this table, ever @@ -102,12 +80,13 @@ CREATE TABLE skip_deltas ( id not null, -- strong hash of target file contents base not null, -- joins with files.id - base_manifest not null, -- joins with manifests.id - target_manifest not null, -- joins with manifests.id + base_revision not null, -- joins with revisions.id + target_revision not null, -- joins with revisions.id delta not null, -- rdiff to construct current from base - unique(id, base, base_manifest, target_manifest) + unique(id, base, base_revision, target_revision) ); CREATE INDEX revision_ancestry__child ON revision_ancestry (child); CREATE INDEX revision_certs__id ON revision_certs (id); CREATE INDEX revision_certs__name_value ON revision_certs (name, value); +PRAGMA user_version = 1598903374; COMMIT;