# # # add_dir "tests/git_export_invalid_mapped_author" # # add_dir "tests/git_export_unmapped_authors" # # add_file "tests/git_export_invalid_mapped_author/__driver__.lua" # content [669a0925ac0812f7dc8e744f31bee795b5b5d638] # # add_file "tests/git_export_unmapped_authors/__driver__.lua" # content [374bdf641ec62d23aece556a162a4f9fbc65a290] # # patch "NEWS" # from [77892b9cee39daedbc6cc3f3d53249e8910e5d04] # to [eaba3c27e16979f78844a631df129d4aed969dbf] # # patch "cmd_othervcs.cc" # from [fc9779a1fb9949632276824c09be2270eef5dc96] # to [6894f2644573582dadfeccb713ae3362c2953189] # # patch "git_export.cc" # from [2dde3cdd8b8215c44ad3df6e755a9c5c6cda86cd] # to [05a325fef7bd56f529e55db02fd2542acd5d06f7] # # patch "git_export.hh" # from [4d57e2dfe57a171318f8821a288dcabc7a7ea1ed] # to [6888ff1cc0d73bf0b30ba975d6c4050bb345d9de] # # patch "lua_hooks.cc" # from [91186ecf4c978d1c000103fafbc841bec2c20f85] # to [9e358c5c997d6ef28d41c463b16b341182e77f6c] # # patch "lua_hooks.hh" # from [01561e2076fc77e99edcff6d2644d9e62b3037fb] # to [e5ae9443d2eb45636e38769fc41839b03fd340f7] # # patch "monotone.texi" # from [a2edd01e7932280ff956deeb57de9daa33e8e4b8] # to [91c774105b13a0ea0f2e1daacf912ac5d62c9e5b] # # patch "std_hooks.lua" # from [70b19548f8b92148147f8fb774e5f5d5c0ba8cc5] # to [e8034da4b8f883a783910810714366dd920a57a1] # # patch "tests/git_export/__driver__.lua" # from [960dc93a9e23d634388d55edcebf81227200ceee] # to [ca3862a9039ee4e3f52329a6d0cfad08c347cc33] # # patch "tests/git_export_rename_loop/__driver__.lua" # from [a1e970df185b451d158f71d0897845d9befa1b89] # to [eedaf865fe84341716765e16f1f0b971b0d93441] # ============================================================ --- tests/git_export_invalid_mapped_author/__driver__.lua 669a0925ac0812f7dc8e744f31bee795b5b5d638 +++ tests/git_export_invalid_mapped_author/__driver__.lua 669a0925ac0812f7dc8e744f31bee795b5b5d638 @@ -0,0 +1,22 @@ +mtn_setup() + +writefile("file1", "file1") + +check(mtn("add", "file1"), 0, false, false) +commit() + +-- attempt to export the monotone history with a bad author map + +writefile("author.map", "address@hidden = \n") + +check(mtn("git_export", "--authors-file", "author.map"), 1, false, true) +check(qgrep("invalid git author", "stderr")) + +-- attempt to export the monotone history with a good author map + +writefile("author.map", "address@hidden = Tester \n") + +check(mtn("git_export", "--authors-file", "author.map"), 0, true, true) +check(not qgrep("invalid git author", "stderr")) +check(qgrep("author Tester ", "stdout")) +check(qgrep("committer Tester ", "stdout")) ============================================================ --- tests/git_export_unmapped_authors/__driver__.lua 374bdf641ec62d23aece556a162a4f9fbc65a290 +++ tests/git_export_unmapped_authors/__driver__.lua 374bdf641ec62d23aece556a162a4f9fbc65a290 @@ -0,0 +1,33 @@ +mtn_setup() + +writefile("file1", "file1") +check(mtn("add", "file1"), 0, false, false) +check(mtn("commit", "-m", "test1", "--author", "address@hidden"), 0, false, true) + +writefile("file1", "file1x") +check(mtn("commit", "-m", "test2", "--author", ""), 0, false, true) + +writefile("file1", "file1y") +check(mtn("commit", "-m", "test3", "--author", "tester3"), 0, false, true) + +writefile("file1", "file1z") +check(mtn("commit", "-m", "test4", "--author", "tester4 "), 0, false, true) + +check(mtn("log"), 0, true, false) + +-- export the monotone history + +check(mtn("git_export"), 0, true, true) +check(qgrep("committer tester ", "stdout")) +check(qgrep("author tester1 ", "stdout")) +check(qgrep("author tester2 ", "stdout")) +check(qgrep("author tester3 ", "stdout")) +check(qgrep("author tester4 ", "stdout")) + +-- one more commit with an invalid author + +writefile("file1", "file1zz") +check(mtn("commit", "-m", "test4", "--author", "' author used by the git_export command has + changed to 'Unknown ' and must be changed in existing author + map files. The old '' author will be rejected by the new + validate_git_author lua hook. + + - The 'git_export' command now validates all git author and committer + values using a new 'validate_git_author' lua hook before they are + written to the output stream. The export will fail if any value is + rejected by this hook. + + - The 'git_export' command now calls a new 'unmapped_git_author' lua + hook for all git author values not found in the author map file. The + default implementation of this hook attempts to produce valid git + authors using several default pattern replacements. + New features - Added portuguese translation (thanks to Américo Monteiro) ============================================================ --- cmd_othervcs.cc fc9779a1fb9949632276824c09be2270eef5dc96 +++ cmd_othervcs.cc 6894f2644573582dadfeccb713ae3362c2953189 @@ -90,6 +90,7 @@ CMD(git_export, "git_export", "", CMD_RE { P(F("reading author mappings from '%s'") % app.opts.authors_file); read_mappings(app.opts.authors_file, author_map); + validate_author_mappings(app.lua, author_map); } if (!app.opts.branches_file.empty()) @@ -122,7 +123,7 @@ CMD(git_export, "git_export", "", CMD_RE load_changes(db, revisions, change_map); // needs author and branch maps - export_changes(db, + export_changes(db, app.lua, revisions, marked_revs, author_map, branch_map, change_map, app.opts.log_revids, app.opts.log_certs, ============================================================ --- git_export.cc 2dde3cdd8b8215c44ad3df6e755a9c5c6cda86cd +++ git_export.cc 05a325fef7bd56f529e55db02fd2542acd5d06f7 @@ -14,6 +14,7 @@ #include "file_io.hh" #include "git_change.hh" #include "git_export.hh" +#include "lua_hooks.hh" #include "outdated_indicator.hh" #include "project.hh" #include "revision.hh" @@ -81,6 +82,19 @@ void } void +validate_author_mappings(lua_hooks & lua, + map const & authors) +{ + for (map::const_iterator i = authors.begin(); + i != authors.end(); ++i) + { + E(lua.hook_validate_git_author(i->second), origin::user, + F("invalid git author '%s' mapped from monotone author '%s'") + % i->second % i->first); + } +} + +void import_marks(system_path const & marks_file, map & marked_revs) { @@ -135,16 +149,16 @@ load_changes(database & db, vector const & revisions, map & change_map) { - // process revisions in reverse order and calculate the file changes for + // process revisions in reverse order and calculate the git changes for // each revision. these are cached in a map for use in the export phase // where revisions are processed in forward order. this trades off memory // for speed, loading rosters in reverse order is ~5x faster than loading - // them in forward order and the memory required for file changes is + // them in forward order and the memory required for git changes is // generally quite small. the memory required here should be comparable to // that for all of the revision texts in the database being exported. // // testing exports of a current monotone database with ~18MB of revision - // text in ~15K revisions and a current piding database with ~20MB of + // text in ~15K revisions and a current pidgin database with ~20MB of // revision text in ~27K revisions indicate that this is a reasonable // approach. the export process reaches around 203MB VSS and 126MB RSS // for the monotone database and around 206MB VSS and 129MB RSS for the @@ -183,7 +197,7 @@ void } void -export_changes(database & db, +export_changes(database & db, lua_hooks & lua, vector const & revisions, map & marked_revs, map const & author_map, @@ -209,6 +223,9 @@ export_changes(database & db, ticker exported(_("exporting"), "r", 1); exported.set_total(revisions.size()); + // keep a map of valid authors to avoid redundant lua validation calls + map valid_authors(author_map); + for (vector::const_iterator r = revisions.begin(); r != revisions.end(); ++r) { @@ -246,8 +263,8 @@ export_changes(database & db, // default to committer and author if no author certs exist // this may be mapped to a different value with the authors-file option - string author_name = ""; // used as the git author - string author_key = ""; // used as the git committer + string author_name = "Unknown "; // used as the git author + string author_key = "Unknown "; // used as the git committer date_t author_date = date_t::now(); cert_iterator author = authors.begin(); @@ -270,24 +287,50 @@ export_changes(database & db, // using the 'db execute' command. the following queries will list all // author keys and author cert values. // - // 'select distinct keypair from revision_certs' + // all values from author certs: + // // 'select distinct value from revision_certs where name = "author"' + // + // all keys that have signed author certs: + // + // 'select distinct public_keys.name + // from public_keys + // left join revision_certs on revision_certs.keypair_id = public_keys.id + // where revision_certs.name = "author"' - lookup_iterator key_lookup = author_map.find(author_key); + lookup_iterator key_lookup = valid_authors.find(author_key); - if (key_lookup != author_map.end()) - author_key = key_lookup->second; - else if (author_key.find('<') == string::npos && - author_key.find('>') == string::npos) - author_key = "<" + author_key + ">"; + if (key_lookup != valid_authors.end()) + { + author_key = key_lookup->second; + } + else + { + string unmapped_key; + lua.hook_unmapped_git_author(author_key, unmapped_key); + E(lua.hook_validate_git_author(unmapped_key), origin::user, + F("invalid git author '%s' from monotone author key '%s'") + % unmapped_key % author_key); + valid_authors.insert(make_pair(author_key, unmapped_key)); + author_key = unmapped_key; + } - lookup_iterator name_lookup = author_map.find(author_name); + lookup_iterator name_lookup = valid_authors.find(author_name); - if (name_lookup != author_map.end()) - author_name = name_lookup->second; - else if (author_name.find('<') == string::npos && - author_name.find('>') == string::npos) - author_name = "<" + author_name + ">"; + if (name_lookup != valid_authors.end()) + { + author_name = name_lookup->second; + } + else + { + string unmapped_name; + lua.hook_unmapped_git_author(author_name, unmapped_name); + E(lua.hook_validate_git_author(unmapped_name), origin::user, + F("invalid git author '%s' from monotone author value '%s'") + % unmapped_name % author_name); + valid_authors.insert(make_pair(author_name, unmapped_name)); + author_name = unmapped_name; + } cert_iterator date = dates.begin(); ============================================================ --- git_export.hh 4d57e2dfe57a171318f8821a288dcabc7a7ea1ed +++ git_export.hh 6888ff1cc0d73bf0b30ba975d6c4050bb345d9de @@ -13,6 +13,10 @@ void read_mappings(system_path const & p void read_mappings(system_path const & path, std::map & mappings); +void validate_author_mappings(lua_hooks & lua, + std::map const & authors); + void import_marks(system_path const & marks_file, std::map & marked_revs); @@ -23,7 +27,7 @@ void load_changes(database & db, std::vector const & revisions, std::map & change_map); -void export_changes(database & db, +void export_changes(database & db, lua_hooks & lua, std::vector const & revisions, std::map & marked_revs, std::map const & author_map, ============================================================ --- lua_hooks.cc 91186ecf4c978d1c000103fafbc841bec2c20f85 +++ lua_hooks.cc 9e358c5c997d6ef28d41c463b16b341182e77f6c @@ -1253,6 +1253,30 @@ lua_hooks::hook_note_mtn_startup(args_ve return ll.ok(); } +bool +lua_hooks::hook_unmapped_git_author(string const & unmapped_author, string & fixed_author) +{ + return Lua(st) + .func("unmapped_git_author") + .push_str(unmapped_author) + .call(1,1) + .extract_str(fixed_author) + .ok(); +} + +bool +lua_hooks::hook_validate_git_author(string const & author) +{ + bool valid = false, exec_ok = false; + exec_ok = Lua(st) + .func("validate_git_author") + .push_str(author) + .call(1,1) + .extract_bool(valid) + .ok(); + return valid && exec_ok; +} + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- lua_hooks.hh 01561e2076fc77e99edcff6d2644d9e62b3037fb +++ lua_hooks.hh e5ae9443d2eb45636e38769fc41839b03fd340f7 @@ -196,6 +196,12 @@ public: size_t revs_in, size_t revs_out, size_t keys_in, size_t keys_out); bool hook_note_mtn_startup(args_vector const & args); + + // git export hooks + bool hook_unmapped_git_author(std::string const & unmapped_author, + std::string & fixed_author); + bool hook_validate_git_author(std::string const & author); + }; #endif // __LUA_HOOKS_HH__ ============================================================ --- monotone.texi a2edd01e7932280ff956deeb57de9daa33e8e4b8 +++ monotone.texi 91c774105b13a0ea0f2e1daacf912ac5d62c9e5b @@ -4025,14 +4025,14 @@ @section Exporting to GIT Monotone author names often look like raw email addresses such as @code{"user@@example.com"}. These are not considered valid by git -which requires leading `<' and trailing `>' characters around email -addresses and by convention often includes the display name with the -email address such as @code{"User Name "}. The git -exporter deals with this difference in several ways: +which requires the display name and leading `<' and trailing `>' +characters around email addresses such as @code{"User Name +"}. The git exporter deals with this difference in +several ways: @itemize @item revisions that don't have any author certs will default to using address@hidden} for both the author and committer. address@hidden } for both the author and committer. @item revisions that have one or more author certs will use the value of one author cert as the author and the key used to sign this cert as @@ -4042,10 +4042,14 @@ @section Exporting to GIT the @option{--authors-file} option described in @ref{GIT} and translated to the specified value if found. @item -any author or committer that does not contain both `<' and `>' -characters will have an initial ``<' character and a final `>' -character added so that they may form valid values. +any author or committer value not found in the authors file will be +processed by the @code{unmapped_git_author} hook which may adjust the +value so that it represents a valid value. @end itemize +All git author and committer values will be validated by the address@hidden hook before being written to the output +stream. The export will abort if any author or committer value is +rejected by the validation hook. Branch names used by monotone are allowed to contain characters that are not considered valid by git. These may be mapped to other names @@ -10056,7 +10060,10 @@ @section GIT from a monotone database with the following sql query: @verbatim -$ mtn db execute 'select distinct keypair from revision_certs' where name = "author"' +$ mtn db execute 'select distinct public_keys.name + from public_keys + left join revision_certs on revision_certs.keypair_id = public_keys.id + where revision_certs.name = "author"' @end verbatim The @option{--branches-file} option may be used to map monotone branch @@ -10068,9 +10075,9 @@ @section GIT monotone-branch-name = git-branch-name @end verbatim -Revisions with no author cert will use for both the author -and the committer. These can be mapped to other values using the address@hidden option. +Revisions with no author cert will use "Unknown " for both +the author and the committer. These can be mapped to other values +using the @var{authors-file} option. The list of branches that might need to be mapped can be extracted from a monotone database with using the @command{ls branches} command: @@ -11164,6 +11171,37 @@ @subsection Attribute Handling @end ftable address@hidden GIT Export Hooks + +Exporting monotone revisions in git-fast-import(1) format often +requires translation of monotone author cert values and associated +signing keys into corresponding git author and committer +values. Translation of author and committer values and validation of +the results is controlled by these hooks. See @ref{Default hooks}. + address@hidden @code + address@hidden unmapped_git_author(@var{author}) + +This hook is called for any git author or committer value that does +not come from the current @emph{author map} file. If no @emph{author +map} file is specified this hook will be called for @emph{every} +unique git author and committer value. It may return the value +unchanged or modify it in some way in an effort to ensure that it is +valid. The default implementation attempts several common pattern +replacements to produce valid authors from monotone authors. + address@hidden validate_git_author(@var{author}) + +This hook is called before the git author or committer value is +written to the export output stream. The @var{author} value is either +the mapped value from the current @emph{author map} file or the value +produced by the @code{unmapped_git_author} hook. This hook may return +true if the author is valid or false if it is not. The export will be +aborted if this hook returns false for any value. + address@hidden ftable + @subsection Validation Hooks If there is a policy decision to make, Monotone defines certain hooks to ============================================================ --- std_hooks.lua 70b19548f8b92148147f8fb774e5f5d5c0ba8cc5 +++ std_hooks.lua e8034da4b8f883a783910810714366dd920a57a1 @@ -1399,3 +1399,38 @@ end return push_hook_functions(notifier) end end + +-- to ensure only mapped authors are allowed through +-- return "" from unmapped_git_author +-- and validate_git_author will fail + +function unmapped_git_author(author) + -- replace "address@hidden" with "foo " + name = author:match("^([^<>]+)@[^<>]+$") + if name then + return name .. " <" .. author .. ">" + end + + -- replace "" with "foo " + name = author:match("^<([^<>]+)@[^<>]+>$") + if name then + return name .. " " .. author + end + + -- replace "foo" with "foo " + name = author:match("^[^<>@]+$") + if name then + return name .. " <" .. name .. ">" + end + + return author -- unchanged +end + +function validate_git_author(author) + -- ensure author matches the "Name " format git expects + if author:match("^[^<]+ <[^>]*>$") then + return true + end + + return false +end ============================================================ --- tests/git_export/__driver__.lua 960dc93a9e23d634388d55edcebf81227200ceee +++ tests/git_export/__driver__.lua ca3862a9039ee4e3f52329a6d0cfad08c347cc33 @@ -2,7 +2,7 @@ mtn_setup() mtn_setup() -writefile("author.map", "address@hidden = \n") +writefile("author.map", "address@hidden = other \n") writefile("file1", "file1") writefile("file2", "file2") @@ -81,8 +81,8 @@ check(indir("git.dir", {"git", "log", "- check(indir("mtn.dir", mtn("log")), 0, true, false) check(qgrep("Author: address@hidden", "stdout")) check(indir("git.dir", {"git", "log", "--summary", "--pretty=raw"}), 0, true, false) -check(qgrep("author ", "stdout")) -check(qgrep("committer ", "stdout")) +check(qgrep("author other ", "stdout")) +check(qgrep("committer other ", "stdout")) -- check branch refs ============================================================ --- tests/git_export_rename_loop/__driver__.lua a1e970df185b451d158f71d0897845d9befa1b89 +++ tests/git_export_rename_loop/__driver__.lua eedaf865fe84341716765e16f1f0b971b0d93441 @@ -2,7 +2,7 @@ mtn_setup() mtn_setup() -writefile("author.map", "address@hidden = \n") +writefile("author.map", "address@hidden = tester \n") writefile("file1", "file1") writefile("file2", "file2")