pax_global_header00006660000000000000000000000064121461607440014517gustar00rootroot0000000000000052 comment=aac396a10f4ba01edaed608f6957bb9452c8df49 svn-all-fast-export-1.0.10/000077500000000000000000000000001214616074400154245ustar00rootroot00000000000000svn-all-fast-export-1.0.10/.gitignore000066400000000000000000000000711214616074400174120ustar00rootroot00000000000000Makefile *~ *.o svn-all-fast-export src/local-config.pri svn-all-fast-export-1.0.10/fast-export2.pro000066400000000000000000000004051214616074400205030ustar00rootroot00000000000000###################################################################### # Automatically generated by qmake (2.01a) dim. dec. 23 13:48:52 2007 ###################################################################### TEMPLATE = subdirs # Directories SUBDIRS = src svn-all-fast-export-1.0.10/samples/000077500000000000000000000000001214616074400170705ustar00rootroot00000000000000svn-all-fast-export-1.0.10/samples/ignore-branch.rules000066400000000000000000000010661214616074400226650ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository myproject end repository # # Declare the rules # Note: rules must end in a slash # match /trunk/ repository myproject branch master end match # Ignore this branch: # We ignore a branch by not telling svn-all-fast-export what to do # with this path # Note that rules are applied in order of appearance, so this rule # must appear before the generic branch rule match /branches/abandoned-work/ end match match /branches/([^/]+)/ repository myproject branch \1 end match # No tag processing svn-all-fast-export-1.0.10/samples/merged-branches-tags.rules000066400000000000000000000012051214616074400241240ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository myproject end repository # # Declare the rules # Note: rules must end in a slash # match /trunk/ repository myproject branch master end match # Subversion doesn't understand the Git concept of tags # In Subversion, tags are really branches # # Only a post-processing (i.e., after converting to Git) of the tag # branches can we be sure that a tag wasn't moved or changed from the # branch it was copied from # # So we don't pretend that SVN tags are Git tags and then import # everything as one match /(branches|tags)/([^/]+)/ repository myproject branch \2 end match svn-all-fast-export-1.0.10/samples/min-max-revision.rules000066400000000000000000000013501214616074400233450ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository myproject end repository # # Declare the rules # Note: rules must end in a slash # # Ignore this particular revision in trunk # See ignore-branch.rules first # Note that rules are applied in order of appearance, so this rule # must appear before the generic trunk rule match /trunk/ min revision 123 max revision 123 end match # Stop importing trunk # If we don't specify a max revision, then there is no limit # The same applies to min revision (i.e., min revision is 0) match /trunk/ min revision 1234 end match match /trunk/ repository myproject branch master end match match /branches/([^/]+)/ repository myproject branch \1 end match # No tag processing svn-all-fast-export-1.0.10/samples/recurse.rules000066400000000000000000000012461214616074400216170ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository project1 end repository create repository project2 end repository # # Declare the rules # Note: rules must end in a slash # match /trunk/([^/]+)/ repository \1 branch master end match # # SVN layout: # /branches/branchname/project1 # /branches/branchname/project2 match /branches/([^/]+)/([^/]+)/ repository \2 branch \1 end match # # Example of the recurse rule: # We tell svn-all-fast-export to not import anything # but to go inside and recurse in the subdirs # Note how the ending slash is missing in this particular case match /branches/[^/]+ action recurse end match # No tag processing svn-all-fast-export-1.0.10/samples/standardlayout.rules000066400000000000000000000014141214616074400232020ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository myproject end repository # # Declare the rules # Note: rules must end in a slash # match /trunk/ repository myproject branch master end match match /branches/([^/]+)/ repository myproject branch \1 end match # Important: # Subversion doesn't understand the Git concept of tags # In Subversion, tags are really branches # # Only a post-processing (i.e., after converting to Git) of the tag # branches can we be sure that a tag wasn't moved or changed from the # branch it was copied from # # This rule will create tags that don't exist in any of the # branches. It's not what you want. # See the merged-branches-tags.rules file match /tags/([^/]+)/ repository myproject branch refs/tags/\1 end match svn-all-fast-export-1.0.10/samples/two-projects.rules000066400000000000000000000007501214616074400226060ustar00rootroot00000000000000# # Declare the repositories we know about: # create repository project1 end repository create repository project2 end repository # # Declare the rules # Note: rules must end in a slash # match /project1/trunk/ repository project1 branch master end match match /project2/trunk/ repository project2 branch master end match # Note how we can use regexp to capture the repository name match /([^/]+)/branches/([^/]+)/ repository \1 branch \2 end match # No tag processing svn-all-fast-export-1.0.10/src/000077500000000000000000000000001214616074400162135ustar00rootroot00000000000000svn-all-fast-export-1.0.10/src/CommandLineParser.cpp000066400000000000000000000313431214616074400222660ustar00rootroot00000000000000/* * This file is part of the vng project * Copyright (C) 2008 Thomas Zander * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "CommandLineParser.h" #include #include #include #include #include CommandLineParser *CommandLineParser::self = 0; class CommandLineParser::Private { public: Private(int argc, char **argv); // functions void addDefinitions(const CommandLineOption * options); void setArgumentDefinition(const char *definition); void parse(); // variables; const int argumentCount; char ** const argumentStrings; bool dirty; int requiredArguments; QString argumentDefinition; struct OptionDefinition { OptionDefinition() : optionalParameters(0), requiredParameters(0) { } QString name; QString comment; QChar shortName; int optionalParameters; int requiredParameters; }; // result of what the user typed struct ParsedOption { QString option; QList parameters; }; QList definitions; QHash options; QList arguments; QList undefinedOptions; QList errors; }; CommandLineParser::Private::Private(int argc, char **argv) : argumentCount(argc), argumentStrings(argv), dirty(true), requiredArguments(0) { } void CommandLineParser::Private::addDefinitions(const CommandLineOption * options) { for (int i=0; options[i].specification != 0; i++) { OptionDefinition definition; QString option = QString::fromLatin1(options[i].specification); // options with optional params are written as "--option required[, optional] if (option.indexOf(QLatin1Char(',')) >= 0 && ( option.indexOf(QLatin1Char('[')) < 0 || option.indexOf(QLatin1Char(']')) < 0) ) { QStringList optionParts = option.split(QLatin1Char(','), QString::SkipEmptyParts); if (optionParts.count() != 2) { qWarning() << "WARN: option definition '" << option << "' is faulty; only one ',' allowed"; continue; } foreach (QString s, optionParts) { s = s.trimmed(); if (s.startsWith(QLatin1String("--")) && s.length() > 2) definition.name = s.mid(2); else if (s.startsWith(QLatin1String("-")) && s.length() > 1) definition.shortName = s.at(1); else { qWarning() << "WARN: option definition '" << option << "' is faulty; the option should start with a -"; break; } } } else if (option.startsWith(QLatin1String("--")) && option.length() > 2) definition.name = option.mid(2); else qWarning() << "WARN: option definition '" << option << "' has unrecognized format. See the api docs for CommandLineParser for a howto"; if(definition.name.isEmpty()) continue; if (option.indexOf(QLatin1Char(' ')) > 0) { QStringList optionParts = definition.name.split(QLatin1Char(' '), QString::SkipEmptyParts); definition.name = optionParts[0]; bool first = true; foreach (QString s, optionParts) { if (first) { first = false; continue; } s = s.trimmed(); if (s[0].unicode() == '[' && s.endsWith(QLatin1Char(']'))) definition.optionalParameters++; else definition.requiredParameters++; } } definition.comment = QString::fromLatin1(options[i].description); definitions << definition; } /* foreach (OptionDefinition def, definitions) { qDebug() << "definition:" << (def.shortName != 0 ? def.shortName : QChar(32)) << "|" << def.name << "|" << def.comment << "|" << def.requiredParameters << "+" << def.optionalParameters; } */ dirty = true; } void CommandLineParser::Private::setArgumentDefinition(const char *defs) { requiredArguments = 0; argumentDefinition = QString::fromLatin1(defs); QStringList optionParts = argumentDefinition.split(QLatin1Char(' '), QString::SkipEmptyParts); bool inArg = false; foreach (QString s, optionParts) { s = s.trimmed(); if (s[0].unicode() == '<') { inArg = true; requiredArguments++; } else if (s[0].unicode() == '[') inArg = true; if (s.endsWith(QLatin1Char('>'))) inArg = false; else if (!inArg) requiredArguments++; } } void CommandLineParser::Private::parse() { if (dirty == false) return; errors.clear(); options.clear(); arguments.clear(); undefinedOptions.clear(); dirty = false; class OptionProcessor { public: OptionProcessor(Private *d) : clp(d) { } void next(Private::ParsedOption &option) { if (! option.option.isEmpty()) { // find the definition to match. OptionDefinition def; foreach (Private::OptionDefinition definition, clp->definitions) { if (definition.name == option.option) { def = definition; break; } } if (! def.name.isEmpty() && def.requiredParameters >= option.parameters.count() && def.requiredParameters + def.optionalParameters <= option.parameters.count()) clp->options.insert(option.option, option); else if (!clp->undefinedOptions.contains(option.option)) clp->undefinedOptions << option.option; else clp->errors.append(QLatin1String("Not enough arguments passed for option `") + option.option +QLatin1Char('\'')); } option.option.clear(); option.parameters.clear(); } private: CommandLineParser::Private *clp; }; OptionProcessor processor(this); bool optionsAllowed = true; ParsedOption option; OptionDefinition currentDefinition; for (int i = 1; i < argumentCount; i++) { QString arg = QString::fromLocal8Bit(argumentStrings[i]); if (optionsAllowed) { if (arg == QLatin1String("--")) { optionsAllowed = false; continue; } if (arg.startsWith(QLatin1String("--"))) { processor.next(option); int end = arg.indexOf(QLatin1Char('=')); option.option = arg.mid(2, end - 2); if (end > 0) option.parameters << arg.mid(end+1); continue; } if (arg[0].unicode() == '-' && arg.length() > 1) { for (int x = 1; x < arg.length(); x++) { bool resolved = false; foreach (OptionDefinition definition, definitions) { if (definition.shortName == arg[x]) { resolved = true; processor.next(option); currentDefinition = definition; option.option = definition.name; if (definition.requiredParameters == 1 && arg.length() >= x+2) { option.parameters << arg.mid(x+1, arg.length()); x = arg.length(); } break; } } if (!resolved) { // nothing found; copy char so it ends up in unrecognized option.option = arg[x]; processor.next(option); } } continue; } } if (! option.option.isEmpty()) { if (currentDefinition.name != option.option) { // find the definition to match. foreach (OptionDefinition definition, definitions) { if (definition.name == option.option) { currentDefinition = definition; break; } } } if (currentDefinition.requiredParameters + currentDefinition.optionalParameters <= option.parameters.count()) processor.next(option); } if (option.option.isEmpty()) arguments << arg; else option.parameters << arg; } processor.next(option); if (requiredArguments > arguments.count()) errors.append(QLatin1String("Not enough arguments, usage: ") + QString::fromLocal8Bit(argumentStrings[0]) + QLatin1Char(' ') + argumentDefinition); /* foreach (QString key, options.keys()) { ParsedOption p = options[key]; qDebug() << "-> " << p.option; foreach (QString v, p.parameters) qDebug() << " +" << v; } qDebug() << "---"; foreach (QString arg, arguments) { qDebug() << arg; } */ } // ----------------------------------- // static void CommandLineParser::init(int argc, char **argv) { if (self) delete self; self = new CommandLineParser(argc, argv); } // static void CommandLineParser::addOptionDefinitions(const CommandLineOption * optionList) { if (!self) { qWarning() << "WARN: CommandLineParser:: Use init before addOptionDefinitions!"; return; } self->d->addDefinitions(optionList); } // static CommandLineParser *CommandLineParser::instance() { return self; } // static void CommandLineParser::setArgumentDefinition(const char *definition) { if (!self) { qWarning() << "WARN: CommandLineParser:: Use init before addOptionDefinitions!"; return; } self->d->setArgumentDefinition(definition); } CommandLineParser::CommandLineParser(int argc, char **argv) : d(new Private(argc, argv)) { } CommandLineParser::~CommandLineParser() { delete d; } void CommandLineParser::usage(const QString &name, const QString &argumentDescription) { QTextStream cout(stdout, QIODevice::WriteOnly); cout << "Usage: " << d->argumentStrings[0]; if (! name.isEmpty()) cout << " " << name; if (d->definitions.count()) cout << " [OPTION]"; if (! argumentDescription.isEmpty()) cout << " " << argumentDescription; cout << endl << endl; if (d->definitions.count() > 0) cout << "Options:" << endl; int commandLength = 0; foreach (Private::OptionDefinition definition, d->definitions) commandLength = qMax(definition.name.length(), commandLength); foreach (Private::OptionDefinition definition, d->definitions) { cout << " "; if (definition.shortName == 0) cout << " --"; else cout << "-" << definition.shortName << " --"; cout << definition.name; for (int i = definition.name.length(); i <= commandLength; i++) cout << ' '; cout << definition.comment <parse(); return d->options.keys(); } bool CommandLineParser::contains(const QString & key) const { d->parse(); return d->options.contains(key); } QStringList CommandLineParser::arguments() const { d->parse(); return d->arguments; } QStringList CommandLineParser::undefinedOptions() const { d->parse(); return d->undefinedOptions; } QString CommandLineParser::optionArgument(const QString &optionName, const QString &defaultValue) const { QStringList answer = optionArguments(optionName); if (answer.isEmpty()) return defaultValue; return answer.first(); } QStringList CommandLineParser::optionArguments(const QString &optionName) const { if (! contains(optionName)) return QStringList(); Private::ParsedOption po = d->options[optionName]; return po.parameters; } QStringList CommandLineParser::parseErrors() const { d->parse(); return d->errors; } svn-all-fast-export-1.0.10/src/CommandLineParser.h000066400000000000000000000071101214616074400217260ustar00rootroot00000000000000/* * This file is part of the vng project * Copyright (C) 2008 Thomas Zander * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef COMMANDLINEPARSER_H #define COMMANDLINEPARSER_H #include struct CommandLineOption { /** * The specification of an option includes the name of the option the user must pass and optional arguments it has. * Example specifications are; *
    *
  1. "-a, --all"
    2. *
  2. "--version"
    3. *
  3. "--type name"
    4. *
  4. "--list item[,item]"
    5. *
  5. "-f, --format name [suffix] [foo]"
* Number 1 allows the user to either type -a or --all (or /A on Windows) to activate this option. * Number 2 allows the user to type --version to activate this option. * Number 3 requires the user to type a single argument after the option. * Number 4 allows the user to use an option that takes a required argument and one or more optional ones * Number 5 Allows the user to either use -f or --format, which is followed by one required argument * and optionally 2 more arguments. */ const char *specification; /** * A textual description of the option that will be printed when the user asks for help. */ const char *description; }; #define CommandLineLastOption { 0, 0 } /** * The CommandLineParser singleton */ class CommandLineParser { public: static void init(int argc, char *argv[]); static void addOptionDefinitions(const CommandLineOption *definitions); static void setArgumentDefinition(const char *definition); static CommandLineParser *instance(); ~CommandLineParser(); void usage(const QString &name, const QString &argumentDescription = QString()); /// return the options that the user passed QStringList options() const; /** * returns true if the option was found. * Consider the following definition "--expert level" The user can type as an argument * "--expert 10". Calling contains("expert") will return true. * @see optionArgument() */ bool contains(const QString & key) const; /// returns the list of items that are not options, note that the first one is the name of the command called QStringList arguments() const; /// return the list of options that the user passed but we don't have a definition for. QStringList undefinedOptions() const; /** * Return the argument passed to an option. * Consider the following definition "--expert level" The user can type as an argument * "--expert 10". Calling optionArgument("expert") will return a string "10" * @see contains() */ QString optionArgument(const QString &optionName, const QString &defaultValue = QString()) const; QStringList optionArguments(const QString &optionName) const; QStringList parseErrors() const; private: CommandLineParser(int argc, char **argv); class Private; Private * const d; static CommandLineParser *self; }; #endif svn-all-fast-export-1.0.10/src/main.cpp000066400000000000000000000233751214616074400176550ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include #include #include #include #include "CommandLineParser.h" #include "ruleparser.h" #include "repository.h" #include "svn.h" QHash loadIdentityMapFile(const QString &fileName) { QHash result; if (fileName.isEmpty()) return result; QFile file(fileName); if (!file.open(QIODevice::ReadOnly)) { fprintf(stderr, "Could not open file %s: %s", qPrintable(fileName), qPrintable(file.errorString())); return result; } while (!file.atEnd()) { QByteArray line = file.readLine(); int comment_pos = line.indexOf('#'); if (comment_pos != -1) line.truncate(comment_pos); line = line.trimmed(); int space = line.indexOf(' '); if (space == -1) continue; // invalid line // Support git-svn author files, too // - svn2git native: loginname Joe User // - git-svn: loginname = Joe User int rightspace = line.indexOf(" = "); int leftspace = space; if (rightspace == -1) { rightspace = space; } else { leftspace = rightspace; rightspace += 2; } QByteArray realname = line.mid(rightspace).trimmed(); line.truncate(leftspace); result.insert(line, realname); }; file.close(); return result; } QSet loadRevisionsFile( const QString &fileName, Svn &svn ) { QRegExp revint("(\\d+)\\s*(?:-\\s*(\\d+|HEAD))?"); QSet revisions; if(fileName.isEmpty()) return revisions; QFile file(fileName); if( !file.open(QIODevice::ReadOnly)) { fprintf(stderr, "Could not open file %s: %s\n", qPrintable(fileName), qPrintable(file.errorString())); return revisions; } bool ok; while(!file.atEnd()) { QByteArray line = file.readLine().trimmed(); revint.indexIn(line); if( revint.cap(2).isEmpty() ) { int rev = revint.cap(1).toInt(&ok); if(ok) { revisions.insert(rev); } else { fprintf(stderr, "Unable to convert %s to int, skipping revision.\n", qPrintable(QString(line))); } } else if( revint.captureCount() == 2 ) { int rev = revint.cap(1).toInt(&ok); if(!ok) { fprintf(stderr, "Unable to convert %s (%s) to int, skipping revisions.\n", qPrintable(revint.cap(1)), qPrintable(QString(line))); continue; } int lastrev = 0; if(revint.cap(2) == "HEAD") { lastrev = svn.youngestRevision(); ok = true; } else { lastrev = revint.cap(2).toInt(&ok); } if(!ok) { fprintf(stderr, "Unable to convert %s (%s) to int, skipping revisions.\n", qPrintable(revint.cap(2)), qPrintable(QString(line))); continue; } for(; rev <= lastrev; ++rev ) revisions.insert(rev); } else { fprintf(stderr, "Unable to convert %s to int, skipping revision.\n", qPrintable(QString(line))); } } file.close(); return revisions; } static const CommandLineOption options[] = { {"--identity-map FILENAME", "provide map between svn username and email"}, {"--identity-domain DOMAIN", "provide user domain if no map was given"}, {"--revisions-file FILENAME", "provide a file with revision number that should be processed"}, {"--rules FILENAME[,FILENAME]", "the rules file(s) that determines what goes where"}, {"--add-metadata", "if passed, each git commit will have svn commit info"}, {"--add-metadata-notes", "if passed, each git commit will have notes with svn commit info"}, {"--resume-from revision", "start importing at svn revision number"}, {"--max-rev revision", "stop importing at svn revision number"}, {"--dry-run", "don't actually write anything"}, {"--debug-rules", "print what rule is being used for each file"}, {"--commit-interval NUMBER", "if passed the cache will be flushed to git every NUMBER of commits"}, {"--stats", "after a run print some statistics about the rules"}, {"--svn-branches", "Use the contents of SVN when creating branches, Note: SVN tags are branches as well"}, {"-h, --help", "show help"}, {"-v, --version", "show version"}, CommandLineLastOption }; int main(int argc, char **argv) { printf("Invoked as:'"); for(int i = 0; i < argc; ++i) printf(" %s", argv[i]); printf("'\n"); CommandLineParser::init(argc, argv); CommandLineParser::addOptionDefinitions(options); Stats::init(); CommandLineParser *args = CommandLineParser::instance(); if(args->contains(QLatin1String("version"))) { printf("Git version: %s\n", VER); return 0; } if (args->contains(QLatin1String("help")) || args->arguments().count() != 1) { args->usage(QString(), "[Path to subversion repo]"); return 0; } if (args->undefinedOptions().count()) { QTextStream out(stderr); out << "svn-all-fast-export failed: "; bool first = true; foreach (QString option, args->undefinedOptions()) { if (!first) out << " : "; out << "unrecognized option or missing argument for; `" << option << "'" << endl; first = false; } return 10; } if (!args->contains("rules")) { QTextStream out(stderr); out << "svn-all-fast-export failed: please specify the rules using the 'rules' argument\n"; return 11; } if (!args->contains("identity-map") && !args->contains("identity-domain")) { QTextStream out(stderr); out << "WARNING; no identity-map or -domain specified, all commits will use default @localhost email address\n\n"; } QCoreApplication app(argc, argv); // Load the configuration RulesList rulesList(args->optionArgument(QLatin1String("rules"))); rulesList.load(); int resume_from = args->optionArgument(QLatin1String("resume-from")).toInt(); int max_rev = args->optionArgument(QLatin1String("max-rev")).toInt(); // create the repository list QHash repositories; int cutoff = resume_from ? resume_from : INT_MAX; retry: int min_rev = 1; foreach (Rules::Repository rule, rulesList.allRepositories()) { Repository *repo = new Repository(rule); if (!repo) return EXIT_FAILURE; repositories.insert(rule.name, repo); int repo_next = repo->setupIncremental(cutoff); /* * cutoff < resume_from => error exit eventually * repo_next == cutoff => probably truncated log */ if (cutoff < resume_from && repo_next == cutoff) /* * Restore the log file so we fail the next time * svn2git is invoked with the same arguments */ repo->restoreLog(); if (cutoff < min_rev) /* * We've rewound before the last revision of some * repository that we've already seen. Start over * from the beginning. (since cutoff is decreasing, * we're sure we'll make forward progress eventually) */ goto retry; if (min_rev < repo_next) min_rev = repo_next; } if (cutoff < resume_from) { qCritical() << "Cannot resume from" << resume_from << "as there are errors in revision" << cutoff; return EXIT_FAILURE; } if (min_rev < resume_from) qDebug() << "skipping revisions" << min_rev << "to" << resume_from - 1 << "as requested"; if (resume_from) min_rev = resume_from; Svn::initialize(); Svn svn(args->arguments().first()); svn.setMatchRules(rulesList.allMatchRules()); svn.setRepositories(repositories); svn.setIdentityMap(loadIdentityMapFile(args->optionArgument("identity-map"))); // Massage user input a little, no guarantees that input makes sense. QString domain = args->optionArgument("identity-domain").simplified().remove(QChar('@')); if (domain.isEmpty()) domain = QString("localhost"); svn.setIdentityDomain(domain); if (max_rev < 1) max_rev = svn.youngestRevision(); bool errors = false; QSet revisions = loadRevisionsFile(args->optionArgument(QLatin1String("revisions-file")), svn); const bool filerRevisions = !revisions.isEmpty(); for (int i = min_rev; i <= max_rev; ++i) { if(filerRevisions) { if( !revisions.contains(i) ) { printf("."); continue; } else { printf("\n"); } } if (!svn.exportRevision(i)) { errors = true; break; } } foreach (Repository *repo, repositories) { repo->finalizeTags(); delete repo; } Stats::instance()->printStats(); return errors ? EXIT_FAILURE : EXIT_SUCCESS; } svn-all-fast-export-1.0.10/src/repository.cpp000066400000000000000000000645071214616074400211520ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * Copyright (C) 2009 Thomas Zander * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "repository.h" #include "CommandLineParser.h" #include #include #include #include #include static const int maxSimultaneousProcesses = 100; static const int maxMark = (1 << 20) - 2; // some versions of git-fast-import are buggy for larger values of maxMark class ProcessCache: QLinkedList { public: void touch(Repository *repo) { remove(repo); // if the cache is too big, remove from the front while (size() >= maxSimultaneousProcesses) takeFirst()->closeFastImport(); // append to the end append(repo); } inline void remove(Repository *repo) { #if QT_VERSION >= 0x040400 removeOne(repo); #else removeAll(repo); #endif } }; static ProcessCache processCache; static QString marksFileName(QString name) { name.replace('/', '_'); name.prepend("marks-"); return name; } Repository::Repository(const Rules::Repository &rule) : name(rule.name), prefix(rule.forwardTo), fastImport(name), commitCount(0), outstandingTransactions(0), last_commit_mark(0), next_file_mark(maxMark), processHasStarted(false) { foreach (Rules::Repository::Branch branchRule, rule.branches) { Branch branch; branch.created = 1; branches.insert(branchRule.name, branch); } // create the default branch branches["master"].created = 1; fastImport.setWorkingDirectory(name); if (!CommandLineParser::instance()->contains("dry-run")) { if (!QDir(name).exists()) { // repo doesn't exist yet. qDebug() << "Creating new repository" << name; QDir::current().mkpath(name); QProcess init; init.setWorkingDirectory(name); init.start("git", QStringList() << "--bare" << "init"); init.waitForFinished(-1); // Write description if (!rule.description.isEmpty()) { QFile fDesc(QDir(name).filePath("description")); if (fDesc.open(QIODevice::WriteOnly | QIODevice::Truncate | QIODevice::Text)) { fDesc.write(rule.description.toUtf8()); fDesc.putChar('\n'); fDesc.close(); } } { QFile marks(name + "/" + marksFileName(name)); marks.open(QIODevice::WriteOnly); marks.close(); } } } } static QString logFileName(QString name) { name.replace('/', '_'); name.prepend("log-"); return name; } static int lastValidMark(QString name) { QFile marksfile(name + "/" + marksFileName(name)); if (!marksfile.open(QIODevice::ReadOnly)) return 0; int prev_mark = 0; int lineno = 0; while (!marksfile.atEnd()) { QString line = marksfile.readLine(); ++lineno; if (line.isEmpty()) continue; int mark = 0; if (line[0] == ':') { int sp = line.indexOf(' '); if (sp != -1) { QString m = line.mid(1, sp-1); mark = m.toInt(); } } if (!mark) { qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?"; return 0; } if (mark == prev_mark) { qCritical() << marksfile.fileName() << "line" << lineno << "marks file has duplicates"; return 0; } if (mark < prev_mark) { qCritical() << marksfile.fileName() << "line" << lineno << "marks file not sorted"; return 0; } if (mark > prev_mark + 1) break; prev_mark = mark; } return prev_mark; } int Repository::setupIncremental(int &cutoff) { QFile logfile(logFileName(name)); if (!logfile.exists()) return 1; logfile.open(QIODevice::ReadWrite); QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)"); int last_valid_mark = lastValidMark(name); int last_revnum = 0; qint64 pos = 0; int retval = 0; QString bkup = logfile.fileName() + ".old"; while (!logfile.atEnd()) { pos = logfile.pos(); QByteArray line = logfile.readLine(); int hash = line.indexOf('#'); if (hash != -1) line.truncate(hash); line = line.trimmed(); if (line.isEmpty()) continue; if (!progress.exactMatch(line)) continue; int revnum = progress.cap(1).toInt(); QString branch = progress.cap(2); int mark = progress.cap(3).toInt(); if (revnum >= cutoff) goto beyond_cutoff; if (revnum < last_revnum) qWarning() << "WARN:" << name << "revision numbers are not monotonic: " << "got" << QString::number(last_revnum) << "and then" << QString::number(revnum); if (mark > last_valid_mark) { qWarning() << "WARN:" << name << "unknown commit mark found: rewinding -- did you hit Ctrl-C?"; cutoff = revnum; goto beyond_cutoff; } last_revnum = revnum; if (last_commit_mark < mark) last_commit_mark = mark; Branch &br = branches[branch]; if (!br.created || !mark || br.marks.isEmpty() || !br.marks.last()) br.created = revnum; br.commits.append(revnum); br.marks.append(mark); } retval = last_revnum + 1; if (retval == cutoff) /* * If a stale backup file exists already, remove it, so that * we don't confuse ourselves in 'restoreLog()' */ QFile::remove(bkup); return retval; beyond_cutoff: // backup file, since we'll truncate QFile::remove(bkup); logfile.copy(bkup); // truncate, so that we ignore the rest of the revisions qDebug() << name << "truncating history to revision" << cutoff; logfile.resize(pos); return cutoff; } void Repository::restoreLog() { QString file = logFileName(name); QString bkup = file + ".old"; if (!QFile::exists(bkup)) return; QFile::remove(file); QFile::rename(bkup, file); } Repository::~Repository() { Q_ASSERT(outstandingTransactions == 0); closeFastImport(); } void Repository::closeFastImport() { if (fastImport.state() != QProcess::NotRunning) { fastImport.write("checkpoint\n"); fastImport.waitForBytesWritten(-1); fastImport.closeWriteChannel(); if (!fastImport.waitForFinished()) { fastImport.terminate(); if (!fastImport.waitForFinished(200)) qWarning() << "WARN: git-fast-import for repository" << name << "did not die"; } } processHasStarted = false; processCache.remove(this); } void Repository::reloadBranches() { bool reset_notes = false; foreach (QString branch, branches.keys()) { Branch &br = branches[branch]; if (br.marks.isEmpty() || !br.marks.last()) continue; reset_notes = true; QByteArray branchRef = branch.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); fastImport.write("reset " + branchRef + "\nfrom :" + QByteArray::number(br.marks.last()) + "\n\n" "progress Branch " + branchRef + " reloaded\n"); } if (reset_notes && CommandLineParser::instance()->contains("add-metadata-notes")) { fastImport.write("reset refs/notes/commits\nfrom :" + QByteArray::number(maxMark + 1) + "\n"); } } int Repository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc) { Branch &brFrom = branches[branchFrom]; if (!brFrom.created) return -1; if (brFrom.commits.isEmpty()) { return -1; } if (branchRevNum == brFrom.commits.last()) { return brFrom.marks.last(); } QVector::const_iterator it = qUpperBound(brFrom.commits, branchRevNum); if (it == brFrom.commits.begin()) { return 0; } int closestCommit = *--it; if (!branchFromDesc.isEmpty()) { branchFromDesc += " at r" + QByteArray::number(branchRevNum); if (closestCommit != branchRevNum) { branchFromDesc += " => r" + QByteArray::number(closestCommit); } } return brFrom.marks[it - brFrom.commits.begin()]; } int Repository::createBranch(const QString &branch, int revnum, const QString &branchFrom, int branchRevNum) { QByteArray branchFromDesc = "from branch " + branchFrom.toUtf8(); int mark = markFrom(branchFrom, branchRevNum, branchFromDesc); if (mark == -1) { qCritical() << branch << "in repository" << name << "is branching from branch" << branchFrom << "but the latter doesn't exist. Can't continue."; return EXIT_FAILURE; } QByteArray branchFromRef = ":" + QByteArray::number(mark); if (!mark) { qWarning() << "WARN:" << branch << "in repository" << name << "is branching but no exported commits exist in repository" << "creating an empty branch."; branchFromRef = branchFrom.toUtf8(); if (!branchFromRef.startsWith("refs/")) branchFromRef.prepend("refs/heads/"); branchFromDesc += ", deleted/unknown"; } qDebug() << "Creating branch:" << branch << "from" << branchFrom << "(" << branchRevNum << branchFromDesc << ")"; // Preserve note branches[branch].note = branches.value(branchFrom).note; return resetBranch(branch, revnum, mark, branchFromRef, branchFromDesc); } int Repository::deleteBranch(const QString &branch, int revnum) { static QByteArray null_sha(40, '0'); return resetBranch(branch, revnum, 0, null_sha, "delete"); } int Repository::resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment) { QByteArray branchRef = branch.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); Branch &br = branches[branch]; QByteArray backupCmd; if (br.created && br.created != revnum && !br.marks.isEmpty() && br.marks.last()) { QByteArray backupBranch; if ((comment == "delete") && branchRef.startsWith("refs/heads/")) backupBranch = "refs/tags/backups/" + branchRef.mid(11) + "@" + QByteArray::number(revnum); else backupBranch = "refs/backups/r" + QByteArray::number(revnum) + branchRef.mid(4); qWarning() << "WARN: backing up branch" << branch << "to" << backupBranch; backupCmd = "reset " + backupBranch + "\nfrom " + branchRef + "\n\n"; } br.created = revnum; br.commits.append(revnum); br.marks.append(mark); QByteArray cmd = "reset " + branchRef + "\nfrom " + resetTo + "\n\n" "progress SVN r" + QByteArray::number(revnum) + " branch " + branch.toUtf8() + " = :" + QByteArray::number(mark) + " # " + comment + "\n\n"; if(comment == "delete") deletedBranches.append(backupCmd).append(cmd); else resetBranches.append(backupCmd).append(cmd); return EXIT_SUCCESS; } void Repository::commit() { if (deletedBranches.isEmpty() && resetBranches.isEmpty()) { return; } startFastImport(); fastImport.write(deletedBranches); fastImport.write(resetBranches); deletedBranches.clear(); resetBranches.clear(); } Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix, int revnum) { if (!branches.contains(branch)) { qWarning() << "WARN: Transaction:" << branch << "is not a known branch in repository" << name << endl << "Going to create it automatically"; } Transaction *txn = new Transaction; txn->repository = this; txn->branch = branch.toUtf8(); txn->svnprefix = svnprefix.toUtf8(); txn->datetime = 0; txn->revnum = revnum; if ((++commitCount % CommandLineParser::instance()->optionArgument(QLatin1String("commit-interval"), QLatin1String("10000")).toInt()) == 0) { startFastImport(); // write everything to disk every 10000 commits fastImport.write("checkpoint\n"); qDebug() << "checkpoint!, marks file trunkated"; } outstandingTransactions++; return txn; } void Repository::forgetTransaction(Transaction *) { if (!--outstandingTransactions) next_file_mark = maxMark; } void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix, int revnum, const QByteArray &author, uint dt, const QByteArray &log) { QString tagName = ref; if (tagName.startsWith("refs/tags/")) tagName.remove(0, 10); if (!annotatedTags.contains(tagName)) printf("Creating annotated tag %s (%s)\n", qPrintable(tagName), qPrintable(ref)); else printf("Re-creating annotated tag %s\n", qPrintable(tagName)); AnnotatedTag &tag = annotatedTags[tagName]; tag.supportingRef = ref; tag.svnprefix = svnprefix.toUtf8(); tag.revnum = revnum; tag.author = author; tag.log = log; tag.dt = dt; } void Repository::finalizeTags() { if (annotatedTags.isEmpty()) return; printf("Finalising tags for %s...", qPrintable(name)); startFastImport(); QHash::ConstIterator it = annotatedTags.constBegin(); for ( ; it != annotatedTags.constEnd(); ++it) { const QString &tagName = it.key(); const AnnotatedTag &tag = it.value(); QByteArray message = tag.log; if (!message.endsWith('\n')) message += '\n'; if (CommandLineParser::instance()->contains("add-metadata")) message += "\n" + formatMetadataMessage(tag.svnprefix, tag.revnum, tagName.toUtf8()); { QByteArray branchRef = tag.supportingRef.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); QByteArray s = "progress Creating annotated tag " + tagName.toUtf8() + " from ref " + branchRef + "\n" + "tag " + tagName.toUtf8() + "\n" + "from " + branchRef + "\n" + "tagger " + tag.author + ' ' + QByteArray::number(tag.dt) + " +0000" + "\n" + "data " + QByteArray::number( message.length() ) + "\n"; fastImport.write(s); } fastImport.write(message); fastImport.putChar('\n'); if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); // Append note to the tip commit of the supporting ref. There is no // easy way to attach a note to the tag itself with fast-import. if (CommandLineParser::instance()->contains("add-metadata-notes")) { Repository::Transaction *txn = newTransaction(tag.supportingRef, tag.svnprefix, tag.revnum); txn->setAuthor(tag.author); txn->setDateTime(tag.dt); txn->commitNote(formatMetadataMessage(tag.svnprefix, tag.revnum, tagName.toUtf8()), true); delete txn; if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); } printf(" %s", qPrintable(tagName)); fflush(stdout); } while (fastImport.bytesToWrite()) if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); printf("\n"); } void Repository::startFastImport() { processCache.touch(this); if (fastImport.state() == QProcess::NotRunning) { if (processHasStarted) qFatal("git-fast-import has been started once and crashed?"); processHasStarted = true; // start the process QString marksFile = marksFileName(name); QStringList marksOptions; marksOptions << "--import-marks=" + marksFile; marksOptions << "--export-marks=" + marksFile; marksOptions << "--force"; fastImport.setStandardOutputFile(logFileName(name), QIODevice::Append); fastImport.setProcessChannelMode(QProcess::MergedChannels); if (!CommandLineParser::instance()->contains("dry-run")) { fastImport.start("git", QStringList() << "fast-import" << marksOptions); } else { fastImport.start("/bin/cat", QStringList()); } fastImport.waitForStarted(-1); reloadBranches(); } } QByteArray Repository::formatMetadataMessage(const QByteArray &svnprefix, int revnum, const QByteArray &tag) { QByteArray msg = "svn path=" + svnprefix + "; revision=" + QByteArray::number(revnum); if (!tag.isEmpty()) msg += "; tag=" + tag; msg += "\n"; return msg; } bool Repository::branchExists(const QString& branch) const { return branches.contains(branch); } const QByteArray Repository::branchNote(const QString& branch) const { return branches.value(branch).note; } void Repository::setBranchNote(const QString& branch, const QByteArray& noteText) { if (branches.contains(branch)) branches[branch].note = noteText; } Repository::Transaction::~Transaction() { repository->forgetTransaction(this); } void Repository::Transaction::setAuthor(const QByteArray &a) { author = a; } void Repository::Transaction::setDateTime(uint dt) { datetime = dt; } void Repository::Transaction::setLog(const QByteArray &l) { log = l; } void Repository::Transaction::noteCopyFromBranch(const QString &branchFrom, int branchRevNum) { if(branch == branchFrom) { qWarning() << "WARN: Cannot merge inside a branch"; return; } static QByteArray dummy; int mark = repository->markFrom(branchFrom, branchRevNum, dummy); Q_ASSERT(dummy.isEmpty()); if (mark == -1) { qWarning() << "WARN:" << branch << "is copying from branch" << branchFrom << "but the latter doesn't exist. Continuing, assuming the files exist."; } else if (mark == 0) { qWarning() << "WARN: Unknown revision r" << QByteArray::number(branchRevNum) << ". Continuing, assuming the files exist."; } else { qWarning() << "WARN: repository " + repository->name + " branch " + branch + " has some files copied from " + branchFrom + "@" + QByteArray::number(branchRevNum); if (!merges.contains(mark)) { merges.append(mark); qDebug() << "adding" << branchFrom + "@" + QByteArray::number(branchRevNum) << ":" << mark << "as a merge point"; } else { qDebug() << "merge point already recorded"; } } } void Repository::Transaction::deleteFile(const QString &path) { QString pathNoSlash = repository->prefix + path; if(pathNoSlash.endsWith('/')) pathNoSlash.chop(1); deletedFiles.append(pathNoSlash); } QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length) { int mark = repository->next_file_mark--; // in case the two mark allocations meet, we might as well just abort Q_ASSERT(mark > repository->last_commit_mark + 1); if (modifiedFiles.capacity() == 0) modifiedFiles.reserve(2048); modifiedFiles.append("M "); modifiedFiles.append(QByteArray::number(mode, 8)); modifiedFiles.append(" :"); modifiedFiles.append(QByteArray::number(mark)); modifiedFiles.append(' '); modifiedFiles.append(repository->prefix + path.toUtf8()); modifiedFiles.append("\n"); if (!CommandLineParser::instance()->contains("dry-run")) { repository->startFastImport(); repository->fastImport.writeNoLog("blob\nmark :"); repository->fastImport.writeNoLog(QByteArray::number(mark)); repository->fastImport.writeNoLog("\ndata "); repository->fastImport.writeNoLog(QByteArray::number(length)); repository->fastImport.writeNoLog("\n", 1); } return &repository->fastImport; } void Repository::Transaction::commitNote(const QByteArray ¬eText, bool append, const QByteArray &commit) { QByteArray branchRef = branch; if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); const QByteArray &commitRef = commit.isNull() ? branchRef : commit; QByteArray message = "Adding Git note for current " + commitRef + "\n"; QByteArray text = noteText; if (append && commit.isNull() && repository->branchExists(branch) && !repository->branchNote(branch).isEmpty()) { text = repository->branchNote(branch) + text; message = "Appending Git note for current " + commitRef + "\n"; } QByteArray s(""); s.append("commit refs/notes/commits\n"); s.append("mark :" + QByteArray::number(maxMark + 1) + "\n"); s.append("committer " + author + " " + QString::number(datetime) + " +0000" + "\n"); s.append("data " + QString::number(message.length()) + "\n"); s.append(message + "\n"); s.append("N inline " + commitRef + "\n"); s.append("data " + QString::number(text.length()) + "\n"); s.append(text + "\n"); repository->fastImport.write(s); if (commit.isNull()) { repository->setBranchNote(QString::fromUtf8(branch), text); } } void Repository::Transaction::commit() { repository->startFastImport(); // We might be tempted to use the SVN revision number as the fast-import commit mark. // However, a single SVN revision can modify multple branches, and thus lead to multiple // commits in the same repo. So, we need to maintain a separate commit mark counter. int mark = ++repository->last_commit_mark; // in case the two mark allocations meet, we might as well just abort Q_ASSERT(mark < repository->next_file_mark - 1); // create the commit message QByteArray message = log; if (!message.endsWith('\n')) message += '\n'; if (CommandLineParser::instance()->contains("add-metadata")) message += "\n" + Repository::formatMetadataMessage(svnprefix, revnum); int parentmark = 0; Branch &br = repository->branches[branch]; if (br.created && !br.marks.isEmpty() && br.marks.last()) { parentmark = br.marks.last(); } else { qWarning() << "WARN: Branch" << branch << "in repository" << repository->name << "doesn't exist at revision" << revnum << "-- did you resume from the wrong revision?"; br.created = revnum; } br.commits.append(revnum); br.marks.append(mark); QByteArray branchRef = branch; if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); QByteArray s(""); s.append("commit " + branchRef + "\n"); s.append("mark :" + QByteArray::number(mark) + "\n"); s.append("committer " + author + " " + QString::number(datetime).toUtf8() + " +0000" + "\n"); s.append("data " + QString::number(message.length()) + "\n"); s.append(message + "\n"); repository->fastImport.write(s); // note some of the inferred merges QByteArray desc = ""; int i = !!parentmark; // if parentmark != 0, there's at least one parent if(log.contains("This commit was manufactured by cvs2svn") && merges.count() > 1) { qSort(merges); repository->fastImport.write("merge :" + QByteArray::number(merges.last()) + "\n"); merges.pop_back(); qWarning() << "WARN: Discarding all but the highest merge point as a workaround for cvs2svn created branch/tag" << "Discarded marks:" << merges; } else { foreach (const int merge, merges) { if (merge == parentmark) { qDebug() << "Skipping marking" << merge << "as a merge point as it matches the parent"; continue; } if (++i > 16) { // FIXME: options: // (1) ignore the 16 parent limit // (2) don't emit more than 16 parents // (3) create another commit on branch to soak up additional parents // we've chosen option (2) for now, since only artificial commits // created by cvs2svn seem to have this issue qWarning() << "WARN: too many merge parents"; break; } QByteArray m = " :" + QByteArray::number(merge); desc += m; repository->fastImport.write("merge" + m + "\n"); } } // write the file deletions if (deletedFiles.contains("")) repository->fastImport.write("deleteall\n"); else foreach (QString df, deletedFiles) repository->fastImport.write("D " + df.toUtf8() + "\n"); // write the file modifications repository->fastImport.write(modifiedFiles); repository->fastImport.write("\nprogress SVN r" + QByteArray::number(revnum) + " branch " + branch + " = :" + QByteArray::number(mark) + (desc.isEmpty() ? "" : " # merge from") + desc + "\n\n"); printf(" %d modifications from SVN %s to %s/%s", deletedFiles.count() + modifiedFiles.count('\n'), svnprefix.data(), qPrintable(repository->name), branch.data()); // Commit metadata note if requested if (CommandLineParser::instance()->contains("add-metadata-notes")) commitNote(Repository::formatMetadataMessage(svnprefix, revnum), false); while (repository->fastImport.bytesToWrite()) if (!repository->fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s for repository %s", qPrintable(repository->fastImport.errorString()), qPrintable(repository->name)); } svn-all-fast-export-1.0.10/src/repository.h000066400000000000000000000134451214616074400206120ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef REPOSITORY_H #define REPOSITORY_H #include #include #include #include #include "ruleparser.h" #include "CommandLineParser.h" class LoggingQProcess : public QProcess { QFile log; bool logging; public: LoggingQProcess(const QString filename) : QProcess(), log() { if(CommandLineParser::instance()->contains("debug-rules")) { logging = true; QString name = filename; name.replace('/', '_'); name.prepend("gitlog-"); log.setFileName(name); log.open(QIODevice::WriteOnly); } else { logging = false; } }; ~LoggingQProcess() { if(logging) { log.close(); } }; qint64 write(const char *data) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data); } qint64 write(const char *data, qint64 length) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data, length); } qint64 write(const QByteArray &data) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data); } qint64 writeNoLog(const char *data) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data); } qint64 writeNoLog(const char *data, qint64 length) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data, length); } qint64 writeNoLog(const QByteArray &data) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data); } bool putChar( char c) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.putChar(c); } return QProcess::putChar(c); } }; class Repository { public: class Transaction { Q_DISABLE_COPY(Transaction) friend class Repository; Repository *repository; QByteArray branch; QByteArray svnprefix; QByteArray author; QByteArray log; uint datetime; int revnum; QVector merges; QStringList deletedFiles; QByteArray modifiedFiles; inline Transaction() {} public: ~Transaction(); void commit(); void setAuthor(const QByteArray &author); void setDateTime(uint dt); void setLog(const QByteArray &log); void noteCopyFromBranch (const QString &prevbranch, int revFrom); void deleteFile(const QString &path); QIODevice *addFile(const QString &path, int mode, qint64 length); void commitNote(const QByteArray ¬eText, bool append, const QByteArray &commit = QByteArray()); }; Repository(const Rules::Repository &rule); int setupIncremental(int &cutoff); void restoreLog(); ~Repository(); void reloadBranches(); int createBranch(const QString &branch, int revnum, const QString &branchFrom, int revFrom); int deleteBranch(const QString &branch, int revnum); Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum); void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, const QByteArray &author, uint dt, const QByteArray &log); void finalizeTags(); void commit(); static QByteArray formatMetadataMessage(const QByteArray &svnprefix, int revnum, const QByteArray &tag = QByteArray()); bool branchExists(const QString& branch) const; const QByteArray branchNote(const QString& branch) const; void setBranchNote(const QString& branch, const QByteArray& noteText); private: struct Branch { int created; QVector commits; QVector marks; QByteArray note; }; struct AnnotatedTag { QString supportingRef; QByteArray svnprefix; QByteArray author; QByteArray log; uint dt; int revnum; }; QHash branches; QHash annotatedTags; QString name; QString prefix; LoggingQProcess fastImport; int commitCount; int outstandingTransactions; QByteArray deletedBranches; QByteArray resetBranches; /* starts at 0, and counts up. */ int last_commit_mark; /* starts at maxMark and counts down. Reset after each SVN revision */ int next_file_mark; bool processHasStarted; void startFastImport(); void closeFastImport(); // called when a transaction is deleted void forgetTransaction(Transaction *t); int resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment); int markFrom(const QString &branchFrom, int branchRevNum, QByteArray &desc); friend class ProcessCache; Q_DISABLE_COPY(Repository) }; #endif svn-all-fast-export-1.0.10/src/ruleparser.cpp000066400000000000000000000320141214616074400211030ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include #include "ruleparser.h" #include "CommandLineParser.h" RulesList::RulesList(const QString &filenames) : m_filenames(filenames) { } RulesList::~RulesList() {} void RulesList::load() { foreach(const QString filename, m_filenames.split(',') ) { qDebug() << "Loading rules from:" << filename; Rules *rules = new Rules(filename); m_rules.append(rules); rules->load(); m_allrepositories.append(rules->repositories()); QList matchRules = rules->matchRules(); m_allMatchRules.append( QList(matchRules)); } } const QList RulesList::allRepositories() const { return m_allrepositories; } const QList > RulesList::allMatchRules() const { return m_allMatchRules; } const QList RulesList::rules() const { return m_rules; } Rules::Rules(const QString &fn) : filename(fn) { } Rules::~Rules() { } const QList Rules::repositories() const { return m_repositories; } const QList Rules::matchRules() const { return m_matchRules; } Rules::Match::Substitution Rules::parseSubstitution(const QString &string) { if (string.at(0) != 's' || string.length() < 5) return Match::Substitution(); const QChar sep = string.at(1); if (string.at(string.length() - 1) != sep) return Match::Substitution(); int i = 2, end = 0; Match::Substitution subst; // Separator might have been escaped with a backslash while (i > end) { int backslashCount = 0; if ((end = string.indexOf(sep, i)) > -1) { for (i = end - 1; i >= 2; i--) { if (string.at(i) == '\\') backslashCount++; else break; } } else { return Match::Substitution(); // error } if (backslashCount % 2 != 0) { // Separator was escaped. Search for another one i = end + 1; } } // Found the end of the pattern subst.pattern = QRegExp(string.mid(2, end - 2)); if (!subst.pattern.isValid()) return Match::Substitution(); // error subst.replacement = string.mid(end + 1, string.length() - 1 - end - 1); return subst; } void Rules::load() { load(filename); } void Rules::load(const QString &filename) { qDebug() << "Loading rules from" << filename; // initialize the regexps we will use QRegExp repoLine("create repository\\s+(\\S+)", Qt::CaseInsensitive); QString varRegex("[A-Za-z0-9_]+"); QRegExp matchLine("match\\s+(.*)", Qt::CaseInsensitive); QRegExp matchActionLine("action\\s+(\\w+)", Qt::CaseInsensitive); QRegExp matchRepoLine("repository\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchDescLine("description\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchRepoSubstLine("substitute repository\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchBranchLine("branch\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchBranchSubstLine("substitute branch\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchRevLine("(min|max) revision (\\d+)", Qt::CaseInsensitive); QRegExp matchAnnotateLine("annotated\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchPrefixLine("prefix\\s+(.*)$", Qt::CaseInsensitive); QRegExp declareLine("declare\\s+("+varRegex+")\\s*=\\s*(\\S+)", Qt::CaseInsensitive); QRegExp variableLine("\\$\\{("+varRegex+")(\\|[^}$]*)?\\}", Qt::CaseInsensitive); QRegExp includeLine("include\\s+(.*)", Qt::CaseInsensitive); enum { ReadingNone, ReadingRepository, ReadingMatch } state = ReadingNone; Repository repo; Match match; int lineNumber = 0; QFile file(filename); if (!file.open(QIODevice::ReadOnly)) qFatal("Could not read the rules file: %s", qPrintable(filename)); QTextStream s(&file); QStringList lines = s.readAll().split('\n', QString::KeepEmptyParts); QStringList::iterator it; for(it = lines.begin(); it != lines.end(); ++it) { ++lineNumber; QString origLine = *it; QString line = origLine; int hash = line.indexOf('#'); if (hash != -1) line.truncate(hash); line = line.trimmed(); if (line.isEmpty()) continue; bool isIncludeRule = includeLine.exactMatch(line); if (isIncludeRule) { int index = filename.lastIndexOf("/"); QString includeFile = filename.left( index + 1) + includeLine.cap(1); load(includeFile); } else { while( variableLine.indexIn(line) != -1 ) { QString replacement; if (m_variables.contains(variableLine.cap(1))) { replacement = m_variables[variableLine.cap(1)]; } else { if (variableLine.cap(2).startsWith('|')) { replacement = variableLine.cap(2).mid(1); } else { qFatal("Undeclared variable: %s", qPrintable(variableLine.cap(1))); } } line = line.replace(variableLine.cap(0), replacement); } if (state == ReadingRepository) { if (matchBranchLine.exactMatch(line)) { Repository::Branch branch; branch.name = matchBranchLine.cap(1); repo.branches += branch; continue; } else if (matchDescLine.exactMatch(line)) { repo.description = matchDescLine.cap(1); continue; } else if (matchRepoLine.exactMatch(line)) { repo.forwardTo = matchRepoLine.cap(1); continue; } else if (matchPrefixLine.exactMatch(line)) { repo.prefix = matchPrefixLine.cap(1); continue; } else if (line == "end repository") { m_repositories += repo; { // clear out 'repo' Repository temp; std::swap(repo, temp); } state = ReadingNone; continue; } } else if (state == ReadingMatch) { if (matchRepoLine.exactMatch(line)) { match.repository = matchRepoLine.cap(1); continue; } else if (matchBranchLine.exactMatch(line)) { match.branch = matchBranchLine.cap(1); continue; } else if (matchRepoSubstLine.exactMatch(line)) { Match::Substitution subst = parseSubstitution(matchRepoSubstLine.cap(1)); if (!subst.isValid()) { qFatal("Malformed substitution in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } match.repo_substs += subst; continue; } else if (matchBranchSubstLine.exactMatch(line)) { Match::Substitution subst = parseSubstitution(matchBranchSubstLine.cap(1)); if (!subst.isValid()) { qFatal("Malformed substitution in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } match.branch_substs += subst; continue; } else if (matchRevLine.exactMatch(line)) { if (matchRevLine.cap(1) == "min") match.minRevision = matchRevLine.cap(2).toInt(); else // must be max match.maxRevision = matchRevLine.cap(2).toInt(); continue; } else if (matchPrefixLine.exactMatch(line)) { match.prefix = matchPrefixLine.cap(1); if( match.prefix.startsWith('/')) match.prefix = match.prefix.mid(1); continue; } else if (matchActionLine.exactMatch(line)) { QString action = matchActionLine.cap(1); if (action == "export") match.action = Match::Export; else if (action == "ignore") match.action = Match::Ignore; else if (action == "recurse") match.action = Match::Recurse; else qFatal("Invalid action \"%s\" on line %d", qPrintable(action), lineNumber); continue; } else if (matchAnnotateLine.exactMatch(line)) { match.annotate = matchAnnotateLine.cap(1) == "true"; continue; } else if (line == "end match") { if (!match.repository.isEmpty()) match.action = Match::Export; m_matchRules += match; Stats::instance()->addRule(match); state = ReadingNone; continue; } } bool isRepositoryRule = repoLine.exactMatch(line); bool isMatchRule = matchLine.exactMatch(line); bool isVariableRule = declareLine.exactMatch(line); if (isRepositoryRule) { // repository rule state = ReadingRepository; repo = Repository(); // clear repo.name = repoLine.cap(1); repo.lineNumber = lineNumber; repo.filename = filename; } else if (isMatchRule) { // match rule state = ReadingMatch; match = Match(); match.rx = QRegExp(matchLine.cap(1), Qt::CaseSensitive, QRegExp::RegExp2); if( !match.rx.isValid() ) qFatal("Malformed regular expression '%s' in file:'%s':%d, Error: %s", qPrintable(matchLine.cap(1)), qPrintable(filename), lineNumber, qPrintable(match.rx.errorString())); match.lineNumber = lineNumber; match.filename = filename; } else if (isVariableRule) { QString variable = declareLine.cap(1); QString value = declareLine.cap(2); m_variables.insert(variable, value); } else { qFatal("Malformed line in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } } } } Stats *Stats::self = 0; class Stats::Private { public: Private(); void printStats() const; void ruleMatched(const Rules::Match &rule, const int rev); void addRule(const Rules::Match &rule); private: QMap m_usedRules; }; Stats::Stats() : d(new Private()) { use = CommandLineParser::instance()->contains("stats"); } Stats::~Stats() { delete d; } void Stats::init() { if(self) delete self; self = new Stats(); } Stats* Stats::instance() { return self; } void Stats::printStats() const { if(use) d->printStats(); } void Stats::ruleMatched(const Rules::Match &rule, const int rev) { if(use) d->ruleMatched(rule, rev); } void Stats::addRule( const Rules::Match &rule) { if(use) d->addRule(rule); } Stats::Private::Private() { } void Stats::Private::printStats() const { printf("\nRule stats\n"); foreach(const QString name, m_usedRules.keys()) { printf("%s was matched %i times\n", qPrintable(name), m_usedRules[name]); } } void Stats::Private::ruleMatched(const Rules::Match &rule, const int rev) { Q_UNUSED(rev); const QString name = rule.info(); if(!m_usedRules.contains(name)) { m_usedRules.insert(name, 1); qWarning() << "WARN: New match rule, should have been added when created."; } else { m_usedRules[name]++; } } void Stats::Private::addRule( const Rules::Match &rule) { const QString name = rule.info(); if(m_usedRules.contains(name)) qWarning() << "WARN: Rule" << name << "was added multiple times."; m_usedRules.insert(name, 0); } #ifndef QT_NO_DEBUG_STREAM QDebug operator<<(QDebug s, const Rules::Match &rule) { s.nospace() << rule.info(); return s.space(); } #endif svn-all-fast-export-1.0.10/src/ruleparser.h000066400000000000000000000071021214616074400205500ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef RULEPARSER_H #define RULEPARSER_H #include #include #include #include #include #include class Rules { public: struct Rule { QString filename; int lineNumber; Rule() : lineNumber(0) {} }; struct Repository : Rule { struct Branch { QString name; }; QString name; QList branches; QString description; QString forwardTo; QString prefix; Repository() { } const QString info() const { const QString info = Rule::filename % ":" % QByteArray::number(Rule::lineNumber); return info; } }; struct Match : Rule { struct Substitution { QRegExp pattern; QString replacement; bool isValid() { return !pattern.isEmpty(); } QString& apply(QString &string) { return string.replace(pattern, replacement); } }; QRegExp rx; QString repository; QList repo_substs; QString branch; QList branch_substs; QString prefix; int minRevision; int maxRevision; bool annotate; enum Action { Ignore, Export, Recurse } action; Match() : minRevision(-1), maxRevision(-1), annotate(false), action(Ignore) { } const QString info() const { const QString info = Rule::filename % ":" % QByteArray::number(Rule::lineNumber) % " " % rx.pattern(); return info; } }; Rules(const QString &filename); ~Rules(); const QList repositories() const; const QList matchRules() const; Match::Substitution parseSubstitution(const QString &string); void load(); private: void load(const QString &filename); QString filename; QList m_repositories; QList m_matchRules; QMap m_variables; }; class RulesList { public: RulesList( const QString &filenames); ~RulesList(); const QList allRepositories() const; const QList > allMatchRules() const; const QList rules() const; void load(); private: QString m_filenames; QList m_rules; QList m_allrepositories; QList > m_allMatchRules; }; class Stats { public: static Stats *instance(); void printStats() const; void ruleMatched(const Rules::Match &rule, const int rev = -1); void addRule( const Rules::Match &rule); static void init(); ~Stats(); private: Stats(); class Private; Private * const d; static Stats *self; bool use; }; #ifndef QT_NO_DEBUG_STREAM class QDebug; QDebug operator<<(QDebug, const Rules::Match &); #endif #endif svn-all-fast-export-1.0.10/src/src.pro000066400000000000000000000021571214616074400175310ustar00rootroot00000000000000###################################################################### # Automatically generated by qmake (2.01a) dim. dc. 23 13:49:28 2007 ###################################################################### SVN_INCLUDE = /usr/include/subversion-1 /usr/local/include/subversion-1 APR_INCLUDE = /usr/include/apr-1.0 /usr/include/apr-1 /usr/local/include/apr-1 exists(local-config.pri):include(local-config.pri) VERSION = $$system(git --no-pager show --pretty=oneline --no-notes | head -1 | cut -b-40) !isEmpty(VERSION){ VERSION = $${VERSION} } VERSTR = '\\"$${VERSION}\\"' # place quotes around the version string DEFINES += VER=\"$${VERSTR}\" # create a VER macro containing the version string TEMPLATE = app TARGET = ../svn-all-fast-export DEPENDPATH += . QT = core INCLUDEPATH += . $$SVN_INCLUDE $$APR_INCLUDE !isEmpty($$SVN_LIBDIR): LIBS += -L$$SVN_LIBDIR LIBS += -lsvn_fs-1 -lsvn_repos-1 -lapr-1 -lsvn_subr-1 # Input SOURCES += ruleparser.cpp \ repository.cpp \ svn.cpp \ main.cpp \ CommandLineParser.cpp \ HEADERS += ruleparser.h \ repository.h \ svn.h \ CommandLineParser.h \ svn-all-fast-export-1.0.10/src/svn.cpp000066400000000000000000000777121214616074400175430ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ /* * Based on svn-fast-export by Chris Lee * License: MIT * URL: git://repo.or.cz/fast-import.git http://repo.or.cz/w/fast-export.git */ #define _XOPEN_SOURCE #define _LARGEFILE_SUPPORT #define _LARGEFILE64_SUPPORT #include "svn.h" #include "CommandLineParser.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "repository.h" #undef SVN_ERR #define SVN_ERR(expr) SVN_INT_ERR(expr) typedef QList MatchRuleList; typedef QHash RepositoryHash; typedef QHash IdentityHash; class AprAutoPool { apr_pool_t *pool; AprAutoPool(const AprAutoPool &); AprAutoPool &operator=(const AprAutoPool &); public: inline AprAutoPool(apr_pool_t *parent = NULL) { pool = svn_pool_create(parent); } inline ~AprAutoPool() { svn_pool_destroy(pool); } inline void clear() { svn_pool_clear(pool); } inline apr_pool_t *data() const { return pool; } inline operator apr_pool_t *() const { return pool; } }; class SvnPrivate { public: QList allMatchRules; RepositoryHash repositories; IdentityHash identities; QString userdomain; SvnPrivate(const QString &pathToRepository); ~SvnPrivate(); int youngestRevision(); int exportRevision(int revnum); int openRepository(const QString &pathToRepository); private: AprAutoPool global_pool; svn_fs_t *fs; svn_revnum_t youngest_rev; }; void Svn::initialize() { // initialize APR or exit if (apr_initialize() != APR_SUCCESS) { fprintf(stderr, "You lose at apr_initialize().\n"); exit(1); } // static destructor static struct Destructor { ~Destructor() { apr_terminate(); } } destructor; } Svn::Svn(const QString &pathToRepository) : d(new SvnPrivate(pathToRepository)) { } Svn::~Svn() { delete d; } void Svn::setMatchRules(const QList &allMatchRules) { d->allMatchRules = allMatchRules; } void Svn::setRepositories(const RepositoryHash &repositories) { d->repositories = repositories; } void Svn::setIdentityMap(const IdentityHash &identityMap) { d->identities = identityMap; } void Svn::setIdentityDomain(const QString &identityDomain) { d->userdomain = identityDomain; } int Svn::youngestRevision() { return d->youngestRevision(); } bool Svn::exportRevision(int revnum) { return d->exportRevision(revnum) == EXIT_SUCCESS; } SvnPrivate::SvnPrivate(const QString &pathToRepository) : global_pool(NULL) { if( openRepository(pathToRepository) != EXIT_SUCCESS) { qCritical() << "Failed to open repository"; exit(1); } // get the youngest revision svn_fs_youngest_rev(&youngest_rev, fs, global_pool); } SvnPrivate::~SvnPrivate() { svn_pool_destroy(global_pool); } int SvnPrivate::youngestRevision() { return youngest_rev; } int SvnPrivate::openRepository(const QString &pathToRepository) { svn_repos_t *repos; QString path = pathToRepository; while (path.endsWith('/')) // no trailing slash allowed path = path.mid(0, path.length()-1); SVN_ERR(svn_repos_open(&repos, QFile::encodeName(path), global_pool)); fs = svn_repos_fs(repos); return EXIT_SUCCESS; } enum RuleType { AnyRule = 0, NoIgnoreRule = 0x01, NoRecurseRule = 0x02 }; static MatchRuleList::ConstIterator findMatchRule(const MatchRuleList &matchRules, int revnum, const QString ¤t, int ruleMask = AnyRule) { MatchRuleList::ConstIterator it = matchRules.constBegin(), end = matchRules.constEnd(); for ( ; it != end; ++it) { if (it->minRevision > revnum) continue; if (it->maxRevision != -1 && it->maxRevision < revnum) continue; if (it->action == Rules::Match::Ignore && ruleMask & NoIgnoreRule) continue; if (it->action == Rules::Match::Recurse && ruleMask & NoRecurseRule) continue; if (it->rx.indexIn(current) == 0) { Stats::instance()->ruleMatched(*it, revnum); return it; } } // no match return end; } static void splitPathName(const Rules::Match &rule, const QString &pathName, QString *svnprefix_p, QString *repository_p, QString *branch_p, QString *path_p) { QString svnprefix = pathName; svnprefix.truncate(rule.rx.matchedLength()); if (svnprefix_p) { *svnprefix_p = svnprefix; } if (repository_p) { *repository_p = svnprefix; repository_p->replace(rule.rx, rule.repository); foreach (Rules::Match::Substitution subst, rule.repo_substs) { subst.apply(*repository_p); } } if (branch_p) { *branch_p = svnprefix; branch_p->replace(rule.rx, rule.branch); foreach (Rules::Match::Substitution subst, rule.branch_substs) { subst.apply(*branch_p); } } if (path_p) { QString prefix = svnprefix; prefix.replace(rule.rx, rule.prefix); *path_p = prefix + pathName.mid(svnprefix.length()); } } static int pathMode(svn_fs_root_t *fs_root, const char *pathname, apr_pool_t *pool) { svn_string_t *propvalue; SVN_ERR(svn_fs_node_prop(&propvalue, fs_root, pathname, "svn:executable", pool)); int mode = 0100644; if (propvalue) mode = 0100755; return mode; } svn_error_t *QIODevice_write(void *baton, const char *data, apr_size_t *len) { QIODevice *device = reinterpret_cast(baton); device->write(data, *len); while (device->bytesToWrite() > 32*1024) { if (!device->waitForBytesWritten(-1)) { qFatal("Failed to write to process: %s", qPrintable(device->errorString())); return svn_error_createf(APR_EOF, SVN_NO_ERROR, "Failed to write to process: %s", qPrintable(device->errorString())); } } return SVN_NO_ERROR; } static svn_stream_t *streamForDevice(QIODevice *device, apr_pool_t *pool) { svn_stream_t *stream = svn_stream_create(device, pool); svn_stream_set_write(stream, QIODevice_write); return stream; } static int dumpBlob(Repository::Transaction *txn, svn_fs_root_t *fs_root, const char *pathname, const QString &finalPathName, apr_pool_t *pool) { AprAutoPool dumppool(pool); // what type is it? int mode = pathMode(fs_root, pathname, dumppool); svn_filesize_t stream_length; SVN_ERR(svn_fs_file_length(&stream_length, fs_root, pathname, dumppool)); svn_stream_t *in_stream, *out_stream; if (!CommandLineParser::instance()->contains("dry-run")) { // open the file SVN_ERR(svn_fs_file_contents(&in_stream, fs_root, pathname, dumppool)); } // maybe it's a symlink? svn_string_t *propvalue; SVN_ERR(svn_fs_node_prop(&propvalue, fs_root, pathname, "svn:special", dumppool)); if (propvalue) { apr_size_t len = strlen("link "); if (!CommandLineParser::instance()->contains("dry-run")) { QByteArray buf; buf.reserve(len); SVN_ERR(svn_stream_read(in_stream, buf.data(), &len)); if (len == strlen("link ") && strncmp(buf, "link ", len) == 0) { mode = 0120000; stream_length -= len; } else { //this can happen if a link changed into a file in one commit qWarning("file %s is svn:special but not a symlink", pathname); // re-open the file as we tried to read "link " svn_stream_close(in_stream); SVN_ERR(svn_fs_file_contents(&in_stream, fs_root, pathname, dumppool)); } } } QIODevice *io = txn->addFile(finalPathName, mode, stream_length); if (!CommandLineParser::instance()->contains("dry-run")) { // open a generic svn_stream_t for the QIODevice out_stream = streamForDevice(io, dumppool); SVN_ERR(svn_stream_copy(in_stream, out_stream, dumppool)); svn_stream_close(out_stream); svn_stream_close(in_stream); // print an ending newline io->putChar('\n'); } return EXIT_SUCCESS; } static int recursiveDumpDir(Repository::Transaction *txn, svn_fs_root_t *fs_root, const QByteArray &pathname, const QString &finalPathName, apr_pool_t *pool) { // get the dir listing apr_hash_t *entries; SVN_ERR(svn_fs_dir_entries(&entries, fs_root, pathname, pool)); AprAutoPool dirpool(pool); // While we get a hash, put it in a map for sorted lookup, so we can // repeat the conversions and get the same git commit hashes. QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, entries); i; i = apr_hash_next(i)) { const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); svn_fs_dirent_t *dirent = reinterpret_cast(value); map.insertMulti(QByteArray(dirent->name), dirent->kind); } QMapIterator i(map); while (i.hasNext()) { dirpool.clear(); i.next(); QByteArray entryName = pathname + '/' + i.key(); QString entryFinalName = finalPathName + QString::fromUtf8(i.key()); if (i.value() == svn_node_dir) { entryFinalName += '/'; if (recursiveDumpDir(txn, fs_root, entryName, entryFinalName, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } else if (i.value() == svn_node_file) { printf("+"); fflush(stdout); if (dumpBlob(txn, fs_root, entryName, entryFinalName, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } } return EXIT_SUCCESS; } static bool wasDir(svn_fs_t *fs, int revnum, const char *pathname, apr_pool_t *pool) { AprAutoPool subpool(pool); svn_fs_root_t *fs_root; if (svn_fs_revision_root(&fs_root, fs, revnum, subpool) != SVN_NO_ERROR) return false; svn_boolean_t is_dir; if (svn_fs_is_dir(&is_dir, fs_root, pathname, subpool) != SVN_NO_ERROR) return false; return is_dir; } time_t get_epoch(const char* svn_date) { struct tm tm; memset(&tm, 0, sizeof tm); QByteArray date(svn_date, strlen(svn_date) - 8); strptime(date, "%Y-%m-%dT%H:%M:%S", &tm); return timegm(&tm); } class SvnRevision { public: AprAutoPool pool; QHash transactions; QList allMatchRules; RepositoryHash repositories; IdentityHash identities; QString userdomain; svn_fs_t *fs; svn_fs_root_t *fs_root; int revnum; // must call fetchRevProps first: QByteArray authorident; QByteArray log; uint epoch; bool ruledebug; bool propsFetched; bool needCommit; SvnRevision(int revision, svn_fs_t *f, apr_pool_t *parent_pool) : pool(parent_pool), fs(f), fs_root(0), revnum(revision), propsFetched(false) { ruledebug = CommandLineParser::instance()->contains( QLatin1String("debug-rules")); } int open() { SVN_ERR(svn_fs_revision_root(&fs_root, fs, revnum, pool)); return EXIT_SUCCESS; } int prepareTransactions(); int fetchRevProps(); int commit(); int exportEntry(const char *path, const svn_fs_path_change_t *change, apr_hash_t *changes); int exportDispatch(const char *path, const svn_fs_path_change_t *change, const char *path_from, svn_revnum_t rev_from, apr_hash_t *changes, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules, apr_pool_t *pool); int exportInternal(const char *path, const svn_fs_path_change_t *change, const char *path_from, svn_revnum_t rev_from, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules); int recurse(const char *path, const svn_fs_path_change_t *change, const char *path_from, const MatchRuleList &matchRules, svn_revnum_t rev_from, apr_hash_t *changes, apr_pool_t *pool); }; int SvnPrivate::exportRevision(int revnum) { SvnRevision rev(revnum, fs, global_pool); rev.allMatchRules = allMatchRules; rev.repositories = repositories; rev.identities = identities; rev.userdomain = userdomain; // open this revision: printf("Exporting revision %d ", revnum); fflush(stdout); if (rev.open() == EXIT_FAILURE) return EXIT_FAILURE; if (rev.prepareTransactions() == EXIT_FAILURE) return EXIT_FAILURE; if (!rev.needCommit) { printf(" nothing to do\n"); return EXIT_SUCCESS; // no changes? } if (rev.commit() == EXIT_FAILURE) return EXIT_FAILURE; printf(" done\n"); return EXIT_SUCCESS; } int SvnRevision::prepareTransactions() { // find out what was changed in this revision: apr_hash_t *changes; SVN_ERR(svn_fs_paths_changed(&changes, fs_root, pool)); QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, changes); i; i = apr_hash_next(i)) { const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); const char *key = reinterpret_cast(vkey); svn_fs_path_change_t *change = reinterpret_cast(value); // If we mix path deletions with path adds/replaces we might erase a // branch after that it has been reset -> history truncated if (map.contains(QByteArray(key))) { // If the same path is deleted and added, we need to put the // deletions into the map first, then the addition. if (change->change_kind == svn_fs_path_change_delete) { // XXX } fprintf(stderr, "\nDuplicate key found in rev %d: %s\n", revnum, key); fprintf(stderr, "This needs more code to be handled, file a bug report\n"); fflush(stderr); exit(1); } map.insertMulti(QByteArray(key), change); } QMapIterator i(map); while (i.hasNext()) { i.next(); if (exportEntry(i.key(), i.value(), changes) == EXIT_FAILURE) return EXIT_FAILURE; } return EXIT_SUCCESS; } int SvnRevision::fetchRevProps() { if( propsFetched ) return EXIT_SUCCESS; apr_hash_t *revprops; SVN_ERR(svn_fs_revision_proplist(&revprops, fs, revnum, pool)); svn_string_t *svnauthor = (svn_string_t*)apr_hash_get(revprops, "svn:author", APR_HASH_KEY_STRING); svn_string_t *svndate = (svn_string_t*)apr_hash_get(revprops, "svn:date", APR_HASH_KEY_STRING); svn_string_t *svnlog = (svn_string_t*)apr_hash_get(revprops, "svn:log", APR_HASH_KEY_STRING); log = svnlog->data; authorident = svnauthor ? identities.value(svnauthor->data) : QByteArray(); epoch = get_epoch(svndate->data); if (authorident.isEmpty()) { if (!svnauthor || svn_string_isempty(svnauthor)) authorident = "nobody "; else authorident = svnauthor->data + QByteArray(" <") + svnauthor->data + QByteArray("@") + userdomain.toUtf8() + QByteArray(">"); } propsFetched = true; return EXIT_SUCCESS; } int SvnRevision::commit() { // now create the commit if (fetchRevProps() != EXIT_SUCCESS) return EXIT_FAILURE; foreach (Repository *repo, repositories.values()) { repo->commit(); } foreach (Repository::Transaction *txn, transactions) { txn->setAuthor(authorident); txn->setDateTime(epoch); txn->setLog(log); txn->commit(); delete txn; } return EXIT_SUCCESS; } int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change, apr_hash_t *changes) { AprAutoPool revpool(pool.data()); QString current = QString::fromUtf8(key); // was this copied from somewhere? svn_revnum_t rev_from; const char *path_from; SVN_ERR(svn_fs_copied_from(&rev_from, &path_from, fs_root, key, revpool)); // is this a directory? svn_boolean_t is_dir; SVN_ERR(svn_fs_is_dir(&is_dir, fs_root, key, revpool)); if (is_dir) { if (change->change_kind == svn_fs_path_change_modify || change->change_kind == svn_fs_path_change_add) { if (path_from == NULL) { // freshly added directory, or modified properties // Git doesn't handle directories, so we don't either //qDebug() << " mkdir ignored:" << key; return EXIT_SUCCESS; } qDebug() << " " << key << "was copied from" << path_from << "rev" << rev_from; } else if (change->change_kind == svn_fs_path_change_replace) { if (path_from == NULL) qDebug() << " " << key << "was replaced"; else qDebug() << " " << key << "was replaced from" << path_from << "rev" << rev_from; } else if (change->change_kind == svn_fs_path_change_reset) { qCritical() << " " << key << "was reset, panic!"; return EXIT_FAILURE; } else { // if change_kind == delete, it shouldn't come into this arm of the 'is_dir' test qCritical() << " " << key << "has unhandled change kind " << change->change_kind << ", panic!"; return EXIT_FAILURE; } } else if (change->change_kind == svn_fs_path_change_delete) { is_dir = wasDir(fs, revnum - 1, key, revpool); } if (is_dir) current += '/'; //MultiRule: loop start //Replace all returns with continue, bool isHandled = false; foreach ( const MatchRuleList matchRules, allMatchRules ) { // find the first rule that matches this pathname MatchRuleList::ConstIterator match = findMatchRule(matchRules, revnum, current); if (match != matchRules.constEnd()) { const Rules::Match &rule = *match; if ( exportDispatch(key, change, path_from, rev_from, changes, current, rule, matchRules, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } else if (is_dir && path_from != NULL) { qDebug() << current << "is a copy-with-history, auto-recursing"; if ( recurse(key, change, path_from, matchRules, rev_from, changes, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } else if (is_dir && change->change_kind == svn_fs_path_change_delete) { qDebug() << current << "deleted, auto-recursing"; if ( recurse(key, change, path_from, matchRules, rev_from, changes, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } } if ( isHandled ) { return EXIT_SUCCESS; } if (wasDir(fs, revnum - 1, key, revpool)) { qDebug() << current << "was a directory; ignoring"; } else if (change->change_kind == svn_fs_path_change_delete) { qDebug() << current << "is being deleted but I don't know anything about it; ignoring"; } else { qCritical() << current << "did not match any rules; cannot continue"; return EXIT_FAILURE; } return EXIT_SUCCESS; } int SvnRevision::exportDispatch(const char *key, const svn_fs_path_change_t *change, const char *path_from, svn_revnum_t rev_from, apr_hash_t *changes, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules, apr_pool_t *pool) { //if(ruledebug) // qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.lineNumber << "(" << rule.rx.pattern() << ")"; switch (rule.action) { case Rules::Match::Ignore: //if(ruledebug) // qDebug() << " " << "ignoring."; return EXIT_SUCCESS; case Rules::Match::Recurse: if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "recursing."; return recurse(key, change, path_from, matchRules, rev_from, changes, pool); case Rules::Match::Export: if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "exporting."; if (exportInternal(key, change, path_from, rev_from, current, rule, matchRules) == EXIT_SUCCESS) return EXIT_SUCCESS; if (change->change_kind != svn_fs_path_change_delete) { if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "Unable to export non path removal."; return EXIT_FAILURE; } // we know that the default action inside recurse is to recurse further or to ignore, // either of which is reasonably safe for deletion qWarning() << "WARN: deleting unknown path" << current << "; auto-recursing"; return recurse(key, change, path_from, matchRules, rev_from, changes, pool); } // never reached return EXIT_FAILURE; } int SvnRevision::exportInternal(const char *key, const svn_fs_path_change_t *change, const char *path_from, svn_revnum_t rev_from, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules) { needCommit = true; QString svnprefix, repository, branch, path; splitPathName(rule, current, &svnprefix, &repository, &branch, &path); Repository *repo = repositories.value(repository, 0); if (!repo) { if (change->change_kind != svn_fs_path_change_delete) qCritical() << "Rule" << rule << "references unknown repository" << repository; return EXIT_FAILURE; } printf("."); fflush(stdout); // qDebug() << " " << qPrintable(current) << "rev" << revnum << "->" // << qPrintable(repository) << qPrintable(branch) << qPrintable(path); if (change->change_kind == svn_fs_path_change_delete && current == svnprefix && path.isEmpty()) { if(ruledebug) qDebug() << "repository" << repository << "branch" << branch << "deleted"; return repo->deleteBranch(branch, revnum); } QString previous; QString prevsvnprefix, prevrepository, prevbranch, prevpath; if (path_from != NULL) { previous = QString::fromUtf8(path_from); if (wasDir(fs, rev_from, path_from, pool.data())) { previous += '/'; } MatchRuleList::ConstIterator prevmatch = findMatchRule(matchRules, rev_from, previous, NoIgnoreRule); if (prevmatch != matchRules.constEnd()) { splitPathName(*prevmatch, previous, &prevsvnprefix, &prevrepository, &prevbranch, &prevpath); } else { qWarning() << "WARN: SVN reports a \"copy from\" @" << revnum << "from" << path_from << "@" << rev_from << "but no matching rules found! Ignoring copy, treating as a modification"; path_from = NULL; } } // current == svnprefix => we're dealing with the contents of the whole branch here if (path_from != NULL && current == svnprefix && path.isEmpty()) { if (previous != prevsvnprefix) { // source is not the whole of its branch qDebug() << qPrintable(current) << "is a partial branch of repository" << qPrintable(prevrepository) << "branch" << qPrintable(prevbranch) << "subdir" << qPrintable(prevpath); } else if (prevrepository != repository) { qWarning() << "WARN:" << qPrintable(current) << "rev" << revnum << "is a cross-repository copy (from repository" << qPrintable(prevrepository) << "branch" << qPrintable(prevbranch) << "path" << qPrintable(prevpath) << "rev" << rev_from << ")"; } else if (path != prevpath) { qDebug() << qPrintable(current) << "is a branch copy which renames base directory of all contents" << qPrintable(prevpath) << "to" << qPrintable(path); // FIXME: Handle with fast-import 'file rename' facility // ??? Might need special handling when path == / or prevpath == / } else { if (prevbranch == branch) { // same branch and same repository qDebug() << qPrintable(current) << "rev" << revnum << "is reseating branch" << qPrintable(branch) << "to an earlier revision" << qPrintable(previous) << "rev" << rev_from; } else { // same repository but not same branch // this means this is a plain branch qDebug() << qPrintable(repository) << ": branch" << qPrintable(branch) << "is branching from" << qPrintable(prevbranch); } if (repo->createBranch(branch, revnum, prevbranch, rev_from) == EXIT_FAILURE) return EXIT_FAILURE; if(CommandLineParser::instance()->contains("svn-branches")) { Repository::Transaction *txn = transactions.value(repository + branch, 0); if (!txn) { txn = repo->newTransaction(branch, svnprefix, revnum); if (!txn) return EXIT_FAILURE; transactions.insert(repository + branch, txn); } if(ruledebug) qDebug() << "Create a true SVN copy of branch (" << key << "->" << branch << path << ")"; txn->deleteFile(path); recursiveDumpDir(txn, fs_root, key, path, pool); } if (rule.annotate) { // create an annotated tag fetchRevProps(); repo->createAnnotatedTag(branch, svnprefix, revnum, authorident, epoch, log); } return EXIT_SUCCESS; } } Repository::Transaction *txn = transactions.value(repository + branch, 0); if (!txn) { txn = repo->newTransaction(branch, svnprefix, revnum); if (!txn) return EXIT_FAILURE; transactions.insert(repository + branch, txn); } // // If this path was copied from elsewhere, use it to infer _some_ // merge points. This heuristic is fairly useful for tracking // changes across directory re-organizations and wholesale branch // imports. // if (path_from != NULL && prevrepository == repository && prevbranch != branch) { if(ruledebug) qDebug() << "copy from branch" << prevbranch << "to branch" << branch << "@rev" << rev_from; txn->noteCopyFromBranch (prevbranch, rev_from); } if (change->change_kind == svn_fs_path_change_replace && path_from == NULL) { if(ruledebug) qDebug() << "replaced with empty path (" << branch << path << ")"; txn->deleteFile(path); } if (change->change_kind == svn_fs_path_change_delete) { if(ruledebug) qDebug() << "delete (" << branch << path << ")"; txn->deleteFile(path); } else if (!current.endsWith('/')) { if(ruledebug) qDebug() << "add/change file (" << key << "->" << branch << path << ")"; dumpBlob(txn, fs_root, key, path, pool); } else { if(ruledebug) qDebug() << "add/change dir (" << key << "->" << branch << path << ")"; txn->deleteFile(path); recursiveDumpDir(txn, fs_root, key, path, pool); } return EXIT_SUCCESS; } int SvnRevision::recurse(const char *path, const svn_fs_path_change_t *change, const char *path_from, const MatchRuleList &matchRules, svn_revnum_t rev_from, apr_hash_t *changes, apr_pool_t *pool) { svn_fs_root_t *fs_root = this->fs_root; if (change->change_kind == svn_fs_path_change_delete) SVN_ERR(svn_fs_revision_root(&fs_root, fs, revnum - 1, pool)); // get the dir listing svn_node_kind_t kind; SVN_ERR(svn_fs_check_path(&kind, fs_root, path, pool)); if(kind == svn_node_none) { qWarning() << "WARN: Trying to recurse using a nonexistant path" << path << ", ignoring"; return EXIT_SUCCESS; } else if(kind != svn_node_dir) { qWarning() << "WARN: Trying to recurse using a non-directory path" << path << ", ignoring"; return EXIT_SUCCESS; } apr_hash_t *entries; SVN_ERR(svn_fs_dir_entries(&entries, fs_root, path, pool)); AprAutoPool dirpool(pool); // While we get a hash, put it in a map for sorted lookup, so we can // repeat the conversions and get the same git commit hashes. QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, entries); i; i = apr_hash_next(i)) { dirpool.clear(); const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); svn_fs_dirent_t *dirent = reinterpret_cast(value); if (dirent->kind != svn_node_dir) continue; // not a directory, so can't recurse; skip map.insertMulti(QByteArray(dirent->name), dirent->kind); } QMapIterator i(map); while (i.hasNext()) { dirpool.clear(); i.next(); QByteArray entry = path + QByteArray("/") + i.key(); QByteArray entryFrom; if (path_from) entryFrom = path_from + QByteArray("/") + i.key(); // check if this entry is in the changelist for this revision already svn_fs_path_change_t *otherchange = (svn_fs_path_change_t*)apr_hash_get(changes, entry.constData(), APR_HASH_KEY_STRING); if (otherchange && otherchange->change_kind == svn_fs_path_change_add) { qDebug() << entry << "rev" << revnum << "is in the change-list, deferring to that one"; continue; } QString current = QString::fromUtf8(entry); if (i.value() == svn_node_dir) current += '/'; // find the first rule that matches this pathname MatchRuleList::ConstIterator match = findMatchRule(matchRules, revnum, current); if (match != matchRules.constEnd()) { if (exportDispatch(entry, change, entryFrom.isNull() ? 0 : entryFrom.constData(), rev_from, changes, current, *match, matchRules, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } else { if (i.value() == svn_node_dir) { qDebug() << current << "rev" << revnum << "did not match any rules; auto-recursing"; if (recurse(entry, change, entryFrom.isNull() ? 0 : entryFrom.constData(), matchRules, rev_from, changes, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } } } return EXIT_SUCCESS; } svn-all-fast-export-1.0.10/src/svn.h000066400000000000000000000025051214616074400171740ustar00rootroot00000000000000/* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef SVN_H #define SVN_H #include #include #include "ruleparser.h" class Repository; class SvnPrivate; class Svn { public: static void initialize(); Svn(const QString &pathToRepository); ~Svn(); void setMatchRules(const QList > &matchRules); void setRepositories(const QHash &repositories); void setIdentityMap(const QHash &identityMap); void setIdentityDomain(const QString &identityDomain); int youngestRevision(); bool exportRevision(int revnum); private: SvnPrivate * const d; }; #endif